diff --git a/README.md b/README.md index 61d9856..4f93e07 100644 --- a/README.md +++ b/README.md @@ -345,4 +345,4 @@ This project's logo features the Go Gopher mascot, [created by Renee French](htt | `k8s.io/api` | [LICENSE](https://github.com/kubernetes/api/blob/master/LICENSE) | | `k8s.io/apimachinery` | [LICENSE](https://github.com/kubernetes/apimachinery/blob/master/LICENSE) | | `k8s.io/client-go` | [LICENSE](https://github.com/kubernetes/client-go/blob/master/LICENSE) | - + \ No newline at end of file diff --git a/api/arke.pb.go b/api/arke.pb.go index 9195f64..df49f5d 100644 --- a/api/arke.pb.go +++ b/api/arke.pb.go @@ -232,7 +232,7 @@ type ConnectionConfiguration struct { state protoimpl.MessageState `protogen:"open.v1"` Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` // Broker hostname or IP address. Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` // Broker port. - Provider string `protobuf:"bytes,3,opt,name=provider,proto3" json:"provider,omitempty"` // Provider type, currently only amqp091. + Provider string `protobuf:"bytes,3,opt,name=provider,proto3" json:"provider,omitempty"` // Provider type, currently only amqp091, and natsjs. Tenant string `protobuf:"bytes,4,opt,name=tenant,proto3" json:"tenant,omitempty"` // Tenant name for this connection. Tenant is not required Credentials *Credentials `protobuf:"bytes,5,opt,name=credentials,proto3" json:"credentials,omitempty"` // Authentication credentials. Tls bool `protobuf:"varint,8,opt,name=tls,proto3" json:"tls,omitempty"` // Should this provider connection use TLS. diff --git a/api/protobuf-spec/arke.proto b/api/protobuf-spec/arke.proto index e107ed1..8e331fa 100644 --- a/api/protobuf-spec/arke.proto +++ b/api/protobuf-spec/arke.proto @@ -22,7 +22,7 @@ package arke; message ConnectionConfiguration { string host = 1; // Broker hostname or IP address. int32 port = 2; // Broker port. - string provider = 3; // Provider type, currently only amqp091. + string provider = 3; // Provider type, currently only amqp091, and natsjs. string tenant = 4; // Tenant name for this connection. Tenant is not required Credentials credentials = 5; // Authentication credentials. bool tls = 8; // Should this provider connection use TLS. diff --git a/doc/arke_protocol.md b/doc/arke_protocol.md index 5310118..3e0af8d 100644 --- a/doc/arke_protocol.md +++ b/doc/arke_protocol.md @@ -100,7 +100,7 @@ RabbitMQ and Kafka. | ----- | ---- | ----- | ----------- | | host | [string](#string) | | Broker hostname or IP address. | | port | [int32](#int32) | | Broker port. | -| provider | [string](#string) | | Provider type, currently only amqp091. | +| provider | [string](#string) | | Provider type, currently only amqp091, and natsjs. | | tenant | [string](#string) | | Tenant name for this connection. Tenant is not required | | credentials | [Credentials](#arke-Credentials) | | Authentication credentials. | | tls | [bool](#bool) | | Should this provider connection use TLS. | diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md new file mode 100644 index 0000000..1172614 --- /dev/null +++ b/doc/design/natsjs-connector.md @@ -0,0 +1,792 @@ +# NATS JetStream Connector + +## Purpose + +This document describes the `natsjs` connector, a backend provider that lets +Arke run against [NATS JetStream](https://docs.nats.io/nats-concepts/jetstream) +in place of (or alongside) RabbitMQ / AMQP 0.9.1. It is a worked example of +the contract in +[provider-connector-interface.md](provider-connector-interface.md): a second +`provider.Provider` registered under the name `natsjs` and selected per +connection via `ConnectionConfiguration.provider`. + +The connector covers the publish / subscribe / ack / delayed-retry / +dead-letter / dedup / header-filter paths an AMQP client drives through Arke. +It maps each onto a native JetStream primitive where one exists and onto a +small amount of stateless proxy-side translation where it does not. + +The guiding rule that keeps the translation honest: + +> Arke may do stateless translation and orchestration of broker primitives. It +> must not become the system of record. Persistence, routing, and HA stay in +> the broker; dead-letter orchestration, delayed-retry, retry-count +> bookkeeping, and header filtering may move into the proxy. + +## Subject and topology mapping + +AMQP routes via an exchange plus routing-key bindings. NATS routes via subjects +with token wildcards. The connector maps the two as follows +(`helpers.go:publishSubjectFor` / `filterSubjectsFor`): + +| AMQP concept | NATS mapping | +| --- | --- | +| exchange / address name | subject root (dots kept; it is a prefix) | +| — | `~` delimiter token, always inserted after the root | +| routing key | appended subject tokens | +| `#` (zero-or-more words) | `>` (NATS allows `>` only as the final token) | +| `*` (exactly one word) | `*` | + +A message published to address `events.orders` with routing key +`region.us.created` travels on subject `events.orders.~.region.us.created`; +an empty routing key maps to the bare prefix `events.orders.~`. A JetStream +stream named `arke_
` captures `
.~` and `
.~.>`, and +each consumer filters on the mapped source subjects. + +The stream for an address is asserted on use: a subscribe (or declare-only +call) creates it, and so does a publish — with one exception. A unary publish +to a `STREAM` address requires the stream to exist already, exactly as +amqp091's stream publisher refuses a stream nobody declared: declaring a +stream is its readers' job, and auto-creating it would turn a typo'd address +name into a junk stream storing messages no reader will ever see. The +streaming publish path does not check, matching amqp091, which sends every +address type over an auto-declared exchange without error. + +Stream names (and durable consumer names, which JetStream validates the same +way) may not contain `.`, whitespace, `*`, `>`, `/` or `\`, so they are +derived from the address (or source / consumer-group) name by swapping dots +for underscores: `events.orders` becomes `arke_events_orders`. That +replacement alone is ambiguous the moment a name contains a literal `_` — +`a.b` and `a_b` would both read `arke_a_b`, and two addresses colliding onto +one stream name silently reconfigure each other's stream — so such names +take an escaped form instead, under the disjoint `arke-` prefix, in which +every `_` starts a two-character escape code (`_d` for `.`, `_u` for `_`, +and codes for the other characters JetStream rejects in names). Distinct +names therefore always yield distinct streams and durables, while names free +of underscores keep the readable historical form. + +The `~` delimiter is what keeps distinct addresses' streams disjoint, which +JetStream requires: two subjects may belong to at most one stream. Address +names are themselves dotted, so two addresses can sit in a prefix +relationship (`events.orders` and `events.orders.filtered`); without the +delimiter their capture wildcards overlap, whichever stream is created first +wins, and the other address fails every publish and subscribe with `subjects +overlap with an existing stream` (err 10065). With it, the token after the +shared prefix differs (`~` vs `filtered`) and token escaping (below) +guarantees no address token ever equals `~`, so any two distinct roots are +disjoint. The delimiter also prevents cross-address message leakage — +without it, publishing routing key `filtered.x` to `events.orders` is +indistinguishable from publishing `x` to `events.orders.filtered`. + +NATS subjects are stricter than AMQP routing keys, so each address and +routing-key token is escaped — injectively, so distinct AMQP names never +merge onto one subject. A token free of reserved characters (`~`, +whitespace, `*`, `>`, `#`) passes through unchanged; every conventional +dotted name keeps its readable, historical form. Any other token takes an +escaped form marked by a leading `~` — no plain token can start with `~`, +since `~` is itself reserved — in which each reserved character becomes a +two-character `~` code (`~~` for `~`, `~w` space, `~t` tab, `~a` `*`, `~g` +`>`, `~h` `#`, plus codes for the remaining whitespace characters); an empty +address token (from consecutive dots) becomes `~e`. The escaped form decodes +unambiguously, which is what makes the mapping injective: a lossy +replacement (`_` for every illegal character) would merge distinct addresses +— `a.~.b`, `a..b`, `a.*.b` and `a._.b` — onto one root, and addresses that +share a root share a stream: each receives the other's traffic and each +ensure reconfigures the other's stream. The same escaping keeps distinct +routing keys distinct, so a binding on `a_b` no longer also matches +published keys `a b` or `a*b`. + +In binding patterns (`translateWildcards`): runs of consecutive `#` collapse +into one first (they are equivalent in AMQP, and `#.#` must keep the +zero-word match its trailing `#` provides), a non-terminal `#` becomes `*` +(NATS `>` is tail-only), literal tokens are escaped exactly like published +tokens (so a literal binding matches exactly the published keys it matches +on RabbitMQ), and empty tokens (from `a..b` or a trailing dot) are dropped — +on the publish side too, so both sides agree. A pattern whose trailing `#` +became `>` also gets the zero-word variant as a second filter subject (AMQP +`#` matches zero or more words, NATS `>` one or more, so binding `a.#` must +match routing key `a`). On the publish side routing keys are literal — AMQP +gives `*`/`#` meaning only in bindings, and NATS forbids wildcard tokens in +published subjects — so wildcard characters are escaped like any other +reserved character. For `Address_QUEUE` (AMQP direct-exchange parity), +binding subjects are exact routing keys rather than topic patterns: `#` and +`*` are escaped literals and match only themselves, instead of matching as +wildcards. + +A redundant binding set — a wildcard binding alongside a specific key it +already covers, such as `orders.#` with `orders.created` — is legal in AMQP, +where a message is routed to the queue once no matter how many bindings +match. JetStream instead rejects a consumer whose filter subjects overlap +(one being a subset of another), so the connector collapses the mapped +filters to the widest set before creating the consumer; the surviving +wildcard filter matches everything the dropped filters did. Intersecting +patterns where neither contains the other are kept as-is. + +A source whose address carries *no* subjects declares no bindings and so +receives nothing, matching amqp091 (which binds nothing at all in that case). +A JetStream consumer must carry at least one filter subject, so "nothing" is +expressed as a filter on a subject no published subject can contain — a token +the escaping above can never emit. Three cases are deliberately not that: + +- an empty binding key (`""`, as distinct from no keys) is a literal, and + selects the empty routing key only, exactly as it matches on a topic or + direct exchange; +- a `STREAM` source is not bound to its address at all — amqp091 reads a + RabbitMQ stream by name and never declares a binding — so it reads the + whole log; +- a headers address with filters selects the whole address, because a headers + exchange ignores routing keys and the header match decides. amqp091 binds a + single `""` key there purely to have somewhere to hang the header + arguments; `evaluateFilters` is this connector's stand-in for those. + +A headers address selects the whole address whether or not it declares +binding keys, for that same reason: a headers exchange matches every one of +its bindings on their header arguments alone and never reads the routing key, +so keys declared on one decide nothing and must not narrow the consumer. +They still travel to the parent in an address-to-address binding, where the +*parent's* type decides how they are matched — a topic parent matches them as +routing keys, a headers parent ignores them in turn. + +### Address-to-address binding + +An address may name a `ParentAddress`, binding it to that parent: what is +published to the parent under the child's binding keys is routed on to the +child, and reaches the child's own consumers. On a broker that routes, this is +a routing rule. Here the stream *is* the storage, so the equivalent is to +source the bound subjects out of the parent's stream into the child's +(`ensureStreamFor`), keeping the subject each message was published under +rather than transforming it. That means only the parent's stream ever listens +on the parent's subjects — no two streams claim one subject space — while a +message routed in from the parent is indistinguishable, to a consumer, from +one published to the child directly. The child's consumers filter on both +their address's subjects and the bound parent subjects. + +The binding keys are the child's own subjects, matched by the *parent* +exchange's type, so they translate exactly like any other binding. Bindings +accumulate: declaring one never removes another, as on RabbitMQ, so a second +subscriber binding a different key on the same address adds to the set rather +than replacing it. Nothing unbinds — a binding outlives the subscription that +declared it, as an AMQP exchange-to-exchange binding outlives the client that +bound it. That holds for assertions carrying no binding at all, which is the +ordinary case: a publisher names the bound address directly and so has no +parent to declare. Because JetStream replaces a stream's source set wholesale +on update, every assertion reads the current set and writes it back, rather +than sending only what the caller happens to know about. + +Reading and writing back makes an assertion a read-modify-write, and the lock +that serializes it is process-local while Arke runs more than one replica. So +an assertion writes only when it would actually change something: it compares +the live configuration against the one it built and returns early if the +stream already satisfies it. That takes the overwhelmingly common case — a +bare assertion against a steady-state stream, which is every publisher's +first touch and every reconnect — out of the race entirely, leaving only +callers that genuinely change the configuration, which for bindings are +symmetric (both are adding). A stream created before a limit existed is still +retrofitted: its configuration differs, so the update runs. Two concurrent +*binding* declarations across replicas remain a real, much narrower window; +closing it would need a compare-and-swap JetStream does not offer. + +This subject scheme — and the stream/durable name encoding above — is part +of the connector's persistence contract: retained messages are stored under +these subjects for the life of the stream's retention limits, so any future +change to the encoding is a breaking change for deployed data +(`CreateOrUpdateStream` moves the stream's captured subjects forward, after +which messages stored under the previous encoding no longer match any +consumer filter and age out; a renamed stream or durable simply strands the +old one and its state). Treat this layout as canonical: external tooling +that reads or writes JetStream subjects directly must use the same mapping, +including the `~` delimiter. Deployments whose address or source names +contain `_`, `/` or `\` predating the escaped name form had those names +colliding or rejected outright, so the escaped form changes only names that +were already broken. Likewise, addresses and routing keys containing +reserved subject characters (whitespace, `*`, `>`, `#`, `~`, consecutive +dots) predating token escaping either collided with each other's `_` +spellings or produced invalid subjects, so escaping changes the stored +subjects only where the previous mapping was already wrong; names and keys +made of conventional tokens map exactly as before. + +AMQP headers-exchange routing has no NATS subject equivalent. It is reproduced +proxy-side in `evaluateFilters`: multiple `Filter`s are OR'd (each is a +separate binding), and within a single filter the matches combine per +`Filter.Type` (`ALL` = and, `ANY` = or). + +## Consumers: durable vs ephemeral + +The connector chooses the consumer kind from the source +(`helpers.go:durableName`): + +- Durable (work-queue) consumers use a stable name and persist across client + reconnects, so a backlog published while the consumer is disconnected is + redelivered on reconnect, matching RabbitMQ durable-queue semantics. Used + for non-transient `QUEUE` sources, `STREAM` sources with a `ConsumerGroup`, + and any `SingleActiveConsumer`. +- Ephemeral consumers auto-expire after an inactivity threshold. Used for + auto-delete / exclusive / `TEMPORARY` sources, which clients commonly use + for per-instance, transient subscriptions — and for `STREAM` sources + without a `ConsumerGroup`, whose subscribers are independent readers of + the shared log, each positioned by its own `Offset` (RabbitMQ stream + consumers do not compete). The exception is a group-less `STREAM` source + asking for `Offset: continue`, which is a request to resume where that + source last stopped and so needs a position the broker keeps between + subscriptions: it gets a durable named after the source, the way amqp091 + answers `continue` from RabbitMQ Streams' server-side offset tracking + (keyed by consumer name). Every other offset positions the reader from the + log itself and stays ephemeral. They are also deleted eagerly + when their subscription ends (or never starts — a declare-only call, or a + failure to begin consuming), so the threshold only has to cover unclean + exits and churning transient clients cannot accumulate dead consumers + against the server's per-stream consumer limit. + +A source with `SingleActiveConsumer` additionally maps onto a pinned-client +priority group (nats-server 2.11+): the server pins the first subscriber to +pull and delivers only to it, standbys' pulls wait, and when the pinned +client stops pulling for `NATSJS_SAC_PINNED_TTL` the pin moves to a standby. +The durable such a source attaches to is named after its `ConsumerGroup` +option when set: the group is the identity the instances coordinate through +(amqp091 uses it as the single-active consumer reference the same way), so +sources that share one name but split their subscribers into groups — one +independently ordered group per partition of a shared stream — get one +pinned consumer per group instead of collapsing onto a single pin that +starves every other group. Without a group, a single-active queue source +falls back to its own name (all instances of a single-active queue share the +queue by definition), and a single-active stream source is rejected at +subscribe time, as in amqp091. +That reproduces RabbitMQ's single-active-consumer semantics — ordered +processing across competing instances with automatic failover — natively, +with two caveats. Failover is bounded by the pinned TTL rather than +instantaneous on disconnect, and the TTL must comfortably exceed the pull +re-issue cadence (~30s with the client defaults) or the pin flaps. And on a +server that predates priority groups the config fields are silently dropped, +so single-active cannot be enforced; the connector detects that from the +effective consumer config and logs a warning that consumers will compete. +Priority-group config is update-mutable, so a durable created before its +source set `SingleActiveConsumer` is upgraded in place, keeping its name and +ack position. The reverse transition needs an extra step: a durable coming +OFF `SingleActiveConsumer` has its pin released explicitly (`UnpinConsumer`) +before the update clears the priority-group config, because unpinning AFTER +the group is gone from the config is rejected by the server ("priority group +does not exist for this consumer"). Skipping that step leaves the previous +pin as server-side state independent of the declared config, silently +blocking ordinary delivery to the downgraded consumer for up to +`NATSJS_SAC_PINNED_TTL` with no error anywhere. + +When a subscription ends, its delivered-but-unresolved messages are released: +their acks could only have arrived on the consume stream that just closed, so +the connector drops its claim on them and (for durable consumers) naks them +so they redeliver promptly. This matches RabbitMQ, which requeues a closed +channel's unacked deliveries immediately; without the nak they would only +redeliver after the full ack wait. + +The `DeliverPolicy` is taken from the `Offset` option, mirroring the amqp091 +connector's offset vocabulary so both accept the same values: `first` -> +deliver all, `last` -> the final message, `next` (or unset) -> deliver new, +`continue` -> resume from the position the source's durable holds (deliver all +on its first creation, when there is no position yet — RabbitMQ answers a +`continue` with no stored offset the same way), and an absolute number -> +start at that offset. Any other value fails the subscribe (as in amqp091's +offset parsing) rather than silently starting the consumer at a different +position than it asked for. The offset only applies on first creation: +JetStream fixes a durable's start position when the consumer is created, so a +re-subscribe that requests a different offset logs a warning and resumes from +the durable's stored ack position — use a new durable (source or +`ConsumerGroup` name) to reposition. Only that start-position conflict is +absorbed; any other error creating or updating the consumer fails the +subscribe, since resuming a consumer whose configuration silently differs from +the requested one would consume the wrong way with no signal. + +A numeric offset counts from 0, naming a message's position in the log the way +a RabbitMQ Stream offset does — not the raw JetStream sequence, which counts +from 1. The connector converts in both directions, so an offset read from +`SourceStats` and handed back as a source's `Offset` names the same message it +did on either broker. The value is still not portable *across* brokers: offset +7 is the eighth message of whichever log is being read, and two brokers' logs +hold different messages. + +## Ack, retry, and dead-letter mapping + +The server translates a client's ack / nack / requeue into provider calls; the +connector maps those onto JetStream primitives: + +| Client action | Provider call | NATS JetStream primitive | +| --- | --- | --- | +| ack | `Ack` | `msg.Ack()` | +| nack, `requeue_delay > 0` | `Retry(delay)` | `msg.NakWithDelay(delay)` | +| nack, `requeue_delay == 0`, DLA set | `DeadLetter` | publish to DLQ + `Term()` | +| nack, `requeue_delay == 0`, no DLA | `Nack` | `msg.Term()` | + +Note which primitive a plain nack maps to. A nack on this contract is a +rejection, not a requeue: amqp091 answers the same call with +`Delivery.Nack(requeue=false)`, so RabbitMQ drops the message, or moves it to +the queue's dead-letter exchange — which the server here asks for explicitly +through `DeadLetter` instead. JetStream's `Nak` means the opposite, redeliver +now, so using it for a nack turns a single nacked message into an unbounded +delivery loop: a client that rejected a message rejects each redelivery +straight back, at thousands of deliveries a second for as long as the +subscription lives. `Term` carries the intended meaning — stop redelivering — +and `Retry` remains the way to ask for redelivery. The one exception is the +server's fallback nack after a failed `DeadLetter` (below), whose purpose is +to put the message back so dead-lettering can be retried; `DeadLetter` marks +the message before returning that error, and a nack of a marked message naks. + +A delivered message that is neither acked nor nacked redelivers after the +consumer's ack wait (`NATSJS_ACK_WAIT`, default 30s). That value sets one +dial between two failure modes: a crashed client's in-flight messages are +stuck until the ack wait passes before another consumer gets them (shorter +is better), while a healthy consumer that holds a message longer than the +ack wait — including time spent queued in the client-side pull buffer behind +a large prefetch — gets a duplicate redelivery (longer is better). The +default favors failover; RabbitMQ's equivalent consumer timeout defaults to +30 minutes, so deployments with legitimately slow consumers should raise it. + +`NakWithDelay` is the native replacement for RabbitMQ's per-message-TTL + +dead-letter retry-queue idiom; JetStream increments the delivery count, which +the connector surfaces back to the client as the `x-retry-count` header (see +below). JetStream has no native dead-letter exchange, so `DeadLetter` +republishes the message to the dead-letter address — under the message's +original routing key unless `DeadLetterSubject` overrides it, exactly as +RabbitMQ dead-letters under the original key unless +`x-dead-letter-routing-key` is set — and then `Term()`s it to stop +redelivery. RabbitMQ performs that move broker-side; +here it is two proxy-side steps, so ordering is what protects the data: the +original is terminated only after the DLQ publish succeeds. If the DLQ stream +cannot be ensured or published to, `DeadLetter` returns an error and leaves +the message ack-pending — the server then falls back to a nack, so the +message is redelivered and dead-lettering is retried instead of the message +being lost. A present but empty `DeadLetterAddress` is treated the same way: +the connector returns an error and leaves the original in flight rather than +terminating it without a DLQ publish. + +That fallback nack is delayed by `NATSJS_DEADLETTER_RETRY_DELAY` (default 5s) +rather than issued immediately. The retry runs the same dead-letter attempt +against the same configuration, so a failure that does not clear on its own — +a `DeadLetterAddress` that cannot be resolved into a stream, or one set to the +empty string — would otherwise spin the message between server and client as +fast as the client can nack it, and nothing bounds that loop (`MaxDeliver` is +deliberately left unset so a slow consumer is never silently cut off). Pacing +it keeps a transient failure recovering promptly while a permanent one costs +one redelivery per interval instead of hundreds a second. + +The DLQ copy carries a `Nats-Msg-Id` +derived from the original's stream sequence, so a retried dead-letter of the +same message deduplicates in the DLQ within its dedup window, and the +`x-retry-count` the consumer saw when it gave the message up — RabbitMQ's +broker-side move preserves the death trail in `x-death`, and the retry count +is this connector's equivalent (a plain republish would lose it). + +### Rejected alternative: advisory-driven dead-lettering + +JetStream publishes advisories when a consumer exhausts `MaxDeliver` +(`$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES..`) or when a +message is terminated (`…MSG_TERMINATED.…`), and a common NATS idiom builds a +"dead-letter queue" as an ordinary stream capturing those subjects. That +idiom was considered and rejected for this connector. + +An advisory carries metadata, not the message: its payload names the origin +`stream_seq`, so a reader has to fetch the original out of the primary stream +to see the body or headers. A dead-letter consumer would then have to speak +two protocols — advisory JSON, then a stream fetch — where its AMQP +counterpart just consumes the dead-lettered message off a queue. It also +inherits the primary stream's retention: once `MaxAge` evicts the original, +the advisory still resolves to nothing, so the dead-letter record decays into +a dangling pointer exactly when it is most likely to be read. + +Republishing the message itself keeps the DLQ a normal address that an +unmodified AMQP client can consume, preserves the body, headers and +`x-retry-count`, and gives the dead-letter copy retention independent of the +original. The cost is that the move is two proxy-side steps rather than one +broker-side one, which the ordering above is designed to make safe. + +The advisory subjects would become relevant only if this connector set +`MaxDeliver`, as the hook for routing a delivery-exhausted message into the +same republish path instead of letting the consumer silently skip it. It does +not set `MaxDeliver` — see the redelivery limitation below. + +## Retry-count header + +Clients that count retries via an `x-retry-count` header (rather than +RabbitMQ's `x-death`) work unchanged: `handleDelivery` synthesizes +`x-retry-count` from JetStream's `NumDelivered` metadata. This is the key piece +that lets an existing AMQP retry policy run against NATS with no `x-death` +equivalent. + +## Deduplication + +Publish-side dedup maps onto JetStream's `Nats-Msg-Id` plus the stream +`Duplicates` window. When a message carries a positive publish id, it must +also carry a publisher name (matching the stream-publish contract of the AMQP +connector); the connector sets `Nats-Msg-Id` from that pair so re-publishes +within the window are collapsed. A publisher name by itself does not enable +deduplication. + +Dedup belongs to `STREAM` addresses, and only to them. JetStream would happily +deduplicate on any address — dedup here is a property of the stream, not of +the address type — but RabbitMQ gets it from the Streams client, which is the +only thing the AMQP connector reaches for a `STREAM` address. So the two paths +mirror that connector rather than what JetStream can do: + +- a **unary publish** (`PublishOne`) carrying a publish id for any other + address type — `TOPIC`, `QUEUE` or `FILTER` — is refused, because the AMQP + connector refuses it. `TOPIC` is the case to watch: it is the protobuf zero + value, so an address that never sets a type is refused too; +- a **streaming publish** carrying one *ignores* it, because the AMQP + connector's streaming path hands every non-`STREAM` address to a plain + channel publish that drops the id on the floor. Refusing would be stricter + than RabbitMQ, and honouring it would hand the client a guarantee that + disappears the moment it runs there. + +## Retention: work-queue vs append-log + +This is the most important behavioral difference between the two brokers. A +RabbitMQ queue is a work queue: a message is deleted the instant a consumer +acks it. A JetStream stream is an append log: with the default `LimitsPolicy` +an acked message stays in the log and is only evicted when a configured limit +(`MaxAge`, `MaxBytes`, `MaxMsgs`) is reached. + +To stop a stream growing without bound, `ensureStream` sets: + +- `MaxAge` (default 72h, override with `NATSJS_STREAM_MAX_AGE`) — the time + guard. It is the natural map for AMQP `MessageTTL` (a message-age duration; + `Expires` is queue disuse and maps to the consumer instead — see Source + options). It is mutable, so it also reins in streams created before a limit + existed. +- `MaxBytes` (default unlimited, override with `NATSJS_STREAM_MAX_BYTES`) — an + optional hard storage cap. With `discard=old` it only evicts near the cap, so + recent backlog is preserved as long as possible. + +There is a tension to respect: `MaxAge` evicts purely by clock, even with free +disk. If it is shorter than the longest tolerable consumer-outage window it +would silently delete a down consumer's backlog and regress the durable-queue +behavior. The default therefore exceeds any realistic outage, and `MaxBytes` +is intended as the primary disk guard. + +## Design decision: storage at the address, not the source + +The connector maps an address to a stream and a source to a consumer, so +message storage sits one level higher than AMQP puts it. In AMQP an exchange +is a stateless routing function and the queue owns storage *and* per-queue +policy — `MessageTTL`, max-length, dead-lettering, delete-on-ack. In +JetStream the stream owns storage and policy, and a consumer is only a +cursor over it. Mapping address to stream therefore trades per-source policy +away, and this section records why that trade is deliberate, because the +obvious alternative is more attractive in outline than in fact. + +The consequences accepted are: per-source `MessageTTL` and max-length cannot +be honored (a shared stream cannot carry one source's duration without +flapping it for every other source on the same address); acked messages stay +in the log until a limit evicts them rather than being deleted on ack; and +publish-rate statistics are per address rather than per source. + +A fourth consequence is subtler and has no clean answer: a transient source +becomes an ephemeral consumer, and no start position for it is faithful. In +AMQP the existence of a queue defines what is retained *for that queue* — a +transient queue holds nothing from before it was declared, and is deleted +when its last consumer leaves. A stream instead retains everything the +address received, regardless of who is listening. `DeliverNew` is therefore +wrong whenever a transient source stands in for a queue that ought to have +existed already and accumulated: a dead-letter target consumed only after +the fact is the clearest case, where the messages are in the stream and the +ephemeral consumer starts past them. `DeliverAll` is wrong in the opposite +direction, replaying history a freshly declared transient queue could never +have held, bounded only by `MaxAge`. The connector picks `DeliverNew` +because replaying a busy address's whole retained log into a temporary +consumer is the more damaging error, but this is a trade rather than a +mapping. A source that must not miss earlier messages should be durable — +non-auto-delete, or carrying a consumer group — which is the AMQP-faithful +way to say the queue exists independently of its consumers. + +The alternative is a stream per source. Because two streams may not listen +on overlapping subjects, each source stream cannot simply capture its bound +subjects — sources binding `orders.*` and `orders.created` would collide. +It would instead have to *source* from a per-address origin stream with a +subject filter, the same mechanism address-to-address binding uses above. +That buys per-source `MaxAge` and max-length, delete-on-ack via +`WorkQueuePolicy`, and per-source statistics. It was rejected for five +reasons, the first of which is decisive: + +- **It does not buy the feature that motivates it.** The usual reason to + want per-source TTL is AMQP's expire-into-the-dead-letter-exchange idiom, + and JetStream cannot express it at any topology. There is no advisory for + expiry: the advisory set covers consumer max-deliveries, nak, term, + lifecycle and leader elections, but nothing fires when a limit evicts a + message. Subject delete markers are not a substitute — they are a + key-value feature, carrying no message body, emitted only for the last + message on a subject, and stamped `Nats-Rollup: sub`, which purges the + subject. Per-source streams would give TTL *deletion*, never TTL *routing*. +- **Publish confirmation would weaken.** Sourcing is asynchronous, so a + confirmed publish would mean "durable in the origin stream", not "durable + in the bound source" — a real regression against AMQP, where a publisher + confirm covers every bound queue. +- **It multiplies cost per message and per source.** Every message is stored + once more (origin plus each source), every source adds a replication group, + and on a replicated stream with synchronous flushing the sourcing hop is an + additional synchronous write per message per source, on the path that + already bounds throughput. +- **`WorkQueuePolicy` is narrower than a queue.** It rejects multiple + unfiltered consumers (err 10099), non-unique filtered consumers (10100), + and any consumer that is not deliver-all (10101), so the delete-on-ack prize + comes with constraints a queue does not have. +- **Origin retention becomes a correctness dependency.** If the origin's + `MaxAge` elapses before a lagging source stream has sourced a message, the + message is lost silently — a failure mode the current topology does not have. + +Switching the shared stream to `InterestPolicy` was also considered, as a +cheaper route to delete-on-ack that keeps one stream per address. It is +wrong here: interest is evaluated at publish time, so a message published +before any consumer exists is discarded immediately. That would break +`Offset: first` replay and any stream-typed source, which exist precisely to +read a retained log. Queue-like and log-like sources share an address +stream and want opposite retention; `LimitsPolicy` is the only policy correct +for both, and over-retention is a storage cost rather than a correctness one. + +Revisit this decision if a deployment needs per-source expiry or max-length, +or if delete-on-ack becomes a storage problem that `MaxAge` and `MaxBytes` +cannot contain — and only if the weaker publish-confirm guarantee is +acceptable there. Note that `Retention` cannot be changed into or out of +`WorkQueuePolicy` on a live stream, so any such move is a stream-recreate +migration, not a configuration change. + +## Operational resilience + +Five mechanisms harden the connector against a cold or busy broker. All are +sized for a clustered (replicated) server, where creating topology also means +forming a raft group per stream and per durable consumer: + +- **Bounded topology calls.** Stream and consumer creation carry an explicit + deadline (`NATSJS_API_TIMEOUT`, default 30s) instead of the JetStream + client's built-in 5s default. First-touch creation of a replicated stream + has to finish raft formation and storage allocation before the API call + returns, which can exceed 5s on cold or network-attached storage. +- **Collapsed concurrent creation.** `ensureStream` calls for the same stream + are collapsed provider-wide: when many clients (re)connect at once — a + proxy restart, a mass reconnect after a broker outage — exactly one + `CreateOrUpdateStream` per stream is in flight at a time, and concurrent + callers share its result. Collapsing is scoped to the broker endpoint plus + the connection's credential identity, so distinct accounts on one server + (disjoint JetStream state and permissions) never share an outcome. + Failures are not cached, and success is still memoized per connection, so + a fresh connection re-asserts its topology. +- **Consumer liveness.** Consumers run with a 5s idle heartbeat and a consume + error handler. If the server stops serving a consumer's pulls (a broker + restart, or a just-created consumer whose raft group is not yet serving), + the missed heartbeat is logged at warn level and the client re-issues its + pull request after roughly twice the heartbeat. With library defaults the + same stall would go unlogged and take ~30s per detection cycle. +- **Stale-topology recovery.** A stream can be deleted out from under a + live connection — an operator reset, a storage wipe — and a NATS client + outlives broker state changes that would sever an AMQP connection, so the + memoized "already ensured" answer can go stale. When a publish gets "no + response from stream" or a subscribe finds the stream missing, the + connection drops its memoized entry, re-asserts the stream, and retries + once (the failed publish attempt was not stored, so the retry cannot + duplicate it), instead of failing every call until the client reconnects. + A failed dead-letter publish likewise drops the entry so the retry that + follows re-creates the DLQ stream. +- **Consumer-loss recovery.** The same class of event can take a live + subscription's server-side consumer: deleted administratively, or expired + (ephemeral inactivity threshold) during an outage the client connection + survives. The client library either stops delivering silently — it treats + "consumer deleted" as terminal — or keeps pulling a consumer that no + longer answers, and RabbitMQ has no analogous state: a deleted queue + closes its consumers' channel. When the consume machinery stops on its + own, when an authoritative consumer-gone error arrives, or when a bounded + single-flight probe (triggered by consume errors such as missed + heartbeats or unanswered pulls, and trusted only for an explicit "not + found" answer) confirms the consumer is gone, `Subscribe` ends with a + non-fatal error. The client's re-subscribe then recreates the consumer — + and, after a storage wipe, the stream. A recreated durable starts from + its configured `Offset`, like a re-declared queue starts empty. + +One teardown case deliberately does NOT end `Subscribe`: a client-initiated +`Disconnect` while its consume stream is still open. Ending the subscription +would end the caller's whole consume stream, but the amqp091 connector +leaves that stream open — its subscribe loop just goes quiet once the AMQP +connection closes — so a client that disconnects and then acks straggler +in-flight messages gets each ack answered with a "could not retrieve broker +details" failure rather than end-of-stream. `Subscribe` blocks until the +stream's own context ends; delivery cannot resume either way, because the +disconnect already stopped the consume machinery and drained the connection. + +## Configuration + +| Environment variable | Default | Meaning | +| --- | --- | --- | +| `NATSJS_STREAM_REPLICAS` | `1` | Stream replication factor. Set to `3` against a clustered server for the HA equivalent of quorum queues. | +| `NATSJS_STREAM_MAX_AGE` | `72h` | Max age before messages are evicted (Go duration; `0` = keep forever). | +| `NATSJS_STREAM_MAX_BYTES` | `0` (unlimited) | Hard per-stream storage cap in bytes. | +| `NATSJS_API_TIMEOUT` | `30s` | Deadline for JetStream management API calls (stream / consumer creation). Go duration. | +| `NATSJS_ACK_WAIT` | `30s` | How long the server waits for an ack before redelivering a message. Go duration. | +| `NATSJS_SAC_PINNED_TTL` | `1m` | Single-active-consumer failover deadline: how long the pinned client may go without pulling before a standby takes over. Go duration. | +| `NATSJS_DEADLETTER_RETRY_DELAY` | `5s` | How long to wait before redelivering a message whose dead-letter attempt failed, so a permanently-failing dead-letter cannot spin. Go duration. | + +TLS and credentials come from the standard `ConnectionConfiguration` (`Tls`, +`Credentials`) and the server's `tlsSkipVerify` flag, exactly as for the AMQP +connector; the broker certificate is verified against the system trust store. + +## Feature-parity matrix + +Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; +**Drop** = not carried forward. + +| RabbitMQ / Arke feature | Disposition | Notes | +| --- | --- | --- | +| Topic routing + wildcard bindings | Native | NATS subjects + `*`/`>`; `#`->`>`, `*`->`*`. | +| Per-subscriber ephemeral queue | Native | Ephemeral consumer with inactivity threshold (per subscriber — see limitations). | +| Durable work queue (reconnect resumes backlog) | Native | Durable consumer for non-transient / consumer-group / single-active sources. | +| Publish confirms | Native | `js.PublishMsg` returns a `PubAck`. | +| Message dedup (`publish_id` + `publisher_name`) | Native | `Nats-Msg-Id` + stream `Duplicates` window. | +| Streams: offsets, start position | Native | JetStream is a log; `DeliverPolicy` maps `Offset`. | +| Single active consumer | Native | Pinned-client priority group on the durable (nats-server 2.11+); standby takes over within `NATSJS_SAC_PINNED_TTL`. | +| Prefetch / QoS | Native | `MaxAckPending`; prefetch 0 (AMQP "unlimited") maps to unlimited (-1). The Arke gRPC server raises a prefetch below 1 to 1 before any provider sees it, so the unlimited mapping applies to direct provider use. | +| HA / quorum queues | Native | JetStream R3 (Raft) via `NATSJS_STREAM_REPLICAS`. | +| Delayed retry (per-msg TTL + DLX idiom) | Proxy -> Native | Replaced with `NakWithDelay`. | +| Retry-count header (`x-retry-count`) | Proxy | Synthesized from JetStream `NumDelivered`. | +| Dead-letter (DLX) | Proxy | No native DLX; republish to DLQ subject then `Term()`. | +| Header-filter exchange (`Filter` / `Match`) | Proxy | NATS routes on subject; evaluated in `evaluateFilters` (see limitations). | +| Address-to-address binding (`ParentAddress`) | Native | The child's stream sources the bound subjects from the parent's, keeping each message's subject. | +| `MessageTTL` (per-queue message TTL) | Drop | Accepted but not applied; retention is the stream-level `MaxAge` (see limitations). | +| `Expires` (queue disuse expiry) | Native | Consumer `InactiveThreshold`; the consumer is deleted after that long without an attached client. | +| Dead-letter on message expiry | Drop | RabbitMQ expires a message off a queue into its DLX; per-source TTL is not applied here, so nothing expires per source to dead-letter (see limitations). | +| Source stats (depth / consumers) | Native | JetStream stream / consumer `Info`. | +| Max message size | Partial | NATS caps a payload at the server's `max_payload` (1MB default) where RabbitMQ's `max_message_size` default is 16MB (see limitations). | +| Publish / deliver rates in stats | Proxy | Sampled: counter deltas between `SourceStats` calls (see limitations). | +| RabbitMQ management HTTP API | Drop | Replaced by the JetStream API over NATS itself. | +| Distributed tracing (`traceparent`/`tracestate`) | Native | Delivery starts (or continues) a span and writes its W3C trace context back into the consumed message, same as amqp091's `queueSubscribe`. | +| Stream reader position header (`x-current-offset`) | Native | STREAM-source deliveries only, matching amqp091's `streamSubscribe`; the message's own stream sequence in the `Offset` vocabulary. | +| Gzip body passthrough (`Transfer-Encoding: gzip`) | Native | STREAM-source deliveries only, matching amqp091's `streamSubscribe`: a gzip-tagged body is decompressed and the header stripped. natsjs never compresses on publish — `max_payload` has no equivalent to the RabbitMQ-Streams client's ~1MiB ceiling amqp091 works around — so this only ever undoes a publisher's own compression. | + +## Source options + +Per the connector-interface contract, `SupportedSourceOptions()` advertises +the `Source.Options` keys the connector accepts — the same list as the +amqp091 connector, so existing client sources validate unchanged: + +| Key | Type | Description | +| --- | --- | --- | +| `MessageTTL` | string (ms) | Accepted for compatibility; not applied per source — retention is the stream-wide `NATSJS_STREAM_MAX_AGE` (see Known limitations). A warning is logged at subscribe time. | +| `Expires` | string (ms) | How long the source may go without an attached consumer before the broker deletes it (AMQP `x-expires`), mapped to the consumer's `InactiveThreshold`. Unset keeps the defaults: transient sources expire after 5 minutes (the same default amqp091 applies to auto-delete/exclusive queues), durable sources never expire. Deletion removes the consumer and its ack state only; the messages stay in the shared stream under its own retention. A non-integer or non-positive value fails the subscribe. | +| `DeadLetterAddress` | string | Address whose stream receives dead-lettered messages. | +| `DeadLetterSubject` | string | Routing-key override for dead-lettered messages; when unset, the copy keeps the message's original routing key (RabbitMQ's default dead-letter behavior). | +| `Offset` | string | Stream starting offset (`first`, `continue`, `last`, `next`, or a number counting from 0, as reported by `SourceStats`). | +| `ConsumerGroup` | string | Durable consumer group name (stream sources; also names the durable that single-active instances coordinate through, and is required for a single-active stream source). | + +## Known limitations + +- **Per-source TTL fidelity, and expiry-driven dead-lettering.** The + connector uses one stream per address root, so a per-source `MessageTTL` + cannot be mapped onto the shared stream without one source's value + flapping another's retention. That option is therefore accepted but not + applied, and a subscribe that sets it logs a warning naming the source — + silent divergence in data retention is the one place a client must not + have to read a design doc to notice. (`Expires` is different: it governs + the *source's* lifetime, not its messages', and consumers are per-source, + so it maps cleanly onto the consumer's `InactiveThreshold` and IS + applied — see Source options.) Retention comes from the stream-wide + `NATSJS_STREAM_MAX_AGE` / `NATSJS_STREAM_MAX_BYTES` configuration. True + per-source TTL (or switching queue sources to a delete-on-ack policy such + as `WorkQueuePolicy` / `InterestPolicy`) needs a per-source stream + topology, and `Retention` is immutable, so that is a stream-recreate + migration rather than an in-place change. + + A consequence worth stating on its own: RabbitMQ's per-queue TTL doubles as + a routing mechanism, expiring a message off its queue and *into* the + queue's dead-letter exchange, which is the classic "unprocessed after N + seconds, send it to the DLQ" idiom. Since no per-source TTL is applied + here, nothing expires per source and so nothing is dead-lettered by expiry; + a message is dead-lettered only when a client nacks it with a dead-letter + address set. Retention limits do eventually evict a message, but eviction + deletes it — there is no expiry hook to route from. Clients relying on + expiry-to-DLQ need an explicit nack, or the per-source topology above. +- **Payloads above the server's `max_payload`.** NATS enforces a maximum + message size server-side (`max_payload`, 1MB by default, 64MB hard + ceiling), where RabbitMQ's `max_message_size` defaults to 16MB. A message + in between publishes on RabbitMQ and is rejected here with `maximum payload + exceeded`. This is a server setting the connector cannot negotiate around, + so a deployment carrying large messages has to raise `max_payload` to cover + them (NATS advises staying at or below 8MB, since a large message is held + whole in memory on every hop) or move the payload out of the message. +- **Publish / deliver rates are sampled, not native.** JetStream exposes + absolute counters, not rates, so `SourceStats` differences them between + successive calls: the first call after a (re)connect returns zero, the + publish rate covers the whole address stream rather than a single binding, + and sources without a durable consumer report a zero deliver rate. The + message count for a durable source is the consumer's backlog (undelivered + plus unacked, with `current_offset` from its ack floor) rather than the + stream depth — the stream retains acked messages under its retention + limits, so its depth keeps growing after consumers catch up, which would + mislead anything using message count as queue length (e.g. consumer + autoscaling). Sources without a durable consumer fall back to the stream + view. A `STREAM` source reports no message count at all, as on amqp091: + the readers of one retained log have no common backlog, and their reading + is the offset pair instead. A stream with no offset to report answers with + RabbitMQ's own `Offset not found` error rather than a silent zero. A + durable source's consumer count is likewise per source: the number + of clients with an open pull request on its consumer — a client working + through a full pull buffer can briefly read as zero — while sources + without a durable consumer report the stream-wide consumer count, which + spans every source on the address. +- **Header filters are evaluated proxy-side.** NATS routes on subjects only, + so a source's header `Filter`s cannot narrow what the server delivers: + every message matching the source's subject filters is delivered to the + connector, which evaluates the headers and acks non-matching messages + without forwarding them. Bandwidth and CPU between broker and proxy + therefore scale with the subject-matched traffic, not the header-matched + traffic. That is fine when header filters refine an already-narrow subject, + but a high-volume address consumed almost entirely through header filters + should be remodeled onto routing keys (subjects) instead. Because this + filtering is per consumer, competing subscribers of one durable must share + the same header filter: a RabbitMQ headers binding is queue-wide, but here + each consumer applies only its own filter, so a message the server hands to + the "wrong" subscriber would be dropped rather than reaching the one that + wanted it. A second live subscriber whose header filter differs from the + durable's is therefore rejected (subject filters, which the server enforces, + still update the shared consumer in place). +- **Dead-letter is a proxy-side republish.** The DLQ is an ordinary stream + fed by the connector; there is no advisory-driven re-consumption (see + "Rejected alternative: advisory-driven dead-lettering"). A failed DLQ + publish fails the dead-letter call — the message stays in flight and is + redelivered — rather than dropping the message. +- **Redelivery is unbounded.** The connector does not set `MaxDeliver`, so a + message that is delivered and then neither acked nor nacked — a consumer + that wedges, or crashes mid-handler in a loop — is redelivered every + `NATSJS_ACK_WAIT` for as long as the subscription lives. RabbitMQ quorum + queues apply a delivery limit (20 by default on 4.x) and, on reaching it, + dead-letter the message if the queue has a dead-letter exchange or drop it + if it does not. The difference is deliberate: the two brokers count + deliveries differently, because RabbitMQ's delayed-retry idiom republishes + the message through a TTL queue and a dead-letter exchange, which resets its + delivery count, while `Retry` here is a `NakWithDelay` that increments the + same `NumDelivered` a delivery limit would cap. A `MaxDeliver` chosen to + match RabbitMQ's limit would therefore cut off *retries* that RabbitMQ + allows without bound — breaking clients whose retry policy deliberately + retries far more than 20 times — and separating the two cases would need + per-message state the broker does not keep. Retaining the message is also + the safer divergence: nothing is lost, and the stall is visible as the + consumer's `num_pending`. Deployments that want a ceiling should alert on + redelivery rate rather than ask the broker to discard. +- **Transient sources are per-subscriber.** A transient (auto-delete / + exclusive / TEMPORARY) source maps to an ephemeral consumer created per + subscription, so two clients subscribing with the same transient source + name each receive every message. Consumers of one auto-delete AMQP queue + would instead compete for its messages, and an exclusive AMQP queue would + reject the second consumer outright. In practice clients generate unique + names for transient sources (the usual exclusive-queue idiom), which is + why this has not warranted the parity fix: a shared named consumer with an + inactivity threshold standing in for the queue's expiry, plus rejecting a + second subscriber when the source is exclusive. +- **Connection authentication.** The connector supports user/password and TLS + today; NKEYs / JWT auth are a natural follow-up for production deployments. + +## Testing + +Unit tests for the pure mapping logic (subject / wildcard translation, +durable-name selection, header-filter evaluation) live in `helpers_test.go`. +Behavioral tests in `natsjs_test.go` drive the publish / subscribe / ack / +retry / dead-letter / dedup paths against an in-process JetStream-enabled +`nats-server`, so they run without an external broker. + +## Migration approach + +Run `natsjs` as a second provider alongside `amqp091` — the `provider` field +in `ConnectionConfiguration` already supports per-connection selection. Migrate +one low-risk message class, measure cost and latency, then expand. Because the +proxy absorbs the broker-specific translation, the migration is incremental and +reversible, and there is no need to remove the AMQP connector to begin. diff --git a/go.mod b/go.mod index 7d8fde7..0377a82 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,8 @@ require ( github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 github.com/hashicorp/go-metrics v0.5.4 + github.com/nats-io/nats-server/v2 v2.14.3 + github.com/nats-io/nats.go v1.52.0 github.com/prometheus/client_golang v1.23.2 github.com/rabbitmq/amqp091-go v1.11.0 github.com/rabbitmq/rabbitmq-stream-go-client v1.8.1 @@ -29,6 +31,7 @@ require ( ) require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -42,6 +45,7 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-tpm v0.9.8 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect @@ -51,9 +55,13 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/highwayhash v1.0.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nats-io/jwt/v2 v2.8.2 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect @@ -71,10 +79,11 @@ require ( go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.53.0 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/term v0.43.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect diff --git a/go.sum b/go.sum index 19d426d..72af085 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -78,6 +80,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250923004556-9e5a51aed1e8 h1:ZI8gCoCjGzPsum4L21jHdQs8shFBIQih1TM9Rd/c+EQ= github.com/google/pprof v0.0.0-20250923004556-9e5a51aed1e8/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= @@ -134,6 +138,8 @@ github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stg github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -146,6 +152,16 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc= +github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc= +github.com/nats-io/nats-server/v2 v2.14.3 h1:+xjydPt7rkit67G+04TN0mcO2n+8nveZE7tK/PPV53A= +github.com/nats-io/nats-server/v2 v2.14.3/go.mod h1:5IlCtBzfwyzQzPMjmoJ9W2/LKmnJRtNyuOs/OT+NHDY= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= @@ -254,6 +270,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= @@ -295,10 +313,11 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/internal/provider/connectors/connectors.go b/internal/provider/connectors/connectors.go index c5cebc0..0751663 100644 --- a/internal/provider/connectors/connectors.go +++ b/internal/provider/connectors/connectors.go @@ -5,4 +5,5 @@ package connectors import ( _ "github.com/sassoftware/arke/internal/provider/connectors/amqp091" // Import the AMQP091 plugin + _ "github.com/sassoftware/arke/internal/provider/connectors/natsjs" // Import the NATS JetStream plugin ) diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go new file mode 100644 index 0000000..378506d --- /dev/null +++ b/internal/provider/connectors/natsjs/helpers.go @@ -0,0 +1,858 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package natsjs + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "io" + "slices" + "strings" + + "github.com/nats-io/nats.go" + pb "github.com/sassoftware/arke/api" +) + +// Subject / topology mapping. +// +// AMQP (RabbitMQ) routes via exchange + routing-key bindings. NATS routes via +// subjects with token wildcards. We map: +// +// exchange/address name -> subject root (dots kept; it is a multi-token prefix) +// "~" -> reserved delimiter token, always inserted after the root +// routing key -> appended subject tokens +// AMQP '#' (zero+ words) -> NATS '>' (NATS only allows '>' as the final token) +// AMQP '*' (one word) -> NATS '*' +// +// A message published to address "events.orders" with routing key +// "region.us.created" travels on subject "events.orders.~.region.us.created". +// A JetStream stream captures ".~" and ".~.>" and consumers filter +// on the mapped source subjects. +// +// The "~" delimiter exists because each address gets its own stream, and +// JetStream requires every stream's subject space to be disjoint. Address +// names themselves are dotted, so two addresses can sit in a prefix +// relationship — "events.orders" and "events.orders.filtered" — and without a +// delimiter their capture wildcards ("events.orders.>" vs +// "events.orders.filtered.>") overlap: whichever stream is created first wins +// and the other fails forever with "subjects overlap with an existing stream" +// (err 10065). The delimiter makes any two distinct roots disjoint: the token +// after "events.orders" is "~" for its own traffic but "filtered" for the +// other address, and token escaping (escapeToken) guarantees no address token +// can ever equal the delimiter. It also keeps the two addresses' messages +// apart (without it, publishing "filtered.x" to "events.orders" would be +// indistinguishable from publishing "x" to "events.orders.filtered"), and +// gives the empty routing key a concrete subject (".~"). +// +// This is a faithful mapping for the dotted, topic-style keys AMQP topic +// exchanges use (e.g. orders.region.us.created.*). It does NOT reproduce AMQP +// headers-exchange routing — that is handled proxy-side in evaluateFilters +// (see Subscribe). + +// addressDelim is the reserved token inserted between the address root and the +// routing-key tokens. Token escaping guarantees no address token equals it. +const addressDelim = "~" + +// tokenEscapes encodes, inside an escaped subject token (see escapeToken), +// the characters a token cannot carry literally: the whitespace and wildcard +// characters NATS forbids in a subject token, '#' (so a literal AMQP '#' +// stays distinct from every other token on both the publish and binding +// sides), and '~', which the connector reserves as its address delimiter and +// as the escape marker itself. Every code is two characters starting with +// '~' and no literal character inside an escaped token is ever '~', so an +// escaped token decodes unambiguously left to right — which is what makes +// the encoding injective. +var tokenEscapes = map[rune]string{ + '~': "~~", + ' ': "~w", + '\t': "~t", + '\r': "~r", + '\n': "~n", + '\f': "~f", + '*': "~a", + '>': "~g", + '#': "~h", +} + +// tokenUnescapes inverts tokenEscapes (second code byte -> original byte). +var tokenUnescapes = map[byte]byte{ + '~': '~', 'w': ' ', 't': '\t', 'r': '\r', 'n': '\n', 'f': '\f', + 'a': '*', 'g': '>', 'h': '#', +} + +// tokenEscapeTriggers are the characters that force a token into the escaped +// form; a token free of them passes through literally. +const tokenEscapeTriggers = "~ \t\r\n\f*>#" //nolint:gosec // G101 false positive: NATS subject reserved characters, not a credential + +// escapeToken maps one address or routing-key token onto a NATS-legal +// subject token, injectively: distinct tokens never yield the same output. A +// lossy replacement here (the obvious "swap illegal characters for '_'") +// would merge distinct AMQP names — "a b" and "a_b", or an address token +// "~" and "_" — onto one subject, and for addresses that is the worst +// failure mode available: two addresses sharing a subject space share a +// stream, leak each other's messages, and reconfigure each other's topology. +// +// Tokens free of reserved characters pass through unchanged, so every +// conventional dotted name keeps its readable, historical form. Any other +// token takes an escaped form marked by a leading '~' — which no plain token +// can start with, since '~' itself is reserved — followed by the token with +// each reserved character replaced by its two-character code (tokenEscapes). +// The empty token (AMQP allows consecutive dots) becomes the fixed marker +// "~e", which cannot collide with an escaped non-empty token: those always +// contain a second '~', because only tokens containing a reserved character +// are escaped and every reserved character encodes to a '~' pair. +func escapeToken(t string) string { + if t == "" { + return "~e" + } + if !strings.ContainsAny(t, tokenEscapeTriggers) { + return t + } + var b strings.Builder + b.WriteByte('~') + for _, r := range t { + if esc, ok := tokenEscapes[r]; ok { + b.WriteString(esc) + } else { + b.WriteRune(r) + } + } + return b.String() +} + +// unescapeToken inverts escapeToken. Tokens the connector did not escape +// come back unchanged; an escape code the connector never produces is kept +// literally (best effort — such tokens do not occur in subjects the +// connector itself built). Escape codes are pure ASCII and '~' (0x7E) never +// occurs inside a UTF-8 multi-byte sequence, so byte-wise scanning is safe. +func unescapeToken(t string) string { + if !strings.HasPrefix(t, "~") { + return t + } + if t == "~e" { + return "" + } + body := t[1:] + var b strings.Builder + for i := 0; i < len(body); i++ { + c := body[i] + if c != '~' || i+1 == len(body) { + b.WriteByte(c) + continue + } + i++ + if orig, ok := tokenUnescapes[body[i]]; ok { + b.WriteByte(orig) + } else { + b.WriteByte('~') + b.WriteByte(body[i]) + } + } + return b.String() +} + +// addressRoot maps an address name onto a valid dotted subject prefix. +// Tokens are escaped rather than lossily replaced or dropped, so distinct +// address names always keep distinct roots — two addresses may never share a +// subject space (see escapeToken). +func addressRoot(addressName string) string { + if addressName == "" { + return "" + } + tokens := strings.Split(addressName, ".") + for i, t := range tokens { + tokens[i] = escapeToken(t) + } + return strings.Join(tokens, ".") +} + +// subjectPrefix returns the subject an empty routing key maps to, which is also +// the prefix of every subject under the address. +func subjectPrefix(addressName string) string { + if root := addressRoot(addressName); root != "" { + return root + "." + addressDelim + } + return addressDelim +} + +// nameEscapes encodes, inside the escaped name form (see streamNameFor), the +// characters JetStream rejects in stream/consumer names plus '_' itself. +// Every code is two characters starting with '_' and no literal character +// maps to '_', so an escaped name decodes unambiguously left to right — +// which is what makes the encoding injective. (This is the name-level +// counterpart of tokenEscapes: subjects reserve '~', names reserve '_'.) +var nameEscapes = map[rune]string{ + '.': "_d", + '_': "_u", + '/': "_s", + '\\': "_b", + '\r': "_r", + '\n': "_n", + '\f': "_f", +} + +// nameEscapeTriggers are the characters that force a root into the escaped +// name form: '_' because the plain form could not tell it apart from an +// encoded '.', and the rest because JetStream rejects them in names outright +// (they are legal in subjects, so sanitization keeps them in the root). +const nameEscapeTriggers = "_/\\\r\n\f" + +// streamNameFor derives a JetStream-legal name from an address (or consumer +// group / source) name; durable consumer names use it too, as JetStream +// validates both identically. Names may not contain whitespace, '.', '*', +// '>', '/' or '\'. Derived from the sanitized root so every address that +// shares a root shares a stream. +// +// The common case — roots free of '_' and of characters JetStream rejects +// in names — keeps the readable historical form, dots swapped for +// underscores ("events.orders" -> "arke_events_orders"). That replacement is +// only unambiguous while no token contains '_' of its own: "a.b" and "a_b" +// would otherwise both read "arke_a_b", and two addresses colliding onto one +// stream name silently reconfigure each other's stream (each re-ensure +// flips the stream's subjects to its own address). Such roots instead take +// an escaped form under the disjoint "arke-" prefix, where every '_' starts +// a two-character escape code (nameEscapes), so distinct roots always yield +// distinct names across both forms. Roots are already injective per address +// (escapeToken), so the composed address -> stream-name mapping is too. +func streamNameFor(addressName string) string { + root := addressRoot(addressName) + if !strings.ContainsAny(root, nameEscapeTriggers) { + return "arke_" + strings.ReplaceAll(root, ".", "_") + } + var b strings.Builder + b.WriteString("arke-") + for _, r := range root { + if esc, ok := nameEscapes[r]; ok { + b.WriteString(esc) + } else { + b.WriteRune(r) + } + } + return b.String() +} + +// publishSubjectFor maps an address + routing key onto the concrete subject a +// message is published to. Routing keys are literal on the publish side — +// AMQP only gives '*' / '#' wildcard meaning in bindings — and NATS forbids +// wildcard tokens in published subjects, so every token is escaped +// (injectively: distinct routing keys never merge onto one subject, see +// escapeToken). Empty tokens (consecutive or trailing dots) are dropped; +// binding patterns drop them identically, so both sides agree. +func publishSubjectFor(addressName, routingKey string) string { + prefix := subjectPrefix(addressName) + tokens := strings.Split(routingKey, ".") + out := make([]string, 0, len(tokens)) + for _, t := range tokens { + if t == "" { + continue + } + out = append(out, escapeToken(t)) + } + if len(out) == 0 { + return prefix + } + return prefix + "." + strings.Join(out, ".") +} + +// translateWildcards converts an AMQP topic routing key into a valid NATS subject +// fragment. NATS is stricter than AMQP about subjects, so besides translating the +// wildcards it also sanitizes each token. Mapping (per dot-delimited token): +// +// "*" -> "*" single-token wildcard (same meaning) +// "#" (last token) -> ">" tail wildcard (same meaning) +// "#" (mid token) -> "*" NATS '>' is tail-only, so a non-terminal AMQP '#' +// has no exact equivalent; '*' keeps the subject valid +// but narrows zero-or-more to exactly-one. +// "" -> dropped (collapses "a..b" and leading/trailing dots, which +// would otherwise be empty — and thus invalid — tokens; +// publishSubjectFor drops them identically) +// other -> escaped like a published token (escapeToken), so a +// literal binding token matches exactly the +// published keys it matches on RabbitMQ +// +// Runs of consecutive '#' are collapsed into one before translation — they +// are equivalent in AMQP ('#' matches zero or more words) — so "#.#" gets +// the exact tail mapping instead of a first-'#' narrowed to '*'. +// +// The result is always a valid NATS subject fragment (possibly empty if every +// token was empty; filterSubjectsFor handles that). +func translateWildcards(key string) string { + raw := strings.Split(key, ".") + // Drop empty tokens, and collapse runs of consecutive '#' into one: AMQP's + // '#' matches zero or more words, so adjacent '#'s are equivalent to a + // single one ("a.#.#" ≡ "a.#"). Collapsing has to happen before the + // position-based translation below, or a trailing run like "#.#" would put + // its first '#' in a non-terminal position and take the lossy '*' mapping — + // losing the zero-word match a lone trailing '#' keeps. + tokens := make([]string, 0, len(raw)) + for _, t := range raw { + if t == "" || (t == "#" && len(tokens) > 0 && tokens[len(tokens)-1] == "#") { + continue + } + tokens = append(tokens, t) + } + out := make([]string, 0, len(tokens)) + for i, t := range tokens { + switch t { + case "*": + out = append(out, "*") + case "#": + if i == len(tokens)-1 { + out = append(out, ">") + } else { + out = append(out, "*") + } + default: + out = append(out, escapeToken(t)) + } + } + return strings.Join(out, ".") +} + +// routingKeyFromSubject recovers the routing key a delivered message was +// published with by dropping everything up to and including the address +// delimiter and decoding each escaped token — the inverse of publishSubjectFor +// for subjects the connector itself produced (the decode is what keeps a +// dead-letter republish of the recovered key on the same tokens instead of +// escaping them a second time). A subject that is just a prefix (an empty +// routing key) yields "", as does one carrying no delimiter at all. +// +// The delimiter is located rather than a specific address's prefix stripped, +// because a message can legitimately arrive under a subject rooted at a +// DIFFERENT address than the one the source names: an address-to-address +// binding sources the parent's messages into the child's stream keeping the +// subject they were published under (see ensureStreamFor), and the child's +// consumer selects exactly those subjects (filterSubjectsFor). Stripping the +// child's prefix would fail to match and report an empty routing key for every +// message routed in from a parent — dead-lettering it under the wrong key, +// where RabbitMQ's broker-side DLX move preserves the original. +// +// Scanning for the delimiter is unambiguous: escapeToken never emits a bare +// "~" token (an escaped token is "~e", or '~' followed by one or more +// two-character codes, so at least three characters), and subjectPrefix +// inserts exactly one. So the first token equal to the delimiter is always the +// one the connector put there. +func routingKeyFromSubject(subject string) string { + tokens := strings.Split(subject, ".") + delim := slices.Index(tokens, addressDelim) + if delim < 0 || delim == len(tokens)-1 { + return "" + } + rk := tokens[delim+1:] + out := make([]string, 0, len(rk)) + for _, t := range rk { + out = append(out, unescapeToken(t)) + } + return strings.Join(out, ".") +} + +// streamSubjectsFor returns the subject set a stream captures for the given +// address: the bare prefix (empty routing key) and everything under it. The +// two are disjoint from each other ('>' needs at least one more token) and, +// thanks to the delimiter, from every other address's set. +func streamSubjectsFor(addressName string) []string { + prefix := subjectPrefix(addressName) + return []string{prefix, prefix + ".>"} +} + +// filterSubjectsFor maps a source's routing-key patterns to NATS consumer +// filter subjects. +// +// A source whose address has a parent also selects the subjects its address is +// bound to in the *parent's* space: messages routed in from the parent are +// sourced into this address's stream keeping the subject they were published +// under (see parentBindingSubjects and ensureStreamFor). +func filterSubjectsFor(source *pb.Source) []string { + own := ownFilterSubjectsFor(source) + parents := parentBindingSubjects(source.GetAddress()) + if len(parents) == 0 { + return own + } + return pruneSubsumed(append(own, parents...)) +} + +// parentBindingSubjects gives the subjects an address's bindings select in its +// parent address's subject space, or nil if it has no parent. +// +// An address-to-address binding uses the child's own subjects as the binding +// keys (amqp091's declareExchange binds each of them from the child exchange +// to the parent), and those keys are matched by the *parent* exchange's type — +// so the mapping is exactly the ordinary binding translation, run against the +// parent's name and type. With no subjects there are no bindings and so +// nothing is routed in, which is why this returns nil rather than asking +// ownFilterSubjectsFor for an unmatchable subject. +func parentBindingSubjects(addr *pb.Address) []string { + parent := addr.GetParentAddress() + if parent.GetName() == "" || len(addr.GetSubjects()) == 0 { + return nil + } + return ownFilterSubjectsFor(&pb.Source{ + Address: &pb.Address{ + Name: parent.GetName(), + Type: parent.GetType(), + Subjects: addr.GetSubjects(), + }, + }) +} + +// ownFilterSubjectsFor maps a source's routing-key patterns to the filter +// subjects of its own address. A pattern whose AMQP '#' tail became '>' also +// gets the zero-word variant: AMQP '#' matches zero or more words while NATS +// '>' matches one or more, so binding "a.#" must match routing key "a" too. +// +// A source with no patterns declares no bindings and so receives nothing — +// see boundNothingSubjects for the exceptions and for how "nothing" is +// expressed to a consumer that must carry at least one filter. +func ownFilterSubjectsFor(source *pb.Source) []string { + if len(source.GetAddress().GetSubjects()) == 0 { + return boundNothingSubjects(source) + } + // A headers address routes on headers alone: RabbitMQ's headers exchange + // never looks at the routing key, so the subjects a source declares on one + // are binding keys the broker discards (amqp091 passes them to QueueBind / + // ExchangeBind all the same, and they decide nothing). Selecting the whole + // address and letting evaluateFilters decide is the equivalent — filtering + // by subject here would drop messages RabbitMQ delivers, which is the same + // reasoning boundNothingSubjects already applies to a headers address that + // carries no subjects at all. + if source.GetAddress().GetType() == pb.Address_FILTER { + return wholeAddressSubjects(source.GetAddress().GetName()) + } + if source.GetAddress().GetType() == pb.Address_QUEUE { + return directFilterSubjectsFor(source) + } + + prefix := subjectPrefix(source.GetAddress().GetName()) + pats := source.GetAddress().GetSubjects() + out := make([]string, 0, len(pats)+1) + seen := make(map[string]bool, len(pats)+1) + add := func(s string) { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + for _, p := range pats { + pat := translateWildcards(p) + switch { + case pat == "": + // An empty binding key is a literal, not a wildcard: on a topic + // exchange it matches the empty routing key and nothing else. + add(prefix) + case strings.HasSuffix(pat, ">"): + base := strings.TrimSuffix(strings.TrimSuffix(pat, ">"), ".") + if base == "" { + add(prefix) + } else { + add(prefix + "." + base) + } + add(prefix + "." + pat) + default: + add(prefix + "." + pat) + } + } + return pruneSubsumed(out) +} + +// pruneSubsumed drops every subject another subject in the set already covers. +// An AMQP binding set may be redundant — "orders.#" alongside "orders.created" +// — which is harmless on a broker that routes a message to a queue once no +// matter how many of its bindings match. JetStream instead rejects a consumer +// whose filter subjects overlap (one being a subset of another). The surviving +// wider filter matches everything the dropped one did. Duplicates are dropped +// too, since a subject trivially covers itself. +func pruneSubsumed(subjects []string) []string { + uniq := make([]string, 0, len(subjects)) + seen := make(map[string]bool, len(subjects)) + for _, s := range subjects { + if !seen[s] { + seen[s] = true + uniq = append(uniq, s) + } + } + kept := make([]string, 0, len(uniq)) + for _, s := range uniq { + subsumed := false + for _, t := range uniq { + if t != s && subjectSubsumes(t, s) { + subsumed = true + break + } + } + if !subsumed { + kept = append(kept, s) + } + } + return kept +} + +// unmatchableToken is a subject token no published subject can contain, so a +// filter subject ending in it selects nothing while still being a legal subset +// of the address's stream — which is how a consumer, obliged to carry at least +// one filter, expresses "bound to nothing". +// +// It is unreachable by construction: escapeToken emits either a token free of +// reserved characters (never starting with '~', since '~' is itself reserved), +// or "~e" for the empty token, or '~' followed by one or more *two*-character +// escape codes — three characters at minimum. No output is ever a '~' plus a +// single character other than "~e". +const unmatchableToken = "~x" + +// wholeAddressSubjects selects everything an address's stream captures — the +// bare prefix (empty routing key) and every subject under it. It is the filter +// set for a source bound to the address as a whole rather than to particular +// routing keys. +func wholeAddressSubjects(addressName string) []string { + prefix := subjectPrefix(addressName) + return []string{prefix, prefix + ".>"} +} + +// boundNothingSubjects gives the filter subjects for a source whose address +// carries no routing-key patterns. amqp091 declares no binding at all in that +// case (declareBinding iterates an empty subject list), so the source receives +// nothing — with two exceptions that both mean "the whole address". +func boundNothingSubjects(source *pb.Source) []string { + prefix := subjectPrefix(source.GetAddress().GetName()) + whole := wholeAddressSubjects(source.GetAddress().GetName()) + // A stream source is not bound to its address at all. amqp091 reads a + // RabbitMQ stream by name — streamSubscribe goes straight to the stream + // connection and never declares an exchange or a binding — so its reader + // sees the whole log. Selecting everything the address's stream captures + // is the equivalent. This is not a corner case: a Source_STREAM on an + // Address_STREAM with no subjects is the ordinary way to read a stream, + // so reading it as "no bindings" would silently deliver nothing. + if source.GetType() == pb.Source_STREAM { + return whole + } + // A headers address with filters: amqp091 binds a single "" key purely to + // have a binding to hang the header arguments on (declareBinding fakes the + // subject in when the address carries none). A headers exchange ignores + // routing keys, so that binding matches every message and the header match + // decides. evaluateFilters is this connector's stand-in for those + // arguments, so select the whole address and let it do the deciding. + if source.GetAddress().GetType() == pb.Address_FILTER && len(source.GetFilters()) > 0 { + return whole + } + return []string{prefix + "." + unmatchableToken} +} + +func directFilterSubjectsFor(source *pb.Source) []string { + pats := source.GetAddress().GetSubjects() + out := make([]string, 0, len(pats)) + seen := make(map[string]bool, len(pats)) + for _, p := range pats { + s := publishSubjectFor(source.GetAddress().GetName(), p) + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + return out +} + +// subjectSubsumes reports whether every subject matched by narrow is also +// matched by wide (both may contain NATS wildcards): a wide '>' covers any +// one or more remaining tokens, and a wide '*' covers a literal token or +// another '*' but never '>'. This is the subset relation JetStream applies +// when it rejects a consumer's filter subjects as overlapping. +func subjectSubsumes(wide, narrow string) bool { + w := strings.Split(wide, ".") + n := strings.Split(narrow, ".") + for i, wt := range w { + if i >= len(n) { + return false + } + if wt == ">" { + return true + } + nt := n[i] + if nt == ">" || (nt == "*" && wt != "*") { + return false + } + if wt != "*" && wt != nt { + return false + } + } + return len(w) == len(n) +} + +// durableName returns the JetStream durable consumer name for a source, or "" +// if the source should use an ephemeral consumer. +// +// Durable (work-queue) consumers persist across client reconnects, so a backlog +// published while the consumer is disconnected is redelivered on reconnect — +// matching RabbitMQ durable-queue semantics. Transient sources stay ephemeral +// and auto-expire (InactiveThreshold). +// +// Clients commonly mark per-instance/transient consumers by converting an +// Exclusive queue into AutoDelete + a UUID-suffixed name, so +// auto-delete/exclusive/TEMPORARY sources are treated as the transient ones. A +// non-transient QUEUE (a durable named listener) gets a durable consumer, as does +// a STREAM source with a ConsumerGroup, or any SingleActiveConsumer. +// dlqSourceName is the conventional name of the source that reads a source's +// dead-letter queue. It has to match amqp091's byte for byte (setupDeadLetter +// there): that connector pre-declares a queue under this name whenever a source +// sets DeadLetterAddress, and clients attach to the dead letters by naming it — +// the arke integration suite's dead-letter tests do exactly that. The '.quorum' +// strip is amqp091's: a quorum queue's dead letters belong to the logical +// source, not to the queue-type suffix. +func dlqSourceName(sourceName string) string { + return strings.Replace(sourceName, ".quorum", "", 1) + dlqSourceNameSuffix +} + +// dlqSourceNameSuffix is what dlqSourceName appends. Named because the +// adoption path tests for it to skip a server lookup for every source that +// cannot possibly be a dead-letter reader (see adoptableDLQDurable). +const dlqSourceNameSuffix = ".dlq" + +func durableName(source *pb.Source) string { + if source.GetAutoDelete() || source.GetExclusive() || source.GetType() == pb.Source_TEMPORARY { + return "" + } + if source.GetSingleActiveConsumer() { + // Single-active instances coordinate by attaching to one shared + // consumer, so the durable must be named for the coordination + // identity, not the individual subscriber. That identity is the + // ConsumerGroup option when set — amqp091 uses it as the consumer + // reference for single-active stream sources for the same reason — + // which also lets sources that share one name but separate their + // instances only by group (one group per partition/tenant/shard of a + // shared stream) get one durable per group instead of collapsing + // onto a single pinned consumer that starves every other group. + if cg := source.GetOptions()["ConsumerGroup"]; cg != "" { + return streamNameFor(cg) + } + // A stream source with no group has no coordination identity — + // Subscribe rejects it, like amqp091. A queue source falls back to + // the queue's own name: all instances of a single-active queue share + // it by definition (RabbitMQ's x-single-active-consumer is a queue + // property). + if source.GetType() == pb.Source_STREAM { + return "" + } + return streamNameFor(source.GetName()) + } + switch source.GetType() { + case pb.Source_QUEUE: + return streamNameFor(source.GetName()) + case pb.Source_STREAM: + if cg := source.GetOptions()["ConsumerGroup"]; cg != "" { + return streamNameFor(cg) + } + // A group-less stream source is ordinarily an independent ephemeral + // reader, but "continue" asks to resume where this source left off, + // which only a position the broker kept between subscriptions can + // answer. amqp091 reads one from RabbitMQ Streams' server-side offset + // tracking, keyed by the consumer name; JetStream keeps a position per + // durable consumer, so name a durable after the source. Every other + // offset positions the reader from the log itself and stays ephemeral, + // so same-named readers still read independently. + if wantsStoredPosition(source) { + return streamNameFor(source.GetName()) + } + case pb.Source_TEMPORARY: + // Transient; already returned above via the TEMPORARY guard. + } + return "" +} + +// wantsStoredPosition reports whether a source's Offset asks to resume from a +// position the broker remembers, rather than one derived from the log itself. +// Only "continue" does. +func wantsStoredPosition(source *pb.Source) bool { + return strings.EqualFold(strings.TrimSpace(source.GetOptions()["Offset"]), "continue") +} + +// Header mapping. +// +// AMQP field-table keys are length-prefixed binary: any bytes are legal, and +// RabbitMQ round-trips a header name containing spaces, colons or non-ASCII +// exactly. NATS headers are HTTP/MIME text lines, and nats.go serializes them +// with net/http's Header.Write (nats.go Msg.headerBytes -> http.Header.Write). +// That writer is lossy in two ways, both SILENT: +// +// name fails httpguts.ValidHeaderFieldName -> the entry is dropped outright +// ("we have no good way to provide the error back ... so just drop +// invalid headers instead", net/http/header.go) +// value -> CR/LF replaced with spaces, then leading/trailing space and tab +// trimmed (textproto.TrimString) +// +// So a straight pass-through loses headers RabbitMQ delivers, with no error to +// the publisher. Measured against a live broker of each kind: of 38 header +// names built from punctuation and non-ASCII, amqp091 delivered all 38 and an +// unescaped natsjs delivered 15. +// +// We therefore escape any entry NATS cannot carry verbatim, exactly as +// escapeToken does for subjects: the mapping is injective and conventional +// names pass through untouched, so ordinary traffic stays natively readable by +// non-Arke NATS consumers. An escaped entry becomes +// +// "": "" +// +// Encoding both halves together keeps one rule and one decode path rather than +// a separate marker for value-only damage. base64url without padding is used +// because its alphabet (A-Za-z0-9, '-', '_') is entirely valid header-name +// characters; standard base64's '+', '/' and '=' are not. +// +// A name that already carries the prefix is escaped too, even when it would +// survive as-is. That is what makes the mapping injective: an unescaped name on +// the wire provably never starts with the prefix, so decoding only ever fires +// on entries the connector itself encoded, and a client may legitimately send a +// header named like an encoded one without it being decoded into something else. +const headerEscapePrefix = "X-Arke-Esc-" + +// tchar reports whether c is valid in an HTTP header field name (RFC 7230 +// token). Checked inline rather than via httpguts to keep the rule explicit +// next to the escaping it drives, and to avoid a dependency for six lines. +func tchar(c byte) bool { + switch { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9': + return true + } + return strings.IndexByte("!#$%&'*+-.^_`|~", c) >= 0 +} + +// headerNameSurvives reports whether a name reaches the broker unchanged. +func headerNameSurvives(name string) bool { + if name == "" || strings.HasPrefix(name, headerEscapePrefix) { + return false + } + for i := 0; i < len(name); i++ { + if !tchar(name[i]) { + return false + } + } + return true +} + +// headerValueSurvives reports whether a value reaches the broker unchanged, +// mirroring what http.Header.Write does to it: CR/LF become spaces, then +// leading and trailing space/tab are trimmed. +func headerValueSurvives(value string) bool { + return !strings.ContainsAny(value, "\r\n") && value == strings.Trim(value, " \t") +} + +// pbToNatsHeader converts Arke's flat string headers to a NATS header, +// escaping any entry the NATS wire format would drop or rewrite. +func pbToNatsHeader(in map[string]string) nats.Header { + if len(in) == 0 { + return nil + } + h := nats.Header{} + for k, v := range in { + if headerNameSurvives(k) && headerValueSurvives(v) { + h.Set(k, v) + continue + } + h.Set(headerEscapePrefix+b64.EncodeToString([]byte(k)), b64.EncodeToString([]byte(v))) + } + return h +} + +// b64 is unpadded base64url: its alphabet is entirely valid header-name +// characters, so an encoded name needs no further escaping. +var b64 = base64.RawURLEncoding + +// copyHeader shallow-copies a NATS header so options applied to the copy at +// publish time (e.g. a message id) do not mutate the source message's header +// map. +func copyHeader(in nats.Header) nats.Header { + if len(in) == 0 { + return nil + } + out := make(nats.Header, len(in)) + for k, v := range in { + out[k] = v + } + return out +} + +// natsToPbHeader converts a NATS header to Arke's flat string headers, keeping +// the first value of each key and restoring any entry pbToNatsHeader escaped. +// +// An entry carrying the escape prefix whose name or value does not decode is +// passed through verbatim rather than dropped: the connector never produces +// such an entry, so it can only come from a foreign publisher, and surfacing it +// unchanged loses nothing. +func natsToPbHeader(in nats.Header) map[string]string { + if len(in) == 0 { + return map[string]string{} + } + out := make(map[string]string, len(in)) + for k, vals := range in { + if len(vals) == 0 { + continue + } + name, value := k, vals[0] + if encoded, ok := strings.CutPrefix(k, headerEscapePrefix); ok { + dn, nerr := b64.DecodeString(encoded) + dv, verr := b64.DecodeString(value) + if nerr == nil && verr == nil { + name, value = string(dn), string(dv) + } + } + out[name] = value + } + return out +} + +// evaluateFilters reproduces RabbitMQ headers-exchange matching proxy-side. +// +// Multiple Filters are OR'd together (each corresponds to a separate AMQP +// binding). Within a single Filter, matches are combined per Filter.Type +// (ALL = and, ANY = or). No filters means the message always passes. +func evaluateFilters(filters []*pb.Filter, headers map[string]string) bool { + if len(filters) == 0 { + return true + } + for _, f := range filters { + if filterMatches(f, headers) { + return true + } + } + return false +} + +func filterMatches(f *pb.Filter, headers map[string]string) bool { + matches := f.GetMatches() + if len(matches) == 0 { + return true + } + any := f.GetType() == pb.Filter_ANY + for _, m := range matches { + got, ok := headers[m.GetName()] + hit := ok && got == m.GetValue() + if any && hit { + return true + } + if !any && !hit { + return false + } + } + // ALL: every match passed; ANY: none passed. + return !any +} + +// decompressBody undoes gzip compression on a STREAM-source message body, +// mirroring amqp091's streamSubscribe (which undoes its own STREAM-publish +// workaround for the RabbitMQ-Streams client's ~1MiB framing limit, but +// applies to any body tagged Transfer-Encoding: gzip regardless of who +// compressed it). natsjs never compresses on publish — JetStream's +// max_payload has no equivalent 1MiB library ceiling — so this only ever +// fires for a publisher's own pre-compressed body. +func decompressBody(b []byte) ([]byte, error) { + r, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + out, err := io.ReadAll(r) + if cerr := r.Close(); cerr != nil && err == nil { + err = cerr + } + if err != nil { + return nil, err + } + return out, nil +} diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go new file mode 100644 index 0000000..878beb0 --- /dev/null +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -0,0 +1,718 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package natsjs + +import ( + "bytes" + "compress/gzip" + "net/http" + "strings" + "testing" + + "github.com/nats-io/nats.go" + pb "github.com/sassoftware/arke/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStreamNameFor(t *testing.T) { + assert.Equal(t, "arke_events_orders", streamNameFor("events.orders")) + assert.Equal(t, "arke_events_audit", streamNameFor("events.audit")) + assert.Equal(t, "arke_", streamNameFor("")) + + // Names whose tokens contain '_' take the escaped "arke-" form so they + // cannot collide with a dotted sibling: under the plain + // dots-to-underscores replacement "a.b" and "a_b" would both read + // "arke_a_b" and clobber each other's stream. + assert.Equal(t, "arke_a_b", streamNameFor("a.b")) + assert.Equal(t, "arke-a_ub", streamNameFor("a_b")) + assert.Equal(t, "arke-a_ub_dc", streamNameFor("a_b.c")) + // subject-escaped chars ('*' -> "~a") stay in the plain name form — the + // escaped root contains '~', not '_', and '~' is JetStream-name-legal + assert.Equal(t, "arke_a_~b~ac", streamNameFor("a.b*c")) + // characters JetStream rejects in names but allows in subjects are escaped + assert.Equal(t, "arke-a_sb", streamNameFor("a/b")) + assert.Equal(t, "arke-a_bb", streamNameFor(`a\b`)) + + // The mapping must be injective: distinct address roots may never share a + // stream name, or the two addresses silently reconfigure each other's + // stream. These are the collision pairs of the plain replacement. + collisionProne := []string{"a.b", "a_b", "a.b.c", "a_b.c", "a.b_c", "a_b_c", "a/b", `a\b`} + seen := map[string]string{} + for _, addr := range collisionProne { + name := streamNameFor(addr) + if prev, ok := seen[name]; ok { + t.Errorf("streamNameFor collision: %q and %q both map to %q", prev, addr, name) + } + seen[name] = addr + assert.NotContains(t, name, ".", "stream name must be JetStream-legal") + assert.NotContains(t, name, "/", "stream name must be JetStream-legal") + assert.NotContains(t, name, `\`, "stream name must be JetStream-legal") + } +} + +func TestAddressRoot(t *testing.T) { + assert.Equal(t, "events.orders", addressRoot("events.orders")) + assert.Equal(t, "", addressRoot("")) + // empty tokens are kept as escaped placeholders, not dropped, so "a..b" + // and "a.b" stay distinct roots + assert.Equal(t, "a.~e.b", addressRoot("a..b")) + // the reserved delimiter token is escaped, never left bare + assert.Equal(t, "a.~~~.b", addressRoot("a.~.b")) + // illegal subject chars inside tokens are escaped + assert.Equal(t, "a.~b~ac", addressRoot("a.b*c")) + // tokens free of reserved characters — every conventional name — pass + // through unchanged, '_' included + assert.Equal(t, "a._.b-c", addressRoot("a._.b-c")) +} + +// TestAddressRootInjective is the regression test for the root collapse: a +// lossy replacement mapped "a.~.b", "a..b", "a. .b" and "a.*.b" all onto +// "a._.b", so those distinct addresses shared one subject space and one +// stream — cross-address message leakage plus mutual stream reconfiguration. +func TestAddressRootInjective(t *testing.T) { + collisionProne := []string{ + "a._.b", "a.~.b", "a..b", "a. .b", "a.\t.b", "a.*.b", "a.>.b", + "a.#.b", "a.~~.b", "a.~e.b", "a b", "a_b", "a*b", "a\rb", + } + seen := map[string]string{} + for _, addr := range collisionProne { + root := addressRoot(addr) + if prev, ok := seen[root]; ok { + t.Errorf("addressRoot collision: %q and %q both map to %q", prev, addr, root) + } + seen[root] = addr + for _, tok := range strings.Split(root, ".") { + assert.NotEqual(t, addressDelim, tok, "no root token may equal the delimiter (%q)", addr) + assert.NotContainsf(t, tok, " ", "root token must be subject-legal (%q)", addr) + assert.False(t, tok == "" || tok == "*" || tok == ">", + "root token must be a concrete subject token (%q -> %q)", addr, root) + } + } + // pairwise: distinct addresses -> disjoint stream subject spaces and + // distinct stream names (the composed guarantee the delimiter and the + // two escape layers exist to provide) + addrs := collisionProne + for i := range addrs { + for j := i + 1; j < len(addrs); j++ { + for _, p := range streamSubjectsFor(addrs[i]) { + for _, c := range streamSubjectsFor(addrs[j]) { + assert.False(t, subjectsOverlap(p, c), + "addresses %q and %q share subject space (%q vs %q)", addrs[i], addrs[j], p, c) + } + } + assert.NotEqual(t, streamNameFor(addrs[i]), streamNameFor(addrs[j]), + "addresses %q and %q share a stream name", addrs[i], addrs[j]) + } + } +} + +func TestEscapeTokenRoundTrip(t *testing.T) { + for _, tok := range []string{ + "", "plain", "with_underscore", "~", "~~", "~e", "a b", "a\tb", + "a*b", "a>b", "a#b", "a~b", "#", "*", ">", " ", "mixed ~*># end", + "uni-cödé", "uni cödé", + } { + esc := escapeToken(tok) + assert.Equal(t, tok, unescapeToken(esc), "round trip of %q via %q", tok, esc) + assert.NotEqual(t, addressDelim, esc, "escaped token may not equal the delimiter") + assert.NotContainsf(t, esc, " ", "escaped token must be subject-legal (%q)", tok) + } + // tokens the connector never produced pass through the decoder unchanged + // (unknown escape codes are kept literally, best effort) + assert.Equal(t, "plain", unescapeToken("plain")) + assert.Equal(t, "x~q", unescapeToken("~x~q")) +} + +func TestPublishSubjectFor(t *testing.T) { + // dotted topic key under an exchange root, behind the delimiter + assert.Equal(t, + "events.orders.~.region.us.order.created.success", + publishSubjectFor("events.orders", "region.us.order.created.success")) + // empty routing key -> the bare prefix (a concrete, publishable subject) + assert.Equal(t, "events.orders.~", publishSubjectFor("events.orders", "")) + // empty address -> subjects live directly under the delimiter + assert.Equal(t, "~.a.b", publishSubjectFor("", "a.b")) + assert.Equal(t, "~", publishSubjectFor("", "")) + // wildcard chars are literal in a published routing key and NATS forbids + // them in publish subjects -> escaped (injectively: "a.#.b" and "a._.b" + // must not merge onto one subject) + assert.Equal(t, "events.orders.~.a.~~h.b", publishSubjectFor("events.orders", "a.#.b")) + assert.Equal(t, "events.orders.~.a.~~a", publishSubjectFor("events.orders", "a.*")) + // empty tokens (double dot / trailing dot) are collapsed, not left invalid + assert.Equal(t, "events.orders.~.a.b", publishSubjectFor("events.orders", "a..b")) + assert.Equal(t, "events.orders.~.a.b", publishSubjectFor("events.orders", "a.b.")) + assert.Equal(t, "events.orders.~", publishSubjectFor("events.orders", ".")) + // illegal chars embedded in a literal token are escaped in place + assert.Equal(t, "events.orders.~.~a~ab.~c~gd", publishSubjectFor("events.orders", "a*b.c>d")) + assert.Equal(t, "events.orders.~.~a~wb", publishSubjectFor("events.orders", "a b")) + // distinct routing keys never merge: "a b" vs "a_b", "#" vs "_" vs "*" + distinct := []string{"a b", "a_b", "#", "_", "*", "~", "~e"} + seen := map[string]string{} + for _, rk := range distinct { + s := publishSubjectFor("events.orders", rk) + if prev, ok := seen[s]; ok { + t.Errorf("publish subject collision: %q and %q both map to %q", prev, rk, s) + } + seen[s] = rk + } +} + +func TestRoutingKeyFromSubject(t *testing.T) { + // inverse of publishSubjectFor + assert.Equal(t, "region.us.created", + routingKeyFromSubject("events.orders.~.region.us.created")) + // the bare prefix is the empty routing key + assert.Equal(t, "", routingKeyFromSubject("events.orders.~")) + // a subject with no delimiter at all is not one the connector produced + assert.Equal(t, "", routingKeyFromSubject("events.orders")) + // empty address: subjects live directly under the delimiter + assert.Equal(t, "a.b", routingKeyFromSubject("~.a.b")) + // A message routed in from a parent address keeps the subject it was + // published under, so it is rooted at the PARENT while the source that + // consumes it names the child. The key still has to come back: RabbitMQ's + // DLX move preserves the original routing key, and a dead-letter here is + // the proxy-side stand-in for that move. + // + // Regression: this stripped the consuming address's own prefix, which a + // parent-rooted subject never matches, so every message routed in from a + // parent dead-lettered under an empty key. + assert.Equal(t, "region.us.created", + routingKeyFromSubject(publishSubjectFor("events.parent", "region.us.created"))) + // escaped tokens decode back to the published key, so a dead-letter + // republish of the recovered key lands on the same tokens (round trip: + // rk -> subject -> rk for keys with reserved characters). The delimiter + // scan must survive a key that itself contains a literal '~', which + // escapes to "~~" and so is never mistaken for the delimiter token. + for _, rk := range []string{"a b.c", "x.~.y", "#", "a*b", "region.us", "~"} { + subj := publishSubjectFor("events.orders", rk) + assert.Equal(t, rk, routingKeyFromSubject(subj), "via %q", subj) + } +} + +func TestStreamSubjectsFor(t *testing.T) { + assert.Equal(t, + []string{"events.orders.~", "events.orders.~.>"}, + streamSubjectsFor("events.orders")) + assert.Equal(t, []string{"~", "~.>"}, streamSubjectsFor("")) +} + +// TestStreamSubjectsDisjointForPrefixAddresses is the regression test for the +// address-prefix collision: without the delimiter, "events.orders" and +// "events.orders.filtered" produce overlapping stream subjects and the +// second stream can never be created (JetStream err 10065). +func TestStreamSubjectsDisjointForPrefixAddresses(t *testing.T) { + parent := streamSubjectsFor("events.orders") + child := streamSubjectsFor("events.orders.filtered") + for _, p := range parent { + for _, c := range child { + assert.False(t, subjectsOverlap(p, c), "%q overlaps %q", p, c) + } + } + // an address token spelled "~" cannot fake its way into the parent's space + evil := streamSubjectsFor("events.orders.~") + for _, p := range parent { + for _, c := range evil { + assert.False(t, subjectsOverlap(p, c), "%q overlaps %q", p, c) + } + } +} + +// subjectsOverlap reports whether two subject patterns can match a common +// subject (token-wise; '>' matches one or more trailing tokens, '*' exactly +// one). +func subjectsOverlap(a, b string) bool { + at, bt := strings.Split(a, "."), strings.Split(b, ".") + for i := 0; ; i++ { + switch { + case i == len(at) && i == len(bt): + return true + case i == len(at) || i == len(bt): + return false + case at[i] == ">" || bt[i] == ">": + return true + case at[i] == "*" || bt[i] == "*": + continue + case at[i] != bt[i]: + return false + } + } +} + +func TestFilterSubjectsFor(t *testing.T) { + src := func(subjects ...string) *pb.Source { + return &pb.Source{Address: &pb.Address{Name: "events.orders", Subjects: subjects}} + } + + // No patterns -> no bindings -> nothing selected, like amqp091 declaring + // no binding at all. The consumer still needs one filter subject, so it + // gets an unmatchable one. + assert.Equal(t, + []string{"events.orders.~." + unmatchableToken}, + filterSubjectsFor(src())) + // An empty pattern is a literal empty routing key on a topic address, not + // a catch-all. + assert.Equal(t, + []string{"events.orders.~"}, + filterSubjectsFor(src(""))) + // plain patterns map one to one + assert.Equal(t, + []string{"events.orders.~.created", "events.orders.~.region.*.updated"}, + filterSubjectsFor(src("created", "region.*.updated"))) + // AMQP '#' matches zero or more words -> both the '>' form and the + // zero-word base are included + assert.Equal(t, + []string{"events.orders.~.region", "events.orders.~.region.>"}, + filterSubjectsFor(src("region.#"))) + // a bare '#' pattern behaves like the empty pattern + assert.Equal(t, + []string{"events.orders.~", "events.orders.~.>"}, + filterSubjectsFor(src("#"))) + // consecutive '#'s are one '#' in AMQP (each matches zero or more words): + // "#.#" must still match the empty routing key, and "a.#.#" must not lose + // zero-or-more to the non-terminal-'#' narrowing + assert.Equal(t, + []string{"events.orders.~", "events.orders.~.>"}, + filterSubjectsFor(src("#.#"))) + assert.Equal(t, + []string{"events.orders.~.a", "events.orders.~.a.>"}, + filterSubjectsFor(src("a.#.#"))) + assert.Equal(t, + []string{"events.orders.~.*.b"}, + filterSubjectsFor(src("#.#.b"))) + // duplicates collapse + assert.Equal(t, + []string{"events.orders.~", "events.orders.~.>"}, + filterSubjectsFor(src("#", ""))) + // redundant bindings collapse to the widest filters: JetStream rejects a + // consumer whose filter subjects overlap, while AMQP happily routes a + // message once however many bindings match + assert.Equal(t, + []string{"events.orders.~", "events.orders.~.>"}, + filterSubjectsFor(src("#", "orders.created"))) + assert.Equal(t, + []string{"events.orders.~.region", "events.orders.~.region.>"}, + filterSubjectsFor(src("region.#", "region.us", "region.us.updated"))) + assert.Equal(t, + []string{"events.orders.~.*"}, + filterSubjectsFor(src("*", "created"))) + // intersecting patterns where neither contains the other both survive + // (JetStream accepts them) + assert.Equal(t, + []string{"events.orders.~.*.b", "events.orders.~.a.*"}, + filterSubjectsFor(src("*.b", "a.*"))) +} + +func TestDirectFilterSubjectsAreExact(t *testing.T) { + src := func(subjects ...string) *pb.Source { + return &pb.Source{Address: &pb.Address{Name: "events.direct", Type: pb.Address_QUEUE, Subjects: subjects}} + } + + assert.Equal(t, + []string{"events.direct.~.~~h"}, + filterSubjectsFor(src("#")), + "direct bindings treat # as a literal routing key, not a wildcard") + // ...and the literal '#' binding must not also match published "_" or + // "*" (the lossy sanitizer merged all three onto "_") + assert.NotEqual(t, filterSubjectsFor(src("#"))[0], publishSubjectFor("events.direct", "_")) + assert.NotEqual(t, filterSubjectsFor(src("#"))[0], publishSubjectFor("events.direct", "*")) + assert.Equal(t, filterSubjectsFor(src("#"))[0], publishSubjectFor("events.direct", "#")) + assert.Equal(t, + []string{"events.direct.~.created", "events.direct.~.region.us"}, + filterSubjectsFor(src("created", "region.us", "created"))) + // No subjects binds nothing, exactly as on a direct exchange. + assert.Equal(t, + []string{"events.direct.~." + unmatchableToken}, + filterSubjectsFor(src())) + // An explicit empty binding key selects the empty routing key only. + assert.Equal(t, + []string{"events.direct.~"}, + filterSubjectsFor(src(""))) +} + +func TestSubjectSubsumes(t *testing.T) { + cover := subjectSubsumes + + assert.True(t, cover("a.>", "a.b")) + assert.True(t, cover("a.>", "a.b.c")) + assert.True(t, cover("a.>", "a.b.>")) + assert.True(t, cover("a.>", "a.*")) + assert.True(t, cover("a.*", "a.b")) + assert.True(t, cover("a.*.c", "a.b.c")) + assert.True(t, cover(">", "anything.at.all")) + + // '>' needs at least one token, so it does not cover the bare base + assert.False(t, cover("a.>", "a")) + // a bounded pattern cannot cover an unbounded one + assert.False(t, cover("a.*", "a.>")) + assert.False(t, cover("a.b.>", "a.>")) + // '*' does not cover a different literal position count + assert.False(t, cover("a.*", "a.b.c")) + assert.False(t, cover("a.b", "a")) + assert.False(t, cover("a", "a.b")) + // literal mismatch + assert.False(t, cover("a.b", "a.c")) + // a literal never covers a wildcard + assert.False(t, cover("a.b", "a.*")) + // intersecting but neither is a subset + assert.False(t, cover("*.b", "a.*")) + assert.False(t, cover("a.*", "*.b")) +} + +// TestDLQSourceName: the name has to match the amqp091 connector's byte for +// byte. It is the name clients attach to when reading a source's dead letters, +// so a mismatch means a reader that works on one broker finds nothing on the +// other — silently, since an absent queue just delivers no messages. +func TestDLQSourceName(t *testing.T) { + assert.Equal(t, "events.orders.listener.dlq", + dlqSourceName("events.orders.listener")) + + // The '.quorum' suffix is stripped: dead letters belong to the logical + // source, not to the queue type the other connector encodes in the name. + assert.Equal(t, "events.orders.listener.dlq", + dlqSourceName("events.orders.listener.quorum")) + + // Only the first occurrence, matching amqp091's strings.Replace(..., 1). + assert.Equal(t, "a.quorum.b.dlq", dlqSourceName("a.quorum.quorum.b")) + + assert.Equal(t, ".dlq", dlqSourceName("")) +} + +func TestDurableName(t *testing.T) { + q := func(name string) *pb.Source { + return &pb.Source{Name: name, Address: &pb.Address{Name: "events.orders"}} + } + + // non-transient QUEUE (default type, not auto-delete) -> durable + assert.Equal(t, "arke_events_orders_shipping_listener", + durableName(q("events.orders.shipping.listener"))) + + // auto-delete (transient/exclusive conversion) -> ephemeral + ad := q("sess-abc") + ad.AutoDelete = true + assert.Equal(t, "", durableName(ad)) + + // exclusive -> ephemeral + ex := q("sess-xyz") + ex.Exclusive = true + assert.Equal(t, "", durableName(ex)) + + // TEMPORARY type -> ephemeral + tmp := q("tmp") + tmp.Type = pb.Source_TEMPORARY + assert.Equal(t, "", durableName(tmp)) + + // SingleActiveConsumer -> durable even if otherwise plain + sac := q("sac") + sac.SingleActiveConsumer = true + assert.Equal(t, "arke_sac", durableName(sac)) + + // SingleActiveConsumer with a ConsumerGroup -> the group is the + // coordination identity, so it names the durable (amqp091 uses the group + // as the single-active consumer reference the same way). Sources sharing + // a name but split into groups must NOT collapse onto one durable. + sacg := q("sac") + sacg.SingleActiveConsumer = true + sacg.Options = map[string]string{"ConsumerGroup": "grp.b"} + assert.Equal(t, "arke_grp_b", durableName(sacg)) + + // single-active STREAM source without a ConsumerGroup: no coordination + // identity exists; Subscribe rejects it (amqp091 parity), and there is no + // durable name for it + sacs := q("sacs") + sacs.SingleActiveConsumer = true + sacs.Type = pb.Source_STREAM + assert.Equal(t, "", durableName(sacs)) + + // STREAM with ConsumerGroup -> durable (named after the group) + st := q("st") + st.Type = pb.Source_STREAM + st.Options = map[string]string{"ConsumerGroup": "grp.a"} + assert.Equal(t, "arke_grp_a", durableName(st)) + + // STREAM without ConsumerGroup -> ephemeral + st2 := q("st2") + st2.Type = pb.Source_STREAM + assert.Equal(t, "", durableName(st2)) + + // durable names inherit streamNameFor's injectivity: source names "grp.a" + // and "grp_a" must not share a durable, or the two sources silently + // become competing consumers on one queue instead of independent queues + assert.NotEqual(t, durableName(q("grp.a")), durableName(q("grp_a"))) +} + +func TestEvaluateFilters(t *testing.T) { + headers := map[string]string{"event-source": "orders", "region": "us"} + + // no filters -> always pass + assert.True(t, evaluateFilters(nil, headers)) + + // ALL: every match must hit + all := &pb.Filter{ + Type: pb.Filter_ALL, + Matches: []*pb.Match{ + {Name: "event-source", Value: "orders"}, + {Name: "region", Value: "us"}, + }, + } + assert.True(t, evaluateFilters([]*pb.Filter{all}, headers)) + + allMiss := &pb.Filter{ + Type: pb.Filter_ALL, + Matches: []*pb.Match{ + {Name: "event-source", Value: "orders"}, + {Name: "region", Value: "eu"}, + }, + } + assert.False(t, evaluateFilters([]*pb.Filter{allMiss}, headers)) + + // ANY: a single hit passes + any := &pb.Filter{ + Type: pb.Filter_ANY, + Matches: []*pb.Match{ + {Name: "event-source", Value: "nope"}, + {Name: "region", Value: "us"}, + }, + } + assert.True(t, evaluateFilters([]*pb.Filter{any}, headers)) + + // ANY where NOTHING hits -> fail. A present header with the wrong value does + // not count, exactly as a RabbitMQ headers exchange with x-match=any requires. + anyMiss := &pb.Filter{ + Type: pb.Filter_ANY, + Matches: []*pb.Match{ + {Name: "event-source", Value: "nope"}, + {Name: "region", Value: "eu"}, // present but wrong value + {Name: "absent", Value: "there"}, // not present at all + }, + } + assert.False(t, evaluateFilters([]*pb.Filter{anyMiss}, headers)) + + // A filter with NO matches passes everything: amqp091 still appends its + // (empty) binding table, and a headers binding with no arguments matches + // every message. Also the only thing that exercises the len(matches)==0 + // short-circuit in filterMatches. + assert.True(t, evaluateFilters([]*pb.Filter{{Type: pb.Filter_ALL}}, headers)) + assert.True(t, evaluateFilters([]*pb.Filter{{Type: pb.Filter_ANY}}, headers)) + + // multiple filters are OR'd: second filter matches + assert.True(t, evaluateFilters([]*pb.Filter{allMiss, all}, headers)) +} + +// TestDecompressBody exercises decompressBody's three outcomes directly: a +// valid gzip body round-trips; a body that is not gzip at all fails at the +// reader header; and a body with a valid gzip header but a truncated payload +// fails during the read, not at construction. The delivery-path test +// (TestStreamGzipBodyIsDecompressed) only ever drove the success case, and the +// stream mislabel test only the bad-header case. +func TestDecompressBody(t *testing.T) { + plain := []byte("payload worth compressing, repeated payload worth compressing") + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + _, err := gw.Write(plain) + require.NoError(t, err) + require.NoError(t, gw.Close()) + gzipped := buf.Bytes() + + got, err := decompressBody(gzipped) + require.NoError(t, err) + assert.Equal(t, plain, got) + + _, err = decompressBody([]byte("not gzip at all")) + assert.Error(t, err, "a non-gzip body must error at the reader header") + + // Valid 10-byte header, payload cut short: NewReader succeeds, the inflate + // fails partway — the io.ReadAll error branch the success case never reaches. + _, err = decompressBody(gzipped[:len(gzipped)-6]) + assert.Error(t, err, "a truncated gzip stream must error during the read") +} + +// TestFilterSubjectsForHeadersAddress pins that a headers address selects its +// whole subject space regardless of the subjects it declares: RabbitMQ's +// headers exchange routes on header arguments alone and never reads the +// routing key, so binding subjects decide nothing there. evaluateFilters is +// the connector's stand-in for the header arguments. +func TestFilterSubjectsForHeadersAddress(t *testing.T) { + whole := []string{"events.hdr.~", "events.hdr.~.>"} + + withSubjects := &pb.Source{Type: pb.Source_QUEUE, Address: &pb.Address{ + Name: "events.hdr", Type: pb.Address_FILTER, Subjects: []string{"bound.one", "bound.two"}}} + assert.Equal(t, whole, filterSubjectsFor(withSubjects), + "subjects on a headers address must not narrow the filter") + + // Already the behaviour when it carries no subjects but has filters, and + // the two must agree. + noSubjects := &pb.Source{Type: pb.Source_QUEUE, + Address: &pb.Address{Name: "events.hdr", Type: pb.Address_FILTER}, + Filters: []*pb.Filter{{Type: pb.Filter_ALL, + Matches: []*pb.Match{{Name: "tenant", Value: "acme"}}}}} + assert.Equal(t, whole, filterSubjectsFor(noSubjects)) + + // A topic address with the same subjects still narrows — the widening is + // scoped to headers addresses. + topic := &pb.Source{Type: pb.Source_QUEUE, Address: &pb.Address{ + Name: "events.hdr", Type: pb.Address_TOPIC, Subjects: []string{"bound.one"}}} + assert.Equal(t, []string{"events.hdr.~.bound.one"}, filterSubjectsFor(topic)) +} + +// TestFilterSubjectsForStreamSourceNoSubjects: a STREAM source with no subjects +// reads the whole log — amqp091's streamSubscribe goes straight to the stream +// by name and declares no binding, so its reader sees everything. Selecting the +// whole address's captured subjects is the equivalent, and is the ordinary way +// to read a stream (a QUEUE/topic source with no subjects instead gets an +// unmatchable filter, per TestFilterSubjectsFor). +func TestFilterSubjectsForStreamSourceNoSubjects(t *testing.T) { + src := &pb.Source{Type: pb.Source_STREAM, Address: &pb.Address{Name: "events.orders"}} + assert.Equal(t, []string{"events.orders.~", "events.orders.~.>"}, filterSubjectsFor(src)) +} + +// TestParentBindingSubjectsHeadersParent: the binding keys of an +// address-to-address binding are matched by the PARENT exchange's type, so a +// headers parent ignores them too — amqp091's ExchangeBind passes the child's +// subjects with no arguments, which a headers exchange matches unconditionally. +func TestParentBindingSubjectsHeadersParent(t *testing.T) { + addr := &pb.Address{ + Name: "events.child", + Subjects: []string{"bound.one"}, + ParentAddress: &pb.Address{Name: "events.parent", Type: pb.Address_FILTER}, + } + assert.Equal(t, []string{"events.parent.~", "events.parent.~.>"}, parentBindingSubjects(addr)) +} + +// TestHeaderRoundTripPreservesEverything: the NATS wire format silently drops +// header names that are not HTTP tokens and trims whitespace off values (see +// the header-mapping comment in helpers.go). Every one of these round-trips +// through a real RabbitMQ unchanged, so the escaping has to make them survive +// here too. The punctuation set is the one measured against a live broker of +// each kind, where an unescaped connector delivered 15 of the 38 names. +func TestHeaderRoundTripPreservesEverything(t *testing.T) { + in := map[string]string{ + // conventional headers — must pass through unescaped + "Content-Type": "application/vnd.example.event+json", + "__namespace": "tenant-a", + "x-event-address": "events.orders", + "traceparent": "00-abc-def-01", + "X-B3-TraceId": "abc", + "x-retry-count": "3", + // values the writer would rewrite + "empty-value": "", + "leading-space": " v", + "trailing-space": "v ", + "surrounding-tab": "\tv\t", + "embedded-newline": "a\nb", + "embedded-crlf": "a\r\nb", + "only-whitespace": " ", + "internal-space-ok": "a b c", + "unicode-value": "héllo wörld", + "looks-like-escaped": headerEscapePrefix + "Zm9v", + } + // names built from every character the live probe exercised + for _, c := range "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ \tàé€中" { + in[string([]rune{'h', 'd', 'r', c, 'k', 'e', 'y'})] = "v" + } + + got := natsToPbHeader(pbToNatsHeader(in)) + assert.Equal(t, in, got, "every header must survive the round trip byte-for-byte") +} + +// TestNatsToPbHeaderSkipsEmptyValueSlice: a raw NATS header can carry a key +// with no values (nats.go's Header is map[string][]string, and a foreign +// publisher or library may leave a key present with an empty slice). +// natsToPbHeader must skip it rather than index vals[0] out of range, while +// keeping the well-formed entries alongside it. pbToNatsHeader never produces +// this shape (it always Sets a single value), so the round-trip tests never +// reach the guard. +func TestNatsToPbHeaderSkipsEmptyValueSlice(t *testing.T) { + in := nats.Header{ + "Present": []string{"here"}, + "Empty": []string{}, + } + assert.Equal(t, map[string]string{"Present": "here"}, natsToPbHeader(in)) +} + +// TestHeaderEscapingOnlyWhereNeeded: conventional names and values reach the +// broker verbatim, so ordinary traffic stays readable to a native NATS +// consumer and only the entries NATS cannot carry are encoded. +func TestHeaderEscapingOnlyWhereNeeded(t *testing.T) { + h := pbToNatsHeader(map[string]string{ + "Content-Type": "application/json", + "x-event-src": "inventory", + "has space": "v", + "trailing": "v ", + }) + + assert.Equal(t, "application/json", h.Get("Content-Type")) + assert.Equal(t, "inventory", h.Get("x-event-src")) + // the two damaged entries are carried under the escape prefix instead + assert.Empty(t, h.Get("has space")) + assert.Empty(t, h.Get("trailing")) + escaped := 0 + for k := range h { + if strings.HasPrefix(k, headerEscapePrefix) { + escaped++ + } + } + assert.Equal(t, 2, escaped, "exactly the unrepresentable entries are escaped") +} + +// TestHeaderEscapingSurvivesTheRealWriter drives the escaped form through the +// actual lossy component — net/http's Header.Write, which is what nats.go uses +// in Msg.headerBytes — and reads it back the way nats.go parses it. Asserting +// against the real writer rather than against tchar (our restatement of its +// rule) is the point: if the stdlib rule and ours ever diverge, this fails +// while a test written against tchar alone would keep passing. +func TestHeaderEscapingSurvivesTheRealWriter(t *testing.T) { + in := map[string]string{ + "hdr key\twith\r\neverything: €": " spaced\r\nvalue ", + "Content-Type": "application/json", + "plain": "v", + "trailing": "v ", + } + + var buf bytes.Buffer + require.NoError(t, http.Header(pbToNatsHeader(in)).Write(&buf)) + buf.WriteString("\r\n") + + // Parse back preserving original case, as nats.go's readMIMEHeader does. + onWire := nats.Header{} + for _, line := range strings.Split(buf.String(), "\r\n") { + name, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + onWire[name] = append(onWire[name], strings.TrimLeft(value, " \t")) + } + + assert.Equal(t, in, natsToPbHeader(onWire), + "headers must survive the real http.Header.Write round trip") +} + +// TestHeaderDecodeLeavesForeignEntriesAlone: a publisher that is not this +// connector can put anything on a subject, including a name that merely looks +// escaped. Such an entry is passed through rather than dropped or mangled. +func TestHeaderDecodeLeavesForeignEntriesAlone(t *testing.T) { + got := natsToPbHeader(nats.Header{ + headerEscapePrefix + "not!valid!base64": []string{"dg"}, + headerEscapePrefix + "Zm9v": []string{"not!base64"}, + "ordinary": []string{"v"}, + }) + assert.Equal(t, map[string]string{ + headerEscapePrefix + "not!valid!base64": "dg", + headerEscapePrefix + "Zm9v": "not!base64", + "ordinary": "v", + }, got) +} + +// TestHeaderNameSurvives pins the RFC 7230 token rule the escaping keys on. +func TestHeaderNameSurvives(t *testing.T) { + for _, ok := range []string{"Content-Type", "__namespace", "x-retry-count", + "a.b.c", "A1", "!#$%&'*+-.^_`|~"} { + assert.True(t, headerNameSurvives(ok), "%q should pass through", ok) + } + for _, bad := range []string{"", "has space", "has:colon", "has/slash", + "has@at", "has,comma", "has=eq", "has[br]", "unicodeé", + headerEscapePrefix + "x"} { + assert.False(t, headerNameSurvives(bad), "%q must be escaped", bad) + } +} diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go new file mode 100644 index 0000000..4c96904 --- /dev/null +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -0,0 +1,2376 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package natsjs is a NATS JetStream backend for Arke. It implements the +// provider.Provider contract so it can be selected via +// ConnectionConfiguration.provider = "natsjs", side by side with amqp091. +// +// It covers the core publish / subscribe / ack / delayed-retry / dead-letter / +// dedup / header-filter paths an AMQP client exercises through Arke, mapping +// each onto a native JetStream primitive where one exists and onto a small +// amount of proxy-side translation where it does not. It is not a one-to-one +// replacement for every RabbitMQ feature — see doc/design/natsjs-connector.md +// for the per-feature parity matrix and known limitations. +package natsjs + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "os" + "slices" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + pb "github.com/sassoftware/arke/api" + "github.com/sassoftware/arke/internal/provider" + "github.com/sassoftware/arke/internal/util" + "github.com/sassoftware/arke/internal/util/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +const providerName = "natsjs" + +// retryCountHeaderName mirrors the amqp091 connector. Clients that count retries +// via this header (rather than RabbitMQ's x-death) keep working unchanged because +// the connector synthesizes it from JetStream's delivery count — JetStream has no +// x-death equivalent. +const retryCountHeaderName = "x-retry-count" + +// timestampHeaderName mirrors the amqp091 connector, where the header is not +// arke's work at all: RabbitMQ's own ingress interceptor +// (message_interceptors.incoming.set_header_timestamp) stamps every incoming +// message with it, so every consumed message carries one whatever the source +// type. NATS has no interceptor equivalent, so the connector synthesizes it +// from the JetStream metadata timestamp — "the time the message was originally +// stored on a stream", which is the same broker-receive instant RabbitMQ +// records. The rabbit conf sets overwrite = true, so a publisher-supplied +// value is replaced rather than kept. +const timestampHeaderName = "timestamp_in_ms" + +// currentOffsetHeaderName mirrors the amqp091 connector's streamSubscribe, +// which stamps it on every STREAM-source delivery from the RabbitMQ Streams +// consumer's own offset (amqp091.go:1279). The JetStream analogue of that +// per-reader position is the message's own stream sequence, converted to the +// Offset vocabulary SourceStats already uses (see offsetOf) so a value read +// here and handed back as a source's Offset option resumes at the same +// message. Queue/topic sources never carry this header on either connector — +// amqp091 only sets it on the RabbitMQ-Streams delivery path. +const currentOffsetHeaderName = "x-current-offset" + +// transferEncodingHeaderName mirrors the amqp091 connector's streamSubscribe. +// amqp091's own STREAM publish path transparently gzips a body over the +// RabbitMQ-Streams client library's ~1MiB framing limit (compression.go) and +// tags it with this header so the delivery path can undo it; JetStream has no +// equivalent library ceiling (max_payload is deploy-configured well above +// 1MiB), so natsjs never compresses on publish. But the header is a plain +// value match, not tied to arke's own compression: a publisher that gzips its +// own body and sets this header itself gets decompressed on amqp091 +// regardless of who set it, so natsjs must honor it too rather than handing a +// STREAM consumer raw gzip bytes it never asked to receive compressed. +const transferEncodingHeaderName = "Transfer-Encoding" + +// Error strings a client may match on, so they have to read identically +// whichever connector answers. Both take amqp091's wording: noMessageError is +// what it returns for an ack, nack, or dead-letter naming a message it does +// not hold, and offsetNotFoundError is the RabbitMQ stream client's answer +// when a stream has no offset to report, which its SourceStats passes through. +const ( + noMessageError = "No message with uuid %s" + offsetNotFoundError = "Offset not found" + unsupportedConfirmError = "Unsupported: Publish does not support publish confirmation" + queueDedupError = "Message deduplication is not supported by Queue types" +) + +// streamMissingError answers a unary publish to a STREAM address whose stream +// was never declared. amqp091's wording there is stream-client plumbing +// ("failed to create a stream publisher"), so instead of copying it this takes +// the RabbitMQ stream client's canonical phrase; the contract a client relies +// on is that the publish errors rather than creating the stream. +const streamMissingError = "publish: stream %q does not exist" + +const ( + // defaultAckWait is how long the server waits for an ack before + // redelivering a message. See ackWait for the trade-off it sets; + // override via NATSJS_ACK_WAIT. + defaultAckWait = 30 * time.Second + defaultInactiveThreshold = 5 * time.Minute + defaultDedupWindow = 2 * time.Minute + connectPollInterval = 50 * time.Millisecond + // defaultJSAPITimeout bounds JetStream management API calls (stream and + // consumer creation). Without an explicit deadline the JetStream client + // applies a 5s default, which can expire while a replicated stream is + // still forming its raft group on cold or network-attached storage. + // Override via NATSJS_API_TIMEOUT. + defaultJSAPITimeout = 30 * time.Second + // defaultConsumeHeartbeat is the pull-consumer idle heartbeat. If the + // delivery path stalls, the missed heartbeat is surfaced through the + // consume error handler and the client re-issues its pull request after + // roughly twice this interval, instead of ~30s with the library defaults. + defaultConsumeHeartbeat = 5 * time.Second + // ephemeralDeleteTimeout bounds the best-effort DeleteConsumer issued when + // an ephemeral subscription ends; teardown must not hang on a broker that + // is already gone. + ephemeralDeleteTimeout = 5 * time.Second + // defaultStreamMaxAge bounds how long a stream retains messages so the + // JetStream log does not grow without bound (RabbitMQ deletes on ack; + // JetStream LimitsPolicy does not). 72h is generous enough not to truncate a + // realistic consumer-outage backlog, short enough to stop indefinite + // accumulation. Override via NATSJS_STREAM_MAX_AGE. + defaultStreamMaxAge = 72 * time.Hour + // sacPriorityGroup is the priority group name used for single-active- + // consumer sources. One group is enough: RabbitMQ's SAC has no notion of + // multiple groups, and every subscriber of the durable joins this one. + sacPriorityGroup = "arke" + // defaultSACPinnedTTL is how long the server waits for a new pull request + // from the pinned (active) client before unpinning and failing over to a + // standby. See sacPinnedTTL; override via NATSJS_SAC_PINNED_TTL. + defaultSACPinnedTTL = time.Minute + // defaultDeadLetterRetryDelay paces the redelivery that retries a failed + // dead-letter. See deadLetterRetryDelay; override via + // NATSJS_DEADLETTER_RETRY_DELAY. + defaultDeadLetterRetryDelay = 5 * time.Second +) + +// dlqDurableMetadataKey marks a durable consumer as one setupDeadLetter +// pre-declared, and records the source whose dead letters it holds. Adoption +// (see adoptableDLQDurable) is gated on this marker rather than on the name +// alone: a transient source must never silently attach to some unrelated +// durable that merely happens to share its name, which would hand it another +// subscriber's pending messages and its stored ack position. +const dlqDurableMetadataKey = "arke_dead_letters_for" + +// supportedSourceOptionsList intentionally matches the amqp091 connector so that +// existing client Sources validate against the server unchanged. +var supportedSourceOptionsList = []string{ + "MessageTTL", "DeadLetterAddress", "DeadLetterSubject", "Expires", "Offset", "ConsumerGroup", +} + +var supportedSourceOptions map[string]bool + +// GetClientIdentifier is a var so tests can override it (matches amqp091). +var GetClientIdentifier = util.GetClientIdentifier + +type natsBrokerDetails struct { + sync.Mutex + nc *nats.Conn + js jetstream.JetStream + clientIdentifier string + connectionConfig *pb.ConnectionConfiguration + + // activeMessages maps an Arke message UUID -> an inflightMsg (the in-flight + // jetstream.Msg plus the subscription that delivered it) so a later + // Ack/Nack/Retry/DeadLetter RPC can resolve it and teardown can release + // only the closing subscription's own deliveries. + activeMessages *util.ConcurrentMap + // activeMu serializes the read-modify-write pairs over activeMessages — + // the take-then-delete that resolves a message, and the read-then-update + // that marks one for redelivery. The map is individually concurrency-safe, + // but without this a mark landing between a concurrent take's read and its + // delete would reinsert an entry nothing can ever resolve, leaking it for + // the life of the connection and inflating the active-message stat. + activeMu sync.Mutex + // knownStreams memoizes streams we have already ensured. + knownStreams *util.ConcurrentMap + // consumeContexts holds each live subscription's jetstream.ConsumeContext + // so Disconnect can stop them and Stats can count them. Keyed per + // subscription (source name plus a serial from subscriptionSeq), NOT per + // source name: one connection may legitimately subscribe the same source + // name more than once — single-active groups share a name and differ only + // by ConsumerGroup — and a name key would make those overwrite each other, + // undercounting Stats and letting one teardown drop the other's entry. + consumeContexts *util.ConcurrentMap + // subscriptionSeq disambiguates consumeContexts keys. + subscriptionSeq atomic.Uint64 + // rates derives SourceStats publish/deliver rates from JetStream counters. + rates *rateTracker + + // durableFilters guards competing consumers on one durable against applying + // different proxy-side header filters. Each consumer evaluates only its own + // Subscribe's filters (evaluateFilters in handleDelivery), so if two + // subscribers of one durable disagree, a message JetStream hands to the + // "wrong" one is filtered out and lost to the subscriber that wanted it. + // Keyed by durable name; the entry records the header-filter fingerprint the + // live subscribers share and how many hold it, so a conflicting declaration + // is rejected instead of silently dropping messages. Subject filters are + // excluded — they are the server's and update in place. + // + // Scope: this map is per connection, so the check catches conflicting + // subscribers that share one, not two clients on separate connections + // attaching to the same durable — the server-side consumer is shared but + // nothing here is. Enforcing it across connections would mean putting the + // fingerprint somewhere both can read (consumer metadata), which buys + // little: subscribers of one durable are ordinarily replicas of one + // service declaring identical filters, so a disagreement is a client bug + // this catches only when it is cheap to. Treat it as a guard rail, not a + // guarantee. + durableFilters map[string]*durableFilterUse + durableFilterMu sync.Mutex + + state atomic.Uint32 + consumed int64 + produced int64 + + // clientDisconnect distinguishes a client-initiated Disconnect from the + // NATS library closing the connection terminally (both read state CLOSED): + // a disconnected client's Subscribe parks until its stream ends instead of + // ending the stream. Set before the consume contexts are stopped, so any + // Subscribe woken by that teardown observes it. Mirrors amqp091's flag of + // the same name. + clientDisconnect atomic.Bool +} + +// durableFilterUse records the shared header-filter fingerprint of a durable's +// live subscribers and a reference count so the entry lives exactly as long as +// at least one subscriber holds it (see natsBrokerDetails.durableFilters). +type durableFilterUse struct { + fingerprint string + refs int +} + +// inflightMsg is a delivered-but-unresolved message together with the +// subscription (subKey) that delivered it. Recording the owner lets teardown +// release only the closing subscription's deliveries even when several +// subscriptions share one server-side consumer (competing consumers on a +// durable, single-active standbys) — otherwise closing one subscription would +// nak and drop a sibling's in-flight messages, failing the sibling's later +// ack and duplicating the message. +type inflightMsg struct { + msg jetstream.Msg + subKey string + // redeliverOnNack inverts Nack's terminate-by-default behaviour for this + // message. It is set only by a failed DeadLetter, whose caller answers the + // error with a nack that has to put the message back for another + // dead-letter attempt rather than destroy it. See Nack. + redeliverOnNack bool +} + +type natsjsProvider struct { + connections *util.ConcurrentMap + // streams collapses concurrent stream-creation calls across connections. + streams *streamRegistry + // connectMu serializes the check-then-insert in Connect. The gRPC server's + // own "already connected?" guard (brokerConnect) is not atomic, so two + // Connect calls for one client identifier can both reach the provider; + // without this, the second would overwrite the first's entry and orphan its + // nats.Conn, which reconnects forever (MaxReconnects(-1)). + connectMu sync.Mutex + // bindMu serializes the read-modify-write of an address's + // address-to-address binding set. See ensureStreamFor. + bindMu sync.Mutex +} + +// streamRegistry collapses concurrent ensureStream calls for the same stream +// into a single JetStream API call. When many clients (re)connect at once — +// e.g. after a broker or proxy restart — every connection would otherwise +// issue its own CreateOrUpdateStream for the same shared streams, piling +// redundant load on the JetStream metadata leader exactly when it is busiest. +// Entries are keyed by broker endpoint + credential identity + stream name +// (streamEnsureKey) so distinct brokers — or distinct accounts on one broker +// — never share results. Only in-flight calls are tracked here; success is +// memoized per connection (knownStreams), preserving the property that a +// fresh connection re-asserts its topology. +type streamRegistry struct { + mu sync.Mutex + inflight map[string]*inflightEnsure +} + +type inflightEnsure struct { + done chan struct{} + err error +} + +func newStreamRegistry() *streamRegistry { + return &streamRegistry{inflight: make(map[string]*inflightEnsure)} +} + +// streamEnsureKey identifies one stream-creation target for the registry: +// broker endpoint, credential identity, stream name. The credential part is +// what keeps two connections that share an endpoint but authenticate as +// different users — on a multi-account server, different accounts with +// disjoint JetStream state and permissions — from coalescing onto one +// CreateOrUpdateStream, which would run under whichever account got there +// first and hand its outcome (success in the wrong account, or that +// account's permission failure) to the other. The stream name goes last and +// can never contain '/' (see streamNameFor), so an embedded '/' in a +// username cannot make two distinct targets read alike. +func streamEnsureKey(cfg *pb.ConnectionConfiguration, streamName string) string { + return fmt.Sprintf("%s:%d/%s/%s", + cfg.GetHost(), cfg.GetPort(), cfg.GetCredentials().GetUsername(), streamName) +} + +// ensure runs create once per key at a time; callers that arrive while a call +// for the same key is in flight wait for it and share its result. Failures +// are not cached — the next caller retries. +func (r *streamRegistry) ensure(ctx context.Context, key string, create func() error) error { + r.mu.Lock() + if e, ok := r.inflight[key]; ok { + r.mu.Unlock() + select { + case <-e.done: + return e.err + case <-ctx.Done(): + return ctx.Err() + } + } + e := &inflightEnsure{done: make(chan struct{})} + r.inflight[key] = e + r.mu.Unlock() + + e.err = create() + + r.mu.Lock() + delete(r.inflight, key) + r.mu.Unlock() + close(e.done) + return e.err +} + +func init() { + provider.Register(providerName, NewNATSJetStreamProvider) + supportedSourceOptions = make(map[string]bool, len(supportedSourceOptionsList)) + for _, o := range supportedSourceOptionsList { + supportedSourceOptions[o] = true + } +} + +// NewNATSJetStreamProvider returns a new natsjs provider singleton. +func NewNATSJetStreamProvider() provider.Provider { + return &natsjsProvider{ + connections: util.NewConcurrentMap(), + streams: newStreamRegistry(), + } +} + +// streamReplicas controls the JetStream replication factor (1 = single node, +// 3 = R3/Raft for HA). Configured via NATSJS_STREAM_REPLICAS. +func streamReplicas() int { + if v := os.Getenv("NATSJS_STREAM_REPLICAS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + return 1 +} + +// streamMaxAge is the primary guard against unbounded JetStream log growth. +// Configured via NATSJS_STREAM_MAX_AGE as a Go duration ("72h", "24h", "0" = +// keep forever); defaults to defaultStreamMaxAge. +// +// Note: because the connector uses one stream per address root, this is a +// stream-wide bound, NOT a faithful per-source x-message-ttl. A per-source +// MessageTTL is deliberately NOT folded in here: ensureStream is also called from +// the publish path (which has no Source), so mixing a per-source TTL with the +// global default would make MaxAge flap on the shared stream depending on whether +// a publish or a subscribe touched it last. Per-source TTL fidelity needs a +// per-source stream topology — see the "Known limitations" section of +// doc/design/natsjs-connector.md. +func streamMaxAge() time.Duration { + if v := os.Getenv("NATSJS_STREAM_MAX_AGE"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d >= 0 { + return d + } + } + return defaultStreamMaxAge +} + +// streamMaxBytes optionally caps a stream's on-disk size as a hard storage +// safety net. With the default discard=old, JetStream evicts the oldest messages +// only when near the cap, so recent backlog is preserved as long as possible. +// Configured via NATSJS_STREAM_MAX_BYTES (bytes); default 0 = unlimited (rely on +// MaxAge). Set it to bound disk per stream when volume is high. +func streamMaxBytes() int64 { + if v := os.Getenv("NATSJS_STREAM_MAX_BYTES"); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 { + return n + } + } + return 0 +} + +// jsAPITimeout bounds JetStream management API calls (CreateOrUpdateStream / +// CreateOrUpdateConsumer). First-touch creation of a replicated stream has to +// finish raft-group formation and storage allocation before the call returns, +// so it can legitimately take longer than the JetStream client's built-in 5s +// default, particularly on network-attached storage. Configured via +// NATSJS_API_TIMEOUT as a Go duration ("30s", "1m"); defaults to +// defaultJSAPITimeout. +func jsAPITimeout() time.Duration { + if v := os.Getenv("NATSJS_API_TIMEOUT"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultJSAPITimeout +} + +// ackWait is how long the server waits for a delivered message's ack before +// redelivering it. It sets one dial between two failure modes: a crashed +// client's in-flight messages are stuck for the full ack wait before another +// consumer gets them (shorter is better), while a healthy consumer that takes +// longer than the ack wait to process a message gets a duplicate redelivery +// (longer is better). The 30s default favors failover; deployments whose +// consumers legitimately hold messages longer — RabbitMQ's equivalent +// consumer timeout defaults to 30 minutes — should raise it via +// NATSJS_ACK_WAIT (Go duration). Note the pull buffer counts too: with a +// large prefetch a backlogged consumer can hold messages client-side longer +// than the ack wait just waiting their turn. +func ackWait() time.Duration { + if v := os.Getenv("NATSJS_ACK_WAIT"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultAckWait +} + +// sacPinnedTTL is the failover deadline for single-active-consumer sources: +// when the pinned (active) client sends no new pull request for this long, +// the server unpins it and the next standby's pull takes over. It must +// comfortably exceed the pull re-issue cadence (~30s with the client +// library's defaults) or the pin flaps between healthy standbys; lower it — +// together with faster pulls — only to shrink failover time. Configured via +// NATSJS_SAC_PINNED_TTL (Go duration). +func sacPinnedTTL() time.Duration { + if v := os.Getenv("NATSJS_SAC_PINNED_TTL"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultSACPinnedTTL +} + +// deadLetterRetryDelay paces the redelivery that retries a dead-letter attempt +// which failed with the message still in flight (see DeadLetter and Nack). +// That redelivery has to be delayed rather than immediate: the retry runs the +// same dead-letter attempt against the same configuration, so a failure that +// does not clear on its own — a DeadLetterAddress that cannot be resolved into +// a stream, or one set to the empty string — otherwise spins the message +// between server and client as fast as the client can nack it, and nothing +// bounds the loop (MaxDeliver is deliberately left unset so a slow consumer is +// never silently cut off). Delaying it keeps a transient failure recovering +// promptly while a permanent one costs one redelivery per interval instead of +// hundreds a second. Configured via NATSJS_DEADLETTER_RETRY_DELAY (Go +// duration). +func deadLetterRetryDelay() time.Duration { + if v := os.Getenv("NATSJS_DEADLETTER_RETRY_DELAY"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultDeadLetterRetryDelay +} + +// dlqDeclareThreshold is the InactiveThreshold given to a pre-declared +// dead-letter consumer (setupDeadLetter). +// +// It defaults to the dead-letter stream's own retention (streamMaxAge), so the +// queue never expires while there is still something in it to read. That is +// the property that matters: a dead-letter reader attaches AFTER the failures +// — finding out what broke is why it is attaching — so any window where the +// messages are still retained but their queue has already gone is a window +// where the reader silently sees nothing, which is the exact bug +// setupDeadLetter exists to fix. Tying the two together closes it by +// construction rather than by guessing a duration. A stream configured for +// unbounded retention (MaxAge 0) yields 0 here too: no expiry, consistently. +// +// amqp091's equivalent queue has no expiry at all, so this is still a +// divergence, but only past the point where the messages themselves are gone. +// Bounding it at all is what stops a leak: a source name is not always stable +// — clients commonly mark a transient consumer by turning an Exclusive source +// into AutoDelete plus a fresh UUID (see durableName) — so an unbounded +// threshold would strand one permanent durable per connection. +// +// NATSJS_DLQ_DECLARE_TTL overrides it; "0" disables expiry outright, which is +// full amqp091 parity for deployments whose source names are stable. +func dlqDeclareThreshold() time.Duration { + v := os.Getenv("NATSJS_DLQ_DECLARE_TTL") + if v == "" { + return streamMaxAge() + } + if v == "0" { + return 0 + } + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + return streamMaxAge() +} + +func (p *natsjsProvider) getBrokerDetails(ctx context.Context) (*natsBrokerDetails, error) { + clientID, err := GetClientIdentifier(ctx) + if err != nil { + return nil, err + } + if bd := p.getBrokerDetailsByIdentifier(clientID); bd != nil { + return bd, nil + } + // Error string matches the amqp091 connector: clients (and the + // integration suite) observe it through the ack path after a disconnect. + return nil, fmt.Errorf("could not retrieve broker details for this connection: %s", clientID) +} + +func (p *natsjsProvider) getBrokerDetailsByIdentifier(id string) *natsBrokerDetails { + if bd, ok := p.connections.Get(id); ok { + return bd.(*natsBrokerDetails) + } + return nil +} + +func (p *natsjsProvider) ClientExists(id string) bool { + return p.getBrokerDetailsByIdentifier(id) != nil +} + +// Connect establishes a NATS connection and JetStream context for the client. +func (p *natsjsProvider) Connect(ctx context.Context, cfg *pb.ConnectionConfiguration, tlsSkipVerify bool) *pb.Error { + clientID, err := GetClientIdentifier(ctx) + if err != nil { + return &pb.Error{Message: err.Error(), IsFatal: true} + } + + scheme := "nats://" + if cfg.GetTls() { + scheme = "tls://" + } + url := fmt.Sprintf("%s%s:%d", scheme, cfg.GetHost(), cfg.GetPort()) + + bd := &natsBrokerDetails{ + clientIdentifier: clientID, + connectionConfig: cfg, + activeMessages: util.NewConcurrentMap(), + knownStreams: util.NewConcurrentMap(), + consumeContexts: util.NewConcurrentMap(), + rates: newRateTracker(), + durableFilters: make(map[string]*durableFilterUse), + } + bd.state.Store(provider.CONNECTING) + + opts := []nats.Option{ + nats.Name(cfg.GetClientName()), + nats.MaxReconnects(-1), + nats.RetryOnFailedConnect(true), + // Keep bd.state honest so WaitForConnect reflects the real link. With + // RetryOnFailedConnect, nats.Connect returns a non-error connection even + // when the broker is unreachable; nats.go then reconnects in the + // background (MaxReconnects(-1)). These callbacks mirror the amqp091 + // connectionWatcher: WaitForConnect waits out a broker outage instead of + // falsely reporting CONNECTED. + nats.ConnectHandler(func(*nats.Conn) { bd.state.Store(provider.CONNECTED) }), + nats.ReconnectHandler(func(*nats.Conn) { bd.state.Store(provider.CONNECTED) }), + nats.DisconnectErrHandler(func(_ *nats.Conn, _ error) { bd.state.Store(provider.DISCONNECTED) }), + nats.ClosedHandler(func(*nats.Conn) { bd.state.Store(provider.CLOSED) }), + } + if creds := cfg.GetCredentials(); creds != nil && creds.GetUsername() != "" { + opts = append(opts, nats.UserInfo(creds.GetUsername(), creds.GetPassword())) + } + if cfg.GetTls() { + // The broker certificate is verified against the system trust store; + // client-provided CA certificates are not supported (matches amqp091). + tlsCfg := &tls.Config{InsecureSkipVerify: tlsSkipVerify} //nolint:gosec // operator opt-in, mirrors amqp091 + opts = append(opts, nats.Secure(tlsCfg)) + } + + nc, err := nats.Connect(url, opts...) + if err != nil { + return &pb.Error{Message: fmt.Sprintf("nats connect failed: %s", err.Error()), IsFatal: true} + } + js, err := jetstream.New(nc) + if err != nil { + nc.Close() + return &pb.Error{Message: fmt.Sprintf("jetstream init failed: %s", err.Error()), IsFatal: true} + } + + bd.nc = nc + bd.js = js + // If the broker was reachable, nats.Connect already completed the handshake + // (the ConnectHandler may not fire for a synchronous connect), so reflect the + // live status here; otherwise stay CONNECTING until a callback flips it. + if nc.IsConnected() { + bd.state.Store(provider.CONNECTED) + } + // Insert under connectMu so a racing Connect for the same client identifier + // cannot both install a connection: the loser closes its freshly-dialed + // nats.Conn instead of overwriting the winner's and leaking a background- + // reconnecting link. A client already has a working connection in that case, + // so this is a success (mirrors the server's "connected more than once"). + p.connectMu.Lock() + if _, exists := p.connections.Get(clientID); exists { + p.connectMu.Unlock() + nc.Close() + util.Logger.Debugf("natsjs: client %s already connected; closing the redundant link", clientID) + return nil + } + p.connections.Add(clientID, bd) + p.connectMu.Unlock() + util.Logger.Debugf("natsjs: client %s connecting to %s", clientID, url) + return nil +} + +func (p *natsjsProvider) WaitForConnect(ctx context.Context) bool { + clientID, err := GetClientIdentifier(ctx) + if err != nil { + return false + } + deadline := time.Now().Add(provider.CONNECTTIMEOUT * time.Second) + for time.Now().Before(deadline) { + if bd := p.getBrokerDetailsByIdentifier(clientID); bd != nil && bd.state.Load() == provider.CONNECTED { + return true + } + select { + case <-ctx.Done(): + return false + case <-time.After(connectPollInterval): + } + } + return false +} + +func (p *natsjsProvider) Disconnect(ctx context.Context) { + clientID, err := GetClientIdentifier(ctx) + if err != nil { + return + } + bd := p.getBrokerDetailsByIdentifier(clientID) + if bd == nil { + return + } + // Mark the connection closed before stopping its consume contexts: + // stopping one wakes its Subscribe (Closed() fires), which reads the + // state to tell deliberate teardown from a consumer lost on the server — + // and the flag to tell a client's own Disconnect from a terminal close. + bd.clientDisconnect.Store(true) + bd.state.Store(provider.CLOSED) + for _, name := range bd.consumeContexts.GetList() { + if cc, ok := bd.consumeContexts.Get(name); ok { + cc.(jetstream.ConsumeContext).Stop() + } + } + if bd.nc != nil { + _ = bd.nc.Drain() + } + p.connections.Delete(clientID) + util.Logger.Debugf("natsjs: client %s disconnected", clientID) +} + +// ensureStream lazily creates (or updates) a JetStream stream that captures all +// subjects under the given address root. Concurrent calls for the same stream +// — across all client connections to the same broker — are collapsed into one +// API call (see streamRegistry). That call gets an explicit deadline +// (jsAPITimeout) and is detached from the triggering caller's cancellation: +// the stream is shared topology and other callers may be waiting on the same +// result. +// ensureStream asserts the stream for a bare address name — one with no +// address-to-address binding to declare. See ensureStreamFor. +func (p *natsjsProvider) ensureStream(ctx context.Context, bd *natsBrokerDetails, addressName string) (string, error) { + return p.ensureStreamFor(ctx, bd, &pb.Address{Name: addressName}) +} + +// ensureStreamFor asserts the stream backing an address, including the +// address-to-address binding its ParentAddress declares, if any. +// +// A binding from a parent address is a routing rule on a broker that routes; +// here, where the stream *is* the storage, the equivalent is to source the +// bound subjects out of the parent's stream into this one. The sourced copies +// keep the subject they were published under (no subject transform), so a +// message routed in from the parent is indistinguishable to a consumer from +// one published to this address directly, and only the parent's stream listens +// on the parent's subjects — no two streams ever claim the same subject space. +func (p *natsjsProvider) ensureStreamFor(ctx context.Context, bd *natsBrokerDetails, addr *pb.Address) (string, error) { + streamName := streamNameFor(addr.GetName()) + bindings := parentBindingSubjects(addr) + // The per-connection memo only remembers that the stream exists, which is + // all an unbound address needs. A binding set is per-address state that + // another subscriber may have extended since, so an address with a parent + // re-asserts it every time; the assertion is idempotent, and addresses + // with parents are rare enough for the extra round-trip not to matter. + if len(bindings) == 0 { + if _, ok := bd.knownStreams.Get(streamName); ok { + return streamName, nil + } + } + // The parent's own stream is asserted before bindMu is taken below: that + // assertion runs through the same lock, so holding it across the nested + // call would deadlock. + var parentStream string + if len(bindings) > 0 { + ps, perr := p.ensureStream(ctx, bd, addr.GetParentAddress().GetName()) + if perr != nil { + return "", fmt.Errorf("ensure parent address stream: %w", perr) + } + parentStream = ps + } + build := func() error { + cctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), jsAPITimeout()) + defer cancel() + cfg := jetstream.StreamConfig{ + Name: streamName, + Subjects: streamSubjectsFor(addr.GetName()), + Storage: jetstream.FileStorage, + Retention: jetstream.LimitsPolicy, + Duplicates: defaultDedupWindow, + // Replicas via NATSJS_STREAM_REPLICAS (default 1). Set to 3 against a + // clustered nats-server for the HA-equivalent of RabbitMQ quorum queues. + Replicas: streamReplicas(), + // MaxAge/MaxBytes bound the log so it does not grow without limit the way + // the default LimitsPolicy otherwise would (RabbitMQ deletes on ack; + // JetStream retains acked messages). MaxAge is the time guard, MaxBytes an + // optional hard storage cap. Both are mutable, so this also reins in + // streams created before the limits existed. See natsjs-connector.md. + MaxAge: streamMaxAge(), + MaxBytes: streamMaxBytes(), + } + // The source set is carried forward on EVERY assertion, not just the + // ones that declare a binding of their own: CreateOrUpdateStream + // replaces it wholesale, so a config that omits it unbinds every + // address-to-address binding another subscriber declared on this + // stream. Nothing here unbinds (see mergeBoundSources), and an + // ordinary publisher to a bound address — which names no parent, so it + // reaches this path with no bindings — must not be the thing that does. + existing, serr := p.existingConfig(cctx, bd, streamName) + if serr != nil { + return serr + } + if existing != nil { + // Its OWN copy, independent of the snapshot in existing: + // mergeBoundSources appends to a StreamSource's transforms in + // place, so sharing them would mutate the very thing the config is + // about to be compared against — the merge would compare equal to + // itself and a newly declared binding would be skipped instead of + // written. Only visible when the parent is already in the source + // set (the second binding onto one parent), since a first binding + // appends a fresh element and changes the length. + cfg.Sources = copySources(existing.Sources) + } + if len(bindings) > 0 { + cfg.Sources = mergeBoundSources(cfg.Sources, parentStream, bindings) + } + // Assert only when the assertion would actually change something. + // Carrying the source set forward makes this a read-modify-write, and + // bindMu serializes that only within one process — Arke runs more than + // one replica, so two processes interleaving a read and a write on one + // stream can still drop a binding the other just added. Skipping the + // write when the live config already satisfies this caller removes the + // overwhelmingly common case from that race entirely: a bare assertion + // against a steady-state stream (every publisher's first touch, every + // reconnect) now reads and stops. What remains is two callers that both + // genuinely change the config, which is far rarer and, for bindings, + // symmetric — both are adding. + // + // A stream created before these limits existed still gets retrofitted: + // its MaxAge/MaxBytes differ, so it does not satisfy the config and the + // update runs. + if existing != nil && streamConfigSatisfied(existing, &cfg) { + return nil + } + _, err := bd.js.CreateOrUpdateStream(cctx, cfg) + return err + } + // build reads the stream's current binding set and writes back a superset, + // so the read-modify-write has to be serialized or one concurrent caller's + // bindings are lost — including a caller that only meant to assert the + // stream exists. The streams registry cannot do this job: it *collapses* + // concurrent callers, which is right when they all want the same stream but + // would silently drop one of two different binding sets. + var err error + if len(bindings) > 0 { + p.bindMu.Lock() + err = build() + p.bindMu.Unlock() + } else { + err = p.streams.ensure(ctx, streamEnsureKey(bd.connectionConfig, streamName), func() error { + p.bindMu.Lock() + defer p.bindMu.Unlock() + return build() + }) + } + if err != nil { + return "", err + } + bd.knownStreams.Add(streamName, true) + return streamName, nil +} + +// lookupStream answers the stream name backing an address only if the stream +// already exists on the server; it never creates one. The unary publish path +// uses it for STREAM addresses, where declaring the stream is the reader's job +// (see PublishOne). A hit is memoized alongside ensureStreamFor's entries: +// either way the connection has established that the stream exists, which is +// all the memo records. +func (p *natsjsProvider) lookupStream(ctx context.Context, bd *natsBrokerDetails, addr *pb.Address) (string, error) { + streamName := streamNameFor(addr.GetName()) + if _, ok := bd.knownStreams.Get(streamName); ok { + return streamName, nil + } + lctx, cancel := context.WithTimeout(ctx, jsAPITimeout()) + defer cancel() + if _, err := bd.js.Stream(lctx, streamName); err != nil { + return "", err + } + bd.knownStreams.Add(streamName, true) + return streamName, nil +} + +// existingConfig reads a stream's live configuration — its address-to-address +// bindings and the limits an assertion may need to update — or nil if the +// stream does not exist yet. Stream() itself fetches the stream info, so +// CachedInfo costs nothing further. +func (p *natsjsProvider) existingConfig(ctx context.Context, bd *natsBrokerDetails, + streamName string) (*jetstream.StreamConfig, error) { + st, err := bd.js.Stream(ctx, streamName) + if errors.Is(err, jetstream.ErrStreamNotFound) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read existing stream config: %w", err) + } + if info := st.CachedInfo(); info != nil { + cfg := info.Config + // Deep-copy the source set. mergeBoundSources appends to a + // StreamSource's transforms in place, and the value CachedInfo hands + // back shares those pointers — so without this the merged config and + // the config it is compared against would be the same objects, + // streamConfigSatisfied would always agree, and a newly declared + // binding would be silently skipped instead of written. + cfg.Sources = copySources(cfg.Sources) + return &cfg, nil + } + return nil, nil +} + +func copySources(in []*jetstream.StreamSource) []*jetstream.StreamSource { + if in == nil { + return nil + } + out := make([]*jetstream.StreamSource, 0, len(in)) + for _, s := range in { + if s == nil { + continue + } + c := *s + c.SubjectTransforms = slices.Clone(s.SubjectTransforms) + out = append(out, &c) + } + return out +} + +// streamConfigSatisfied reports whether a stream's live config already provides +// everything an assertion asks for, so the assertion can skip its update (see +// ensureStreamFor). Only the fields the connector sets are compared; anything +// an operator tuned outside them is left alone, exactly as CreateOrUpdateStream +// would leave it. +func streamConfigSatisfied(live, want *jetstream.StreamConfig) bool { + if live.Storage != want.Storage || live.Retention != want.Retention || + live.Duplicates != want.Duplicates || live.Replicas != want.Replicas || + live.MaxAge != want.MaxAge || !sameLimit(live.MaxBytes, want.MaxBytes) { + return false + } + if !sameStringSet(live.Subjects, want.Subjects) { + return false + } + return sameSources(live.Sources, want.Sources) +} + +// sameLimit compares two JetStream size/count limits, treating every +// non-positive value as the same "unlimited". The server normalizes an unset +// limit to -1 and echoes it back that way, so a config asking for 0 describes +// the same stream as a live -1 — comparing them literally would make every +// assertion look like a change. +func sameLimit(live, want int64) bool { + if live <= 0 && want <= 0 { + return true + } + return live == want +} + +func sameStringSet(a, b []string) bool { + if len(a) != len(b) { + return false + } + seen := make(map[string]int, len(a)) + for _, s := range a { + seen[s]++ + } + for _, s := range b { + seen[s]-- + if seen[s] < 0 { + return false + } + } + return true +} + +// sameSources compares two address-to-address binding sets by the fields +// mergeBoundSources builds: the parent stream each sources from and the +// subjects it routes in. Order is irrelevant on both levels — the server may +// return either in any order. +func sameSources(live, want []*jetstream.StreamSource) bool { + if len(live) != len(want) { + return false + } + byName := make(map[string]*jetstream.StreamSource, len(live)) + for _, s := range live { + byName[s.Name] = s + } + for _, w := range want { + l, ok := byName[w.Name] + if !ok { + return false + } + if !sameStringSet(transformSubjects(l), transformSubjects(w)) { + return false + } + } + return true +} + +func transformSubjects(s *jetstream.StreamSource) []string { + out := make([]string, 0, len(s.SubjectTransforms)) + for _, t := range s.SubjectTransforms { + out = append(out, t.Source+">"+t.Destination) + } + return out +} + +// mergeBoundSources merges bindings into a stream's existing source set, so a +// second address-to-address binding adds to the first rather than replacing +// it. Bindings accumulate the same way on RabbitMQ, where declaring one never +// removes another; nothing here unbinds, so a binding outlives the +// subscription that declared it, exactly as an AMQP exchange-to-exchange +// binding outlives the client that bound it. +func mergeBoundSources(sources []*jetstream.StreamSource, + parentStream string, bindings []string) []*jetstream.StreamSource { + var from *jetstream.StreamSource + for _, s := range sources { + if s.Name == parentStream { + from = s + break + } + } + if from == nil { + from = &jetstream.StreamSource{Name: parentStream} + sources = append(sources, from) + } + for _, subject := range bindings { + if slices.ContainsFunc(from.SubjectTransforms, func(t jetstream.SubjectTransformConfig) bool { + return t.Source == subject + }) { + continue + } + // Destination == Source: the copy keeps the subject it was published + // under, which is what lets one consumer filter select both the + // messages routed in from the parent and those published here directly. + from.SubjectTransforms = append(from.SubjectTransforms, + jetstream.SubjectTransformConfig{Source: subject, Destination: subject}) + } + return sources +} + +func firstSubject(addr *pb.Address) string { + if subs := addr.GetSubjects(); len(subs) > 0 { + return subs[0] + } + return "" +} + +// Publish drains the message channel, publishing each message to JetStream. +func (p *natsjsProvider) Publish(ctx context.Context, in <-chan *pb.Message, errChan chan<- *pb.Error) *pb.Error { + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return &pb.Error{Message: err.Error(), IsFatal: true} + } + for { + select { + case <-ctx.Done(): + return nil + case msg, ok := <-in: + if !ok || msg == nil { + return nil + } + // The server blocks on <-errChan for every message it hands the + // provider (server.go: `mc <- msg` then `pubErr := <-ec`), so we + // must always reply exactly once — nil on success. + if msg.GetConfirm() { + // The streaming publish is the fire-and-forget path; a client + // that wants a broker confirmation calls PublishOne, which + // returns one. amqp091 refuses the combination and so does + // this: a JetStream publish is acknowledged either way, so + // quietly accepting Confirm here would leave a client believing + // the option means the same thing on both brokers when only one + // of them honours it. + errChan <- &pb.Error{Message: unsupportedConfirmError} + continue + } + errChan <- p.publishMsg(ctx, bd, msg, false /* unary */) + } + } +} + +func (p *natsjsProvider) PublishOne(ctx context.Context, msg *pb.Message) *pb.Error { + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return &pb.Error{Message: err.Error(), IsFatal: true} + } + return p.publishMsg(ctx, bd, msg, true /* unary */) +} + +// publishMsg publishes one message. unary distinguishes PublishOne from the +// streaming Publish, which amqp091 routes through entirely different client +// code — so two parts of the contract below differ by path, not just by +// address type. +func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, msg *pb.Message, unary bool) *pb.Error { + addr := msg.GetAddress() + // Dedup is a STREAM-address feature on both brokers: RabbitMQ gets it from + // the Streams client, which is the only thing amqp091's publish paths + // reach for a STREAM address. + dedupAddress := addr.GetType() == pb.Address_STREAM + // Message-contract refusals come before any topology is touched: amqp091 + // refuses these before it reaches for the stream or exchange, so a publish + // that is wrong twice over reports the contract error, not the missing + // stream — and a refused publish creates nothing. + if msg.GetPublishId() > 0 && unary && !dedupAddress { + // The refusal covers every non-STREAM address, not just Address_QUEUE: + // amqp091's PublishOne routes STREAM to publishOneStream and everything + // else — TOPIC, QUEUE and FILTER alike — to publishOneQueue, which is + // where the refusal lives (amqp091.go:1501). TOPIC matters most: it is + // the proto zero value, so an address that never sets a type takes this + // path. JetStream would happily deduplicate any of them, since dedup + // here is a property of the stream rather than of the address type; + // refusing anyway keeps the option meaning one thing across brokers, + // instead of letting a client write a publish that works here and + // fails outright on RabbitMQ. + return &pb.Error{Message: queueDedupError} + } + // The companion refusal, and unary-only for the same reason: it lives in + // amqp091's publishOneStream (amqp091.go:1544), which only PublishOne + // reaches. The streaming Publish calls prepareAndSend directly and reads + // neither field, so a streaming publish carrying an id but no publisher + // name is accepted on RabbitMQ — refusing it here would break a publisher + // that works there today. + if unary && msg.GetPublishId() > 0 && msg.GetPublisherName() == "" { + return &pb.Error{Message: "PublisherName not set on message, PublisherName is required when PublishID is set"} + } + // A STREAM address must name a stream that already exists: amqp091 routes + // unary stream publishes through the RabbitMQ Streams client, which + // refuses a stream nobody declared (declaring is the reader's job — + // streamSubscribe's DeclareStream). Auto-creating here instead would turn + // a typo'd address name into a junk stream that swallows messages no + // reader will ever see. Only the unary path checks: amqp091's streaming + // Publish sends every address type, STREAM included, over an + // auto-declared exchange and reports no error either, so the streaming + // path keeps the auto-create. + assertStream := p.ensureStreamFor + if unary && dedupAddress { + assertStream = p.lookupStream + } + streamName, err := assertStream(ctx, bd, addr) + if errors.Is(err, jetstream.ErrStreamNotFound) { + return &pb.Error{Message: fmt.Sprintf(streamMissingError, addr.GetName())} + } + if err != nil { + return &pb.Error{Message: fmt.Sprintf("ensure stream: %s", err.Error())} + } + nmsg := &nats.Msg{ + Subject: publishSubjectFor(addr.GetName(), firstSubject(addr)), + Data: msg.GetBody(), + Header: pbToNatsHeader(msg.GetHeaders()), + } + var pubOpts []jetstream.PublishOpt + // publish_id + publisher_name -> Nats-Msg-Id (dedup within the stream window), + // but only where amqp091 would also have deduplicated: the unary path, on a + // STREAM address. That is the one combination RabbitMQ deduplicates on, + // because publishOneStream is the only place amqp091 forwards the id + // (streamPrepareAndSend, amqp091.go:1638) and PublishOne is the only caller + // that routes there. + // + // Everywhere else the id is ignored rather than honoured, and the streaming + // path is ignored for EVERY address type, STREAM included: amqp091's + // Publish hands them all to the plain channel publish, which drops the id + // on the floor. Deduplicating a STREAM address here because the type looks + // right would silently discard a message RabbitMQ stores — the client + // reusing an id on that path is doing something RabbitMQ told it was free. + // (The unary path never reaches here on a non-STREAM address; it is refused + // above.) + if unary && msg.GetPublishId() > 0 && dedupAddress { + pubOpts = append(pubOpts, jetstream.WithMsgID(fmt.Sprintf("%s-%d", msg.GetPublisherName(), msg.GetPublishId()))) + } + _, perr := bd.js.PublishMsg(ctx, nmsg, pubOpts...) + if errors.Is(perr, jetstream.ErrNoStreamResponse) { + // No stream captured the subject even though this connection ensured + // one: the stream has been deleted underneath the connection (an + // operator reset, a storage wipe). The memoized entry would pin that + // stale answer for the connection's lifetime, failing every subsequent + // publish — and a NATS client outlives broker state changes that would + // sever an AMQP connection. Drop the entry, re-assert the stream, and + // retry once; the first attempt was not stored (no stream answered), + // so the retry cannot duplicate it. For a STREAM address the + // re-assertion is the existence check again, so a stream deleted out + // from under the connection is reported missing, not resurrected. + bd.knownStreams.Delete(streamName) + if _, rerr := assertStream(ctx, bd, addr); rerr == nil { + _, perr = bd.js.PublishMsg(ctx, nmsg, pubOpts...) + } else if errors.Is(rerr, jetstream.ErrStreamNotFound) { + return &pb.Error{Message: fmt.Sprintf(streamMissingError, addr.GetName())} + } + } + if perr != nil { + return &pb.Error{Message: fmt.Sprintf("publish: %s", perr.Error())} + } + atomic.AddInt64(&bd.produced, 1) + return nil +} + +// Subscribe ensures topology, creates a consumer, and forwards deliveries to the +// message channel until the context is cancelled. Blocks per the Provider +// contract. It mirrors amqp091's queueSubscribe: one linear setup path with +// many small guards (hence the complexity nolint). +func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out chan<- *pb.Message) *pb.Error { //nolint:gocognit,gocyclo + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return &pb.Error{Message: err.Error(), IsFatal: true} + } + // MessageTTL is accepted so existing client sources validate against + // SupportedSourceOptions, but retention here is stream-wide: warn + // instead of silently ignoring it, so a source owner asking for a short + // TTL learns their data outlives it without reading the design doc. + if unapplied := unappliedSourceOptions(source); len(unapplied) > 0 { + util.Logger.Warn( + "natsjs: source {0} sets {1}, which natsjs accepts but does not apply; retention is stream-wide (NATSJS_STREAM_MAX_AGE / NATSJS_STREAM_MAX_BYTES)", + source.GetName(), strings.Join(unapplied, ", ")) + } + // Expires, by contrast, IS applied: it becomes the consumer's + // InactiveThreshold (see below), so validate it up front before any + // topology is created. + expires, xerr := expiresThreshold(source) + if xerr != nil { + return &pb.Error{ + Message: fmt.Sprintf("source %q: %s", source.GetName(), xerr.Error()), + IsFatal: true, + } + } + // A single-active stream source must name the ConsumerGroup its instances + // coordinate through (see durableName); amqp091 rejects this combination + // with the same message. Without a group each subscriber would get its own + // durable and they would all compete — the opposite of single-active. + if source.GetSingleActiveConsumer() && source.GetType() == pb.Source_STREAM && + source.GetOptions()["ConsumerGroup"] == "" { + return &pb.Error{ + Message: fmt.Sprintf("source %q: single active consumer requested but no ConsumerGroup option set", source.GetName()), + IsFatal: true, + } + } + // Also validated before any topology is created: an unrecognized Offset is + // a refusal, and a refused subscribe must not leave a stream behind for an + // address the client only ever named by mistake. + deliverPolicy, startSeq, derr := deliverPolicyFor(source) + if derr != nil { + return &pb.Error{Message: fmt.Sprintf("source %q: %s", source.GetName(), derr.Error()), IsFatal: true} + } + addr := source.GetAddress() + streamName, serr := p.ensureStreamFor(ctx, bd, addr) + if serr != nil { + return &pb.Error{Message: fmt.Sprintf("ensure stream: %s", serr.Error()), IsFatal: true} + } + // The management API calls below get the same explicit deadline as stream + // creation (see jsAPITimeout): creating a replicated consumer forms its own + // raft group, which can outlast the client library's 5s default while the + // server is cold or busy. + tctx, tcancel := context.WithTimeout(ctx, jsAPITimeout()) + defer tcancel() + stream, serr := bd.js.Stream(tctx, streamName) + if errors.Is(serr, jetstream.ErrStreamNotFound) { + // Same stale-memo case as the publish path: the stream was deleted out + // from under the connection after this connection ensured it. Drop the + // entry and re-assert the stream instead of failing the subscribe until + // the client reconnects. + bd.knownStreams.Delete(streamName) + if _, rerr := p.ensureStreamFor(ctx, bd, addr); rerr == nil { + stream, serr = bd.js.Stream(tctx, streamName) + } + } + if serr != nil { + return &pb.Error{Message: fmt.Sprintf("open stream: %s", serr.Error()), IsFatal: true} + } + + // AMQP prefetch 0 means unlimited (amqp091 leaves the channel default in + // that case rather than calling SetPrefetch), so map it to JetStream's + // explicit unlimited (-1) — not to the server default of 1000, and + // certainly not to 1, which would silently serialize the consumer. Note + // that Arke's gRPC server raises a prefetch below 1 to 1 before any + // provider sees the source (SetSourceDefaults, internal/server), so a + // subscribe arriving through the server never takes this branch; it keeps + // the provider contract honest for direct (in-process) users and for any + // future change to the server-side defaulting. + prefetch := int(source.GetPrefetchCount()) + if prefetch <= 0 { + prefetch = -1 + } + consCfg := jetstream.ConsumerConfig{ + FilterSubjects: filterSubjectsFor(source), + AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: deliverPolicy, + OptStartSeq: startSeq, // 0 (ignored) unless DeliverByStartSequence + AckWait: ackWait(), + MaxAckPending: prefetch, + } + // Durable work-queue consumer for non-transient QUEUE / stream-group sources: + // it persists across client reconnects so a backlog published during a + // consumer outage is redelivered (RabbitMQ durable-queue parity). The + // DeliverPolicy only applies on first creation; reconnects resume from the + // durable's stored ack position. Transient (auto-delete/exclusive/TEMPORARY) + // sources stay ephemeral and auto-expire after InactiveThreshold. + // + // The Expires option (AMQP x-expires: delete the queue after this many ms + // without consumers) maps onto InactiveThreshold for BOTH kinds — the + // broker deletes the consumer once nothing has pulled from it for the + // threshold, exactly the disuse-deletion x-expires asks for. Only the + // consumer is deleted; the messages stay in the shared per-address stream + // under its own retention (see Known limitations in the design doc). + // Unset keeps the amqp091 defaults: transient sources expire after + // defaultInactiveThreshold (amqp091 defaults auto-delete/exclusive queues + // to the same 5 minutes), durables never expire. + consCfg.InactiveThreshold = expires + durable := durableName(source) + adoptedDLQ := false + if durable == "" { + // A transient source is ordinarily an independent ephemeral reader, but + // the reader of a dead-letter queue is transient by convention while + // needing the opposite of a tail start: it exists to read what was + // dead-lettered before it attached. If setupDeadLetter pre-declared a + // durable under this source's name, that durable is the queue being + // named, so adopt it and inherit its position. + var marker map[string]string + durable, marker = adoptableDLQDurable(tctx, stream, source) + adoptedDLQ = durable != "" + // Carried forward so attaching does not erase the marker it matched on + // (see adoptableDLQDurable). + consCfg.Metadata = marker + } + if durable != "" { + consCfg.Durable = durable + if adoptedDLQ && expires == 0 { + // Keep the bound setupDeadLetter declared. Adoption otherwise + // rewrites the pre-declared threshold to "never expire", so a + // reader that attached once and left would silently convert its + // dead-letter queue into permanent server state — the leak + // dlqDeclareThreshold exists to bound. An explicit Expires still + // wins, as it does for every other source. + consCfg.InactiveThreshold = dlqDeclareThreshold() + } + if source.GetSingleActiveConsumer() { + // RabbitMQ's single-active-consumer delivers to one consumer at a + // time — the point is ordered processing across competing + // instances — and fails over when that consumer goes away. The + // JetStream equivalent is a pinned-client priority group + // (nats-server 2.11+): the server pins the first client to pull + // and standbys' pulls wait; when the pinned client stops pulling + // for PinnedTTL (or its pin is otherwise released) the next + // standby takes over. + consCfg.PriorityPolicy = jetstream.PriorityPolicyPinned + consCfg.PriorityGroups = []string{sacPriorityGroup} + consCfg.PinnedTTL = sacPinnedTTL() + } + } else if expires == 0 { + consCfg.InactiveThreshold = defaultInactiveThreshold + } + + // Competing consumers on one durable each apply their own proxy-side header + // filter, so reject a second live subscriber whose header filter differs + // rather than silently dropping the messages JetStream routes to the + // "wrong" one (see natsBrokerDetails.durableFilters). Claimed before the + // consumer is touched and held for the life of this Subscribe; the defer + // covers every return path below. + if consCfg.Durable != "" { + if err := bd.claimDurableFilter(consCfg.Durable, headerFilterFingerprint(source.GetFilters())); err != nil { + return &pb.Error{Message: fmt.Sprintf("source %q: %s", source.GetName(), err.Error()), IsFatal: true} + } + defer bd.releaseDurableFilter(consCfg.Durable) + } + + // A durable that previously had SingleActiveConsumer pinned a client to + // its priority group. If this subscribe no longer asks for that, the pin + // must be released HERE, before CreateOrUpdateConsumer below clears the + // group from the consumer's declared config: once the group is gone from + // the config, unpinning it errors ("priority group does not exist for + // this consumer" — verified against an embedded server), so a pin + // released only after the update can no longer be released at all. Left + // alone, the stale pin blocks ordinary (non-priority-group) delivery for + // up to NATSJS_SAC_PINNED_TTL (default 1m) with no error or warning + // anywhere, even though the live config already shows no priority policy. + if consCfg.Durable != "" && !source.GetSingleActiveConsumer() { + releaseStalePins(tctx, stream, consCfg.Durable) + } + + cons, cerr := stream.CreateOrUpdateConsumer(tctx, consCfg) + if cerr != nil && consCfg.Durable != "" && isStartPositionConflict(cerr) { + // A durable's start position (DeliverPolicy/OptStartSeq) is fixed at + // creation — JetStream rejects updates to it (err 10012), so a client + // re-subscribing with a different Offset would otherwise fail here + // forever. The documented contract is that Offset applies on first + // creation only and a reconnecting durable resumes from its stored ack + // position, so attach to the existing consumer as-is; to reposition, + // use a new durable (source/ConsumerGroup) name. Only start-position + // conflicts are absorbed this way: any other configuration error stays + // fatal, because attaching to a consumer whose effective config + // silently differs from the requested one (wrong ack policy, stale + // filter subjects) would consume the wrong way without any signal. + if existing, aerr := stream.Consumer(tctx, consCfg.Durable); aerr == nil { + util.Logger.Warn( + "natsjs: durable consumer {0} exists with a different start position ({1}); resuming it unchanged", + consCfg.Durable, cerr.Error()) + cons, cerr = existing, nil + } + } + if cerr != nil { + // Include the mapped filter subjects: a NATS "invalid subject" (10052) + // otherwise gives no hint which source/subject produced it. + return &pb.Error{ + Message: fmt.Sprintf("create consumer (filters=%v): %s", consCfg.FilterSubjects, cerr.Error()), + IsFatal: true, + } + } + + // Pre-declare the dead-letter reader's consumer now that this source's own + // topology exists, matching where amqp091 does it (after its queue and + // binding, before the DeclareOnly return) so a declare-only subscribe + // establishes the same topology on both connectors. + p.setupDeadLetter(tctx, bd, source) + + // DeclareOnly: topology established, do not consume. An ephemeral consumer + // created only to validate that topology is garbage the moment we return — + // its server-generated name is never seen again — so delete it eagerly + // rather than letting it linger for the full inactivity threshold. + if source.GetDeclareOnly() { + if consCfg.Durable == "" { + deleteEphemeralConsumer(ctx, stream, cons.CachedInfo().Name) + } + return nil + } + + // consumerLost carries the first authoritative sign that this + // subscription's server-side consumer no longer exists (deleted + // administratively, or expired after an outage longer than the ephemeral + // inactivity threshold). The final wait below turns it into a + // subscription-ending error so the client's re-subscribe recreates the + // consumer — without it the subscription would sit deaf forever, warning + // (or, after a terminal consume error, silently) while delivering + // nothing. RabbitMQ parity: a deleted queue closes its consumers' + // channel, and the re-subscribe re-declares it. + consumerName := cons.CachedInfo().Name + consumerLost := make(chan error, 1) + noteLost := func(err error) { + select { + case consumerLost <- err: + default: + } + } + // probeConsumer asks the server whether the consumer still exists. A + // consumer that vanished while no pull was pending (expiry during a long + // outage) never produces an authoritative error on the consume path — + // re-issued pulls just find no responder and heartbeats go missing, + // indefinitely — so those symptoms trigger this bounded, single-flight + // lookup instead. Only an explicit "not found" answer counts (for the + // consumer, or for the whole stream — a wiped store takes both): a + // timeout or network failure proves nothing about the consumer. + var probing atomic.Bool + probeConsumer := func() { + if bd.state.Load() != provider.CONNECTED || !probing.CompareAndSwap(false, true) { + return + } + go func() { + defer probing.Store(false) + pctx, pcancel := context.WithTimeout(context.WithoutCancel(ctx), ephemeralDeleteTimeout) + defer pcancel() + if _, perr := stream.Consumer(pctx, consumerName); errors.Is(perr, jetstream.ErrConsumerNotFound) || + errors.Is(perr, jetstream.ErrStreamNotFound) { + noteLost(perr) + } + }() + } + consumeOpts := []jetstream.PullConsumeOpt{ + // A short idle heartbeat plus an error handler keep a stalled delivery + // path from failing silently: if the server stops serving this + // consumer's pulls (broker restart, consumer raft leader not yet + // serving after creation), the missed heartbeat is logged at warn and + // the library re-issues its pull request after ~2x the heartbeat, + // instead of ~30s — and invisibly — with the defaults. Standby pulls + // of a pinned consumer receive idle heartbeats too, so single-active + // standbys do not trip this. + jetstream.PullHeartbeat(defaultConsumeHeartbeat), + jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { + util.Logger.Warn("natsjs: consume error on source {0}: {1}", source.GetName(), err.Error()) + // The deleted/not-found errors are authoritative; anything else + // (missed heartbeats, pulls with no responder) is only a symptom + // that warrants checking. + if errors.Is(err, jetstream.ErrConsumerDeleted) || errors.Is(err, jetstream.ErrConsumerNotFound) { + noteLost(err) + return + } + probeConsumer() + }), + } + // Pull requests must carry the priority group exactly when the consumer + // has one, so key off the effective (server-reported) config, not the + // requested one: a server too old for priority groups (< 2.11) silently + // drops the fields, and an existing durable the subscribe merely attached + // to may predate them. In either case single-active cannot be enforced — + // consumers compete, today's behavior — and that degradation must be + // visible, not silent. + if groups := cons.CachedInfo().Config.PriorityGroups; len(groups) > 0 { + consumeOpts = append(consumeOpts, jetstream.PullPriorityGroup(groups[0])) + } else if consCfg.PriorityPolicy == jetstream.PriorityPolicyPinned { + util.Logger.Warn( + "natsjs: source {0} requested a single active consumer but the consumer has no priority group "+ + "(broker predates them, or an existing durable could not be updated); consumers will compete", + source.GetName()) + } + // subKey tags this subscription: it keys its consume context and stamps the + // messages it delivers (inflightMsg), so teardown releases only its own + // deliveries even when it shares a server-side consumer with siblings. + // Generated before Consume so the delivery callback can capture it. + subKey := fmt.Sprintf("%s#%d", source.GetName(), bd.subscriptionSeq.Add(1)) + cc, cerr := cons.Consume(func(m jetstream.Msg) { + p.handleDelivery(ctx, bd, source, subKey, m, out) + }, consumeOpts...) + if cerr != nil { + if consCfg.Durable == "" { + deleteEphemeralConsumer(ctx, stream, consumerName) + } + return &pb.Error{Message: fmt.Sprintf("consume: %s", cerr.Error()), IsFatal: true} + } + bd.consumeContexts.Add(subKey, cc) + defer func() { + cc.Stop() + // Stop is asynchronous: wait (bounded) for the consume loop to finish + // unsubscribing, then flush so the server has processed the + // unsubscribe before the naks below — otherwise a nak'd message can + // be redelivered straight into this subscription's still-registered + // pull request and sit unclaimed until AckWait expires it again. + select { + case <-cc.Closed(): + case <-time.After(ephemeralDeleteTimeout): + } + _ = bd.nc.FlushTimeout(ephemeralDeleteTimeout) + bd.consumeContexts.Delete(subKey) + releaseInFlight(bd, subKey, consCfg.Durable != "") + if consCfg.Durable == "" { + deleteEphemeralConsumer(ctx, stream, consumerName) + } + }() + + select { + case <-ctx.Done(): + return nil + case lerr := <-consumerLost: + return consumerLostError(source, consumerName, lerr) + case <-cc.Closed(): + // The consume machinery stopped on its own: the library treats a + // handful of consume errors as terminal ("consumer deleted" when the + // server removes the consumer under a pending pull) and silently + // stops the ConsumeContext. Blocking on ctx here regardless would + // leave a subscription that never delivers again. Closed() also + // fires for this connection's own teardown (Disconnect stops every + // consume context, Drain closes the subscriptions), which is not an + // error — the context or connection state says which case this is. + if bd.clientDisconnect.Load() && ctx.Err() == nil { + // Client-initiated Disconnect with the consume stream still + // open: do not end the subscription — returning here would end + // the caller's whole consume stream, but the amqp091 connector + // leaves it open (its subscribe loop just goes quiet once the + // AMQP connection closes), so a client that disconnects and + // then acks its in-flight messages gets each ack answered with + // a "could not retrieve broker details" failure rather than + // end-of-stream. Delivery cannot resume — the consume context + // is stopped and the connection is draining — so just wait for + // the client to close its stream. Scoped to the client's own + // Disconnect (not any CLOSED state): a connection the library + // closed terminally still ends the subscription. + <-ctx.Done() + return nil + } + if ctx.Err() != nil || bd.state.Load() == provider.CLOSED { + return nil + } + select { + case lerr := <-consumerLost: + return consumerLostError(source, consumerName, lerr) + default: + } + return &pb.Error{Message: fmt.Sprintf( + "source %q: consuming stopped unexpectedly (consumer %s); re-subscribe to recreate the consumer", + source.GetName(), consumerName)} + } +} + +// consumerLostError is the non-fatal error Subscribe ends with when the +// server-side consumer disappeared: non-fatal so the client's re-subscribe +// path runs Subscribe again, which recreates the consumer (and, after a +// storage wipe, the stream). Matches the amqp091 connector, which ends a +// subscription with a non-fatal error when the broker closes the channel of +// a deleted queue. +func consumerLostError(source *pb.Source, consumerName string, err error) *pb.Error { + return &pb.Error{Message: fmt.Sprintf( + "source %q: server-side consumer %s no longer exists (%s); re-subscribe to recreate it", + source.GetName(), consumerName, err.Error())} +} + +// releaseInFlight drops a just-ended subscription's unresolved deliveries from +// activeMessages, identified by the owning subKey. Acks travel on the consume +// stream that closed, so nothing can resolve these uuids anymore; left alone +// they would sit in the map for the life of the connection (a leak that also +// inflates the active-message stat). Durable consumers additionally get a +// best-effort Nak so the messages redeliver promptly — RabbitMQ requeues a +// closed channel's unacked deliveries immediately, whereas an untouched +// JetStream delivery waits out the full AckWait. Ephemeral consumers skip the +// Nak: the consumer is deleted on this same teardown path, which discards its +// delivery state anyway. +// +// Ownership is by subKey, not by the (stream, consumer) pair the delivery +// carries: several subscriptions can share one server-side consumer (competing +// consumers on a durable, single-active standbys), so filtering on the +// consumer would release a sibling's still-in-flight messages — failing the +// sibling's later ack and duplicating the message. A message the owner +// resolves concurrently with teardown is deleted by whichever side gets there +// first; the loser's ack/nak fails and is ignored (at-least-once either way). +func releaseInFlight(bd *natsBrokerDetails, subKey string, nak bool) { + // Claim the entries under activeMu (so a concurrent mark cannot reinsert + // one), then do the network-bound naks outside it. + released := make(map[string]jetstream.Msg) + bd.activeMu.Lock() + for _, uuid := range bd.activeMessages.GetList() { + mu, ok := bd.activeMessages.Get(uuid) + if !ok { + continue + } + im := mu.(inflightMsg) + if im.subKey != subKey { + continue + } + bd.activeMessages.Delete(uuid) + released[uuid] = im.msg + } + bd.activeMu.Unlock() + if !nak { + return + } + for uuid, msg := range released { + if err := msg.Nak(); err != nil { + util.Logger.Debugf("natsjs: nak of in-flight message %s at subscription end: %s", uuid, err.Error()) + } + } +} + +// claimDurableFilter registers this subscription's header-filter fingerprint +// against a durable. The first subscriber sets the fingerprint; later +// subscribers must match it (they share the server-side consumer, so their +// proxy-side header filters must agree or one drops the other's messages — +// see natsBrokerDetails.durableFilters). A mismatch is refused; a match takes +// a reference. Every successful claim must be paired with releaseDurableFilter. +func (bd *natsBrokerDetails) claimDurableFilter(durable, fingerprint string) error { + bd.durableFilterMu.Lock() + defer bd.durableFilterMu.Unlock() + if use, ok := bd.durableFilters[durable]; ok { + if use.fingerprint != fingerprint { + return fmt.Errorf( + "durable consumer %q already has a live subscriber with a different header filter; "+ + "competing consumers on one durable must share their header filters", durable) + } + use.refs++ + return nil + } + bd.durableFilters[durable] = &durableFilterUse{fingerprint: fingerprint, refs: 1} + return nil +} + +// releaseDurableFilter drops one reference taken by claimDurableFilter, +// forgetting the durable's fingerprint once the last live subscriber leaves so +// a later re-subscribe may legitimately declare a new filter. +func (bd *natsBrokerDetails) releaseDurableFilter(durable string) { + bd.durableFilterMu.Lock() + defer bd.durableFilterMu.Unlock() + if use, ok := bd.durableFilters[durable]; ok { + use.refs-- + if use.refs <= 0 { + delete(bd.durableFilters, durable) + } + } +} + +// headerFilterFingerprint canonically (order-independently) serializes a +// source's proxy-side header filters so two subscriptions sharing a durable can +// be compared. Only the header filters (evaluateFilters) are fingerprinted; +// subject filters are the server's and update the shared consumer in place. +func headerFilterFingerprint(filters []*pb.Filter) string { + if len(filters) == 0 { + return "" + } + parts := make([]string, 0, len(filters)) + for _, f := range filters { + matches := make([]string, 0, len(f.GetMatches())) + for _, m := range f.GetMatches() { + matches = append(matches, fmt.Sprintf("%q=%q", m.GetName(), m.GetValue())) + } + sort.Strings(matches) + parts = append(parts, fmt.Sprintf("t%d[%s]", f.GetType(), strings.Join(matches, ","))) + } + sort.Strings(parts) + return strings.Join(parts, ";") +} + +// deleteEphemeralConsumer removes an ephemeral consumer as soon as its +// subscription ends. Ephemeral consumers do expire on their own after +// InactiveThreshold, but until then each one still counts against the server's +// per-stream consumer limit, so clients that churn transient subscriptions +// faster than the threshold would accumulate dead consumers on the server. +// Deleting eagerly leaves the threshold as a janitor for unclean exits (a +// crashed client never reaches this path) only. Best-effort: the subscription +// context is already cancelled when this runs, so the call gets a detached, +// bounded context, and a failure is only logged — the consumer then simply +// expires via the threshold as before. +func deleteEphemeralConsumer(ctx context.Context, stream jetstream.Stream, name string) { + dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), ephemeralDeleteTimeout) + defer cancel() + if err := stream.DeleteConsumer(dctx, name); err != nil { + util.Logger.Debugf("natsjs: delete ephemeral consumer %s: %s", name, err.Error()) + } +} + +// unappliedSourceOptions returns the retention options the source sets that +// natsjs accepts (so sources written for the amqp091 connector still validate) +// but does not apply — retention on natsjs is stream-wide (see streamMaxAge +// and the design doc's Known limitations). Expires is NOT in this list: it +// maps onto the consumer's InactiveThreshold (see expiresThreshold). +func unappliedSourceOptions(source *pb.Source) []string { + var unapplied []string + for _, opt := range []string{"MessageTTL"} { + if source.GetOptions()[opt] != "" { + unapplied = append(unapplied, opt) + } + } + return unapplied +} + +// expiresThreshold parses the Expires source option — AMQP x-expires, the +// number of milliseconds a queue may go unused before the broker deletes it — +// into the consumer InactiveThreshold that implements the same disuse +// deletion here. Zero (with nil error) means the option is unset and the +// caller applies its kind's default. The error strings mirror the amqp091 +// connector, which validates the same option the same way; the broker itself +// rejects a non-positive x-expires, so that case is refused here too rather +// than passed through (a zero threshold would mean "never expire" for a +// durable but "server default" for an ephemeral — neither is what the client +// asked for). +func expiresThreshold(source *pb.Source) (time.Duration, error) { + raw := source.GetOptions()["Expires"] + if raw == "" { + return 0, nil + } + val, err := strconv.Atoi(raw) + if err != nil { + return 0, errors.New("value for Expires option must be a quoted integer") + } + if val <= 0 { + return 0, errors.New("value for Expires option must be a positive integer") + } + return time.Duration(val) * time.Millisecond, nil +} + +// deliverPolicyFor maps the RabbitMQ-Streams `Offset` option onto a JetStream +// DeliverPolicy, mirroring the amqp091 connector's toStreamOffset so both +// connectors accept the same offset vocabulary. The second return value is the +// start sequence, meaningful only when the policy is DeliverByStartSequence. +// +// Mapping: `first`/`continue` -> deliver all (a durable "continue"s from its +// stored ack floor on reconnect regardless, so all-from-start is the correct +// first-creation fallback); `last` -> the final message; `next`/"" -> only +// messages published after the consumer is created; an absolute number -> +// start at that stream sequence. +// +// NOTE: a numeric offset is an offset in the Arke/RabbitMQ-Streams sense — +// 0 names the first message in the log — not a raw JetStream sequence, so it +// round-trips through SourceStats (see offsetOf/seqOf). It still is not +// portable *across* brokers: offset 7 names the eighth message of whichever +// log is being read, and two brokers' logs hold different messages. An +// unrecognized offset is an error, like toStreamOffset: starting a consumer at +// a silently different position than the one it asked for loses or replays +// data. +func deliverPolicyFor(source *pb.Source) (jetstream.DeliverPolicy, uint64, error) { + off := source.GetOptions()["Offset"] + switch strings.ToLower(off) { + case "first", "continue": + return jetstream.DeliverAllPolicy, 0, nil + case "last": + return jetstream.DeliverLastPolicy, 0, nil + case "next", "": + return jetstream.DeliverNewPolicy, 0, nil + default: + if offset, err := strconv.ParseUint(strings.TrimSpace(off), 10, 64); err == nil { + if offset == 0 { + return jetstream.DeliverAllPolicy, 0, nil + } + return jetstream.DeliverByStartSequencePolicy, seqOf(offset), nil + } + return jetstream.DeliverNewPolicy, 0, fmt.Errorf( + "invalid offset: %q (expected first, continue, last, next, or a numeric stream sequence)", off) + } +} + +// startPositionConflictMessages are the server's refusals to change a durable +// consumer's start position, which JetStream fixes at creation. They arrive +// wrapped in the generic consumer-create API error (err 10012) whose +// description carries the specific refusal, so the description is what +// distinguishes an immutable start position from every other create or update +// failure sharing that code. +var startPositionConflictMessages = []string{ + "deliver policy can not be updated", + "start sequence can not be updated", + "start time can not be updated", +} + +// releaseStalePins unpins every priority group an existing durable's live +// config still carries, ahead of a subscribe that is about to strip +// SingleActiveConsumer's config from it (see the call site in Subscribe). +// Best-effort and read-first: a durable that does not exist yet (the +// ordinary first-subscribe case) answers ErrConsumerNotFound immediately and +// nothing is unpinned; a durable that was never single-active has an empty +// group list and the loop below does nothing. Only a durable coming OFF a +// pin pays the extra UnpinConsumer call. +func releaseStalePins(ctx context.Context, stream jetstream.Stream, durable string) { + existing, err := stream.Consumer(ctx, durable) + if err != nil { + return + } + for _, group := range existing.CachedInfo().Config.PriorityGroups { + if uerr := stream.UnpinConsumer(ctx, durable, group); uerr != nil { + util.Logger.Debugf("natsjs: release stale single-active pin on %s (group %s): %s", + durable, group, uerr.Error()) + } + } +} + +// isStartPositionConflict reports whether err is JetStream refusing to move an +// existing durable consumer's start position (DeliverPolicy / OptStartSeq / +// OptStartTime) — the one configuration conflict Subscribe absorbs by +// attaching to the existing consumer unchanged. +func isStartPositionConflict(err error) bool { + var apiErr *jetstream.APIError + if !errors.As(err, &apiErr) || apiErr.ErrorCode != jetstream.JSErrCodeConsumerCreate { + return false + } + for _, m := range startPositionConflictMessages { + if strings.Contains(apiErr.Description, m) { + return true + } + } + return false +} + +func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetails, source *pb.Source, subKey string, m jetstream.Msg, out chan<- *pb.Message) { + defer util.RecoverPanic() + + headers := natsToPbHeader(m.Headers()) + body := m.Data() + // Synthesize the headers a RabbitMQ client receives that have no NATS + // equivalent on the wire. Both come from the same metadata read. + if md, err := m.Metadata(); err == nil { + // x-retry-count from JetStream's delivery count, so a client's retry + // policy works without RabbitMQ's x-death header. + if md.NumDelivered > 1 { + headers[retryCountHeaderName] = strconv.FormatUint(md.NumDelivered-1, 10) + } + // timestamp_in_ms unconditionally, matching the overwrite = true + // RabbitMQ ingress interceptor it stands in for. + headers[timestampHeaderName] = strconv.FormatInt(md.Timestamp.UnixMilli(), 10) + // x-current-offset, STREAM sources only, matching amqp091's + // streamSubscribe (queue/topic sources never carry it on either + // connector). The message's own stream sequence is this reader's + // current position in the log at the point of delivery. + if source.GetType() == pb.Source_STREAM { + headers[currentOffsetHeaderName] = strconv.FormatInt(offsetOf(md.Sequence.Stream), 10) + } + } + + // STREAM-source parity with amqp091's streamSubscribe: undo a publisher's + // (or amqp091's own 1MiB-workaround) gzip compression rather than handing + // a consumer raw compressed bytes it never asked for. Queue/topic sources + // never decompress on either connector. + if source.GetType() == pb.Source_STREAM && headers[transferEncodingHeaderName] == "gzip" { + if decompressed, derr := decompressBody(body); derr != nil { + util.Logger.Debugf("natsjs: failed to decompress message on source %s: %s", + source.GetName(), derr.Error()) + } else { + body = decompressed + delete(headers, transferEncodingHeaderName) + } + } + + // Proxy-side replacement for RabbitMQ headers-exchange routing. A failed + // ack here only means the message will be redelivered and re-filtered — + // harmless, but it inflates NumDelivered (and so the synthesized + // x-retry-count) of a message the client never saw, so leave a trace. + if !evaluateFilters(source.GetFilters(), headers) { + if err := m.Ack(); err != nil { + util.Logger.Debugf("natsjs: ack of header-filtered message on source %s failed: %s", + source.GetName(), err.Error()) + } + return + } + + // Mirrors amqp091's queueSubscribe: start (or continue, if the publisher + // already set traceparent/tracestate) a span for this delivery and, when + // tracing is enabled, write its context back into the headers the + // consumer receives. Without this a distributed trace has no record of + // arke's own "received from broker" hop on natsjs, even though amqp091 + // always injects one. + _, span := tracing.SpanFromHeaders(ctx, headers, source.GetAddress().GetName()+" received from broker", trace.SpanKindInternal) + if tracing.Enabled() { + span.SetAttributes(attribute.String("source.name", source.GetName()), + attribute.String("messaging.client_id", bd.clientIdentifier)) + headers[tracing.HeaderTraceState] = span.SpanContext().TraceState().String() + headers[tracing.HeaderTraceParent] = fmt.Sprintf("00-%s-%s-%s", + span.SpanContext().TraceID().String(), + span.SpanContext().SpanID().String(), + span.SpanContext().TraceFlags()) + } + span.End() + + uuid := util.GenUUID() + bd.activeMessages.Add(uuid, inflightMsg{msg: m, subKey: subKey}) + pbmsg := &pb.Message{Uuid: uuid, Headers: headers, Body: body} + + select { + case out <- pbmsg: + atomic.AddInt64(&bd.consumed, 1) + case <-ctx.Done(): + // The subscription ended before the server took the message: drop the + // claim and nak (best-effort) so a durable redelivers it promptly + // instead of after AckWait. Harmless for ephemerals, whose consumer is + // deleted on teardown. + bd.activeMessages.Delete(uuid) + _ = m.Nak() + } +} + +func (p *natsjsProvider) takeMessage(ctx context.Context, uuid string) (jetstream.Msg, *pb.Error) { + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return nil, &pb.Error{Message: err.Error()} + } + bd.activeMu.Lock() + defer bd.activeMu.Unlock() + mu, ok := bd.activeMessages.Get(uuid) + if !ok { + return nil, &pb.Error{Message: fmt.Sprintf(noMessageError, uuid)} + } + bd.activeMessages.Delete(uuid) + return mu.(inflightMsg).msg, nil +} + +// takeMessageForNack is takeMessage plus the message's redeliverOnNack flag, +// which only Nack needs. +func (p *natsjsProvider) takeMessageForNack(ctx context.Context, uuid string) (jetstream.Msg, bool, *pb.Error) { + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return nil, false, &pb.Error{Message: err.Error()} + } + bd.activeMu.Lock() + defer bd.activeMu.Unlock() + mu, ok := bd.activeMessages.Get(uuid) + if !ok { + return nil, false, &pb.Error{Message: fmt.Sprintf(noMessageError, uuid)} + } + bd.activeMessages.Delete(uuid) + im := mu.(inflightMsg) + return im.msg, im.redeliverOnNack, nil +} + +// markForRedelivery records that a nack of this message must put it back +// rather than terminate it. Called on every path where DeadLetter gives up +// with the message still in flight: the error it returns makes the server nack +// the same uuid, and that nack is the retry, not a rejection. +func markForRedelivery(bd *natsBrokerDetails, uuid string) { + bd.activeMu.Lock() + defer bd.activeMu.Unlock() + mu, ok := bd.activeMessages.Get(uuid) + if !ok { + return + } + im := mu.(inflightMsg) + im.redeliverOnNack = true + bd.activeMessages.Add(uuid, im) +} + +func (p *natsjsProvider) Ack(ctx context.Context, uuid string) *pb.Error { + m, perr := p.takeMessage(ctx, uuid) + if perr != nil { + return perr + } + if err := m.Ack(); err != nil { + return &pb.Error{Message: err.Error()} + } + return nil +} + +// Nack rejects a message without putting it back, which is what a nack means +// on the Arke contract: amqp091 answers the same call with +// Delivery.Nack(requeue=false), so RabbitMQ drops the message (or moves it to +// the queue's dead-letter exchange, which the server instead asks for +// explicitly through DeadLetter). JetStream's Nak means the opposite — +// redeliver immediately — so naking here turns one nacked message into an +// unbounded redelivery loop between server and client, at thousands of +// deliveries a second, for as long as the subscription lives. Term is the +// primitive that carries the intended meaning: stop redelivering. +// +// The exception is the server's fallback nack after a failed DeadLetter, which +// exists to put the message back so dead-lettering can be retried. DeadLetter +// marks the message for redelivery before returning that error. +func (p *natsjsProvider) Nack(ctx context.Context, uuid string) *pb.Error { + m, redeliver, perr := p.takeMessageForNack(ctx, uuid) + if perr != nil { + return perr + } + if redeliver { + // Delayed, not immediate: this redelivery is a dead-letter retry, and + // retrying a permanently-failing dead-letter as fast as the client can + // nack is a redelivery storm nothing bounds. See deadLetterRetryDelay. + if err := m.NakWithDelay(deadLetterRetryDelay()); err != nil { + return &pb.Error{Message: err.Error()} + } + return nil + } + if err := m.Term(); err != nil { + return &pb.Error{Message: err.Error()} + } + return nil +} + +// Retry requeues the message after a delay. NakWithDelay is the native +// JetStream primitive that replaces RabbitMQ's per-message-TTL + dead-letter +// retry-queue idiom; JetStream increments the delivery count for us, which +// handleDelivery surfaces as x-retry-count. +// +// Mirrors DeadLetter's ordering: the server falls back to nacking this same +// uuid when Retry returns an error (server.go), and that fallback can only +// resolve anything while the uuid is still in activeMessages. Deleting it +// up front (as a plain takeMessage would) strands the message the moment +// NakWithDelay fails — neither nak'd nor resolvable — until AckWait expires +// on its own. markForRedelivery also makes that fallback nack mean "put it +// back", not "reject": a freshly delivered message's redeliverOnNack default +// is false, and a failed Retry is a request to try again, not to give up. +func (p *natsjsProvider) Retry(ctx context.Context, _ *pb.Source, uuid string, delay int32) *pb.Error { + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return &pb.Error{Message: err.Error()} + } + mu, ok := bd.activeMessages.Get(uuid) + if !ok { + return &pb.Error{Message: fmt.Sprintf(noMessageError, uuid)} + } + m := mu.(inflightMsg).msg + markForRedelivery(bd, uuid) + if err := m.NakWithDelay(time.Duration(delay) * time.Second); err != nil { + return &pb.Error{Message: err.Error()} + } + bd.activeMu.Lock() + bd.activeMessages.Delete(uuid) + bd.activeMu.Unlock() + return nil +} + +// setupDeadLetter pre-declares the consumer that holds a source's dead letters, +// mirroring the amqp091 connector's setupDeadLetter. Best-effort: a failure +// here leaves dead-lettering itself working (DeadLetter ensures the stream on +// its own) and must not fail the subscribe, which is how amqp091 treats it too. +// +// Why this exists at all. On RabbitMQ the pass is not a broker property: arke +// declares a queue named dlqSourceName(source) and binds it to the DLX the +// moment a source declares a DeadLetterAddress, before any message can be +// dead-lettered. From then on that queue accumulates, so a consumer attaching +// later — after the failures, which is the normal shape for a DLQ reader — +// finds them waiting. natsjs had no equivalent: it ensured the DLQ stream +// lazily inside DeadLetter, and a late reader (Source_TEMPORARY, hence +// ephemeral, hence DeliverNew) started at the stream's tail and saw an empty +// queue while its messages sat in the stream just behind it. +// +// The faithful start position is therefore neither "tail at attach" (what we +// did: misses everything) nor DeliverAll (replays up to MaxAge of history the +// RabbitMQ queue never held). It is the tail *as of declare time*, which is +// exactly what a durable created here with DeliverNew records, and what the +// reader inherits by adopting it (adoptableDLQDurable). +func (p *natsjsProvider) setupDeadLetter(ctx context.Context, bd *natsBrokerDetails, origSource *pb.Source) { + opts := origSource.GetOptions() + dla := opts["DeadLetterAddress"] + // Absent or empty are both no-ops. Empty is not hypothetical: amqp091's own + // Retry builds a source with DeadLetterAddress "" to mean "expire back to + // the default exchange", and nothing should be declared for that. + if dla == "" { + return + } + durable := streamNameFor(dlqSourceName(origSource.GetName())) + + streamName, serr := p.ensureStream(ctx, bd, dla) + if serr != nil { + util.Logger.Warn( + "natsjs: source {0} declares dead-letter address {1} but its stream could not be ensured ({2}); "+ + "dead letters published before a reader attaches will not be visible to it", + origSource.GetName(), dla, serr.Error()) + return + } + stream, serr := bd.js.Stream(ctx, streamName) + if serr != nil { + util.Logger.Warn("natsjs: open dead-letter stream {0} for source {1}: {2}", + streamName, origSource.GetName(), serr.Error()) + return + } + // Already declared: skip the create. This saves a management round trip on + // every subscribe of a source with a dead-letter address; it is not what + // protects the position a queue has been accumulating. Nothing here could + // move that anyway — after creation a durable's position is its stored ack + // floor, not its declared DeliverPolicy, so re-asserting the same config is + // a no-op for it, and a genuinely different start position is refused + // outright (err 10012, which Subscribe absorbs separately). Creating rather + // than updating below keeps that true even if this check is removed. + if _, cerr := stream.Consumer(ctx, durable); cerr == nil { + return + } + // Capture the whole dead-letter address, not just where DeadLetterSubject + // says this source's copies will land. Narrowing here would be false + // precision: what a reader ultimately sees is decided by the filter IT + // asks for, because adopting rewrites the filter (mutable) while + // inheriting the position (not). The pre-declared filter therefore only + // affects num_pending until someone attaches — and a source with no + // DeadLetterSubject cannot be narrowed anyway, since DeadLetter then + // preserves each message's own routing key. + if _, cerr := stream.CreateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: durable, + FilterSubjects: wholeAddressSubjects(dla), + AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: jetstream.DeliverNewPolicy, + AckWait: ackWait(), + InactiveThreshold: dlqDeclareThreshold(), + Metadata: map[string]string{dlqDurableMetadataKey: origSource.GetName()}, + }); cerr != nil { + util.Logger.Warn( + "natsjs: pre-declare of dead-letter consumer {0} for source {1} failed ({2}); "+ + "dead-lettering still works, but a reader attaching later may not see earlier dead letters", + durable, origSource.GetName(), cerr.Error()) + } +} + +// adoptableDLQDurable returns the pre-declared dead-letter durable this source +// is the reader for, or "" if there is none. +// +// A DLQ reader is conventionally transient (the suite's dead-letter tests use +// Source_TEMPORARY and AutoDelete), which would otherwise make it ephemeral and +// start it at the stream tail — after the dead letters it exists to read. When +// setupDeadLetter has already declared a durable under this source's name, that +// durable *is* the queue the reader is naming, so attach to it and inherit the +// position it has been holding since the source was declared. +// +// Adoption is deliberately gated on the marker metadata, not on the name alone: +// name-only matching would let any transient source hijack an unrelated durable +// that happens to share its name, taking over another subscriber's pending +// messages and ack position. A lookup miss of any kind means no adoption. +// +// It returns the metadata to carry forward alongside the name. That is not +// cosmetic: attaching re-asserts the consumer's whole config, and a config +// without the marker REPLACES the stored metadata and erases it — after which +// the durable is unadoptable and the next reader of the same queue silently +// falls back to an ephemeral at the tail, which is the exact failure this +// whole mechanism exists to prevent. setupDeadLetter cannot repair that later: +// it leaves an existing consumer alone precisely so it never moves a position +// that has been accumulating. +func adoptableDLQDurable(ctx context.Context, stream jetstream.Stream, source *pb.Source) (string, map[string]string) { + // Only a source named like a dead-letter reader can match one, so check the + // name before asking the server. This is exact, not a heuristic: the only + // durables setupDeadLetter creates are named streamNameFor(.dlq), + // and streamNameFor is injective, so a match requires the reader's own name + // to end in the suffix. Skipping the lookup matters because every OTHER + // transient subscribe would otherwise pay a management round trip whose + // answer is always "consumer not found" — and that answer is an error + // response, so it also inflates the JetStream API error counter that + // deployments watch as a health signal, worst during the reconnect storm + // after a restart when every transient consumer re-subscribes at once. + if !strings.HasSuffix(source.GetName(), dlqSourceNameSuffix) { + return "", nil + } + durable := streamNameFor(source.GetName()) + cons, err := stream.Consumer(ctx, durable) + if err != nil { + return "", nil + } + marker, ok := cons.CachedInfo().Config.Metadata[dlqDurableMetadataKey] + if !ok { + return "", nil + } + return durable, map[string]string{dlqDurableMetadataKey: marker} +} + +// DeadLetter publishes the message to the configured dead-letter subject (there +// is no native DLX in JetStream) and terminates redelivery. +// +// RabbitMQ moves a nacked message to its DLX broker-side; here the move is two +// proxy-side steps, so ordering is what protects the data: the original is +// Term'd only after the DLQ publish succeeds. If the DLQ stream cannot be +// ensured or published to, the message is left in flight — still ack-pending on +// the server and still resolvable by uuid — and an error is returned, so the +// caller's fallback (the server nacks on a DeadLetter error) redelivers it and +// dead-lettering is retried, instead of the message being removed from +// redelivery while also absent from the DLQ. The DLQ publish carries a +// Nats-Msg-Id derived from the original's stream sequence, so a retried +// dead-letter of the same message cannot duplicate it in the DLQ within the +// dedup window, and the x-retry-count the consumer saw when it gave the +// message up (the proxy-side analogue of the x-death trail RabbitMQ's +// broker-side move preserves). +func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid string) *pb.Error { + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return &pb.Error{Message: err.Error()} + } + mu, ok := bd.activeMessages.Get(uuid) + if !ok { + return &pb.Error{Message: fmt.Sprintf(noMessageError, uuid)} + } + m := mu.(inflightMsg).msg + // Every way out of this function short of a stored DLQ copy leaves the + // message in flight for the caller's fallback nack to retry, so make that + // nack mean "put it back" for as long as the attempt is unresolved. The + // success path removes the message from activeMessages outright, so the + // flag cannot outlive a dead-letter that worked. + markForRedelivery(bd, uuid) + opts := source.GetOptions() + dla, hasDLA := opts["DeadLetterAddress"] + if !hasDLA { + return &pb.Error{Message: "DeadLetterAddress not set"} + } + if dla == "" { + return &pb.Error{Message: "DeadLetterAddress is empty"} + } + if _, serr := p.ensureStream(ctx, bd, dla); serr != nil { + util.Logger.Warn( + "natsjs: dead-letter of message {0} failed (ensure stream for {1}: {2}); message stays in flight", + uuid, dla, serr.Error()) + return &pb.Error{Message: fmt.Sprintf("dead-letter ensure stream: %s", serr.Error())} + } + // Copied so neither the retry count below nor the publish options + // mutate the original's header map. + dlqHeader := copyHeader(m.Headers()) + var pubOpts []jetstream.PublishOpt + if md, merr := m.Metadata(); merr == nil { + pubOpts = append(pubOpts, + jetstream.WithMsgID(fmt.Sprintf("dlq-%s-%d", md.Stream, md.Sequence.Stream))) + // RabbitMQ stamps a dead-lettered copy broker-side (x-death) with + // how it died; the JetStream copy is a plain publish that would + // otherwise lose that trail. Carry the same retry count the + // consumer saw when it gave the message up, synthesized exactly + // like handleDelivery does for deliveries. + if md.NumDelivered > 1 { + if dlqHeader == nil { + dlqHeader = nats.Header{} + } + dlqHeader.Set(retryCountHeaderName, strconv.FormatUint(md.NumDelivered-1, 10)) + } + } + // RabbitMQ dead-letters a message under its original routing key + // unless the queue overrides it (x-dead-letter-routing-key, which is + // what DeadLetterSubject maps to on the AMQP side). Preserve the + // original key the same way when no override is set, so DLQ consumers + // can bind by routing key and reprocessing tools can see where each + // message was originally headed. + dlSubject := opts["DeadLetterSubject"] + if dlSubject == "" { + dlSubject = routingKeyFromSubject(m.Subject()) + } + dlqMsg := &nats.Msg{ + Subject: publishSubjectFor(dla, dlSubject), + Data: m.Data(), + Header: dlqHeader, + } + if _, perr := bd.js.PublishMsg(ctx, dlqMsg, pubOpts...); perr != nil { + if errors.Is(perr, jetstream.ErrNoStreamResponse) { + // The DLQ stream was deleted after this connection ensured it. + // Drop the memoized entry so the dead-letter retry that follows + // the returned error (the server nacks, the message redelivers) + // re-creates the stream instead of failing the same way forever. + bd.knownStreams.Delete(streamNameFor(dla)) + } + util.Logger.Warn( + "natsjs: dead-letter of message {0} failed (publish to {1}: {2}); message stays in flight", + uuid, dlqMsg.Subject, perr.Error()) + return &pb.Error{Message: fmt.Sprintf("dead-letter publish: %s", perr.Error())} + } + // Term only after the DLQ copy is safely stored, and drop the active-message + // entry only after Term succeeds: if Term fails, DeadLetter returns an error + // and the server falls back to nacking this uuid (server.go), which can only + // resolve while the entry is still present. Deleting first would strand the + // original ack-pending until AckWait instead of redelivering promptly. + if err := m.Term(); err != nil { + return &pb.Error{Message: err.Error()} + } + bd.activeMu.Lock() + bd.activeMessages.Delete(uuid) + bd.activeMu.Unlock() + return nil +} + +func (p *natsjsProvider) SupportedSourceOptions() map[string]bool { + return supportedSourceOptions +} + +func (p *natsjsProvider) Stats() *provider.Stats { + stats := &provider.Stats{} + for _, id := range p.connections.GetList() { + bd := p.getBrokerDetailsByIdentifier(id) + if bd == nil { + continue + } + stats.Clients = append(stats.Clients, &provider.ClientStats{ + ID: id, + ActiveMessages: bd.activeMessages.Length(), + Streams: bd.consumeContexts.Length(), + Produced: int(atomic.LoadInt64(&bd.produced)), + Consumed: int(atomic.LoadInt64(&bd.consumed)), + }) + } + return stats +} + +// rateTracker derives messages-per-second rates from the monotonic counters +// JetStream exposes. RabbitMQ's management API computes publish/deliver rates +// server-side; JetStream reports only absolute counters, so the connector +// differences them between successive SourceStats calls. The first +// observation of a key establishes a baseline and reports zero, as does a +// counter that moved backwards (a recreated stream or consumer). +type rateTracker struct { + mu sync.Mutex + samples map[string]rateSample +} + +type rateSample struct { + when time.Time + count uint64 +} + +func newRateTracker() *rateTracker { + return &rateTracker{samples: make(map[string]rateSample)} +} + +// observe records (now, count) for key and returns the average rate since the +// previous observation of that key. +func (rt *rateTracker) observe(key string, now time.Time, count uint64) float32 { + rt.mu.Lock() + defer rt.mu.Unlock() + prev, ok := rt.samples[key] + rt.samples[key] = rateSample{when: now, count: count} + if !ok || count < prev.count || !now.After(prev.when) { + return 0 + } + return float32(float64(count-prev.count) / now.Sub(prev.when).Seconds()) +} + +// offsetOf converts a JetStream stream sequence to the Offset vocabulary Arke +// reports and accepts. The two do not share an origin: JetStream numbers the +// first message in a stream 1, while a RabbitMQ Stream — the vocabulary +// amqp091 established and clients read SourceStats through — numbers it 0. A +// sequence of 0 is JetStream for "no message", which has no offset, and stays +// 0 rather than going negative; callers distinguish it by LastSeq or by the +// "Offset not found" error. seqOf inverts this. +func offsetOf(seq uint64) int64 { + if seq == 0 { + return 0 + } + return int64(seq - 1) //nolint:gosec // a stream sequence cannot exceed int64 +} + +// seqOf converts an Arke Offset back to the JetStream stream sequence to start +// a consumer at. It inverts offsetOf, so an offset read from SourceStats and +// handed back as a source's Offset resumes at exactly the message it named. +func seqOf(offset uint64) uint64 { return offset + 1 } + +func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb.SourceStats { + stats := &pb.SourceStats{Name: source.GetName()} + bd, err := p.getBrokerDetails(ctx) + if err != nil { + stats.Error = &pb.Error{Message: err.Error()} + return stats + } + streamName := streamNameFor(source.GetAddress().GetName()) + stream, serr := bd.js.Stream(ctx, streamName) + if serr != nil { + stats.Error = &pb.Error{Message: serr.Error()} + return stats + } + info, ierr := stream.Info(ctx) + if ierr != nil { + stats.Error = &pb.Error{Message: ierr.Error()} + return stats + } + now := time.Now() + stats.MessageCount = int64(info.State.Msgs) //nolint:gosec + stats.ConsumerCount = int32(info.State.Consumers) //nolint:gosec + stats.LastOffset = offsetOf(info.State.LastSeq) + durable := durableName(source) + // JetStream exposes no rates, only counters; sample them between calls + // (see rateTracker). The stream's LastSeq counts every publish to the + // address root, so the publish rate is per address, not per binding. + // The sample key includes the polling identity — durable and source name, + // since consumer groups share a source name and differ only by their + // durable: the counters are shared, but each poller runs on its own + // cadence, and a shared key would make every observation window the gap + // since whichever poller came last — with several pollers on one address + // the later polls of a cycle would report rates over milliseconds instead + // of the poll interval. + stats.PublishRate = bd.rates.observe("pub/"+streamName+"/"+durable+"/"+source.GetName(), now, info.State.LastSeq) + // For a source with a durable consumer, report that consumer's actual + // backlog (undelivered + delivered-but-unacked) instead of the stream + // depth: the stream retains acked messages under its retention limits, so + // its message count keeps growing after consumers are caught up. This + // matches the amqp091 connector, which reports the queue's ready+unacked + // count — the number consumer-scaling logic wants. Without a durable + // consumer (or before it exists), fall back to the stream view. + if durable != "" { + if cons, cerr := stream.Consumer(ctx, durable); cerr == nil { + // Consumer() fetched fresh info to return the handle; reuse it + // instead of paying a second round-trip on every stats poll. + ci := cons.CachedInfo() + stats.MessageCount = int64(ci.NumPending) + int64(ci.NumAckPending) //nolint:gosec + stats.CurrentOffset = offsetOf(ci.AckFloor.Stream) + // The stream-wide consumer count set above spans every source on + // the address (streams are shared per address root), so it says + // nothing about THIS source — it cannot even distinguish zero + // attached clients from many, because the durable itself counts. + // The per-source equivalent of RabbitMQ's queue consumer count is + // the clients pulling this durable: report its open pull requests. + // (A client whose pull buffer is full can briefly read as zero.) + stats.ConsumerCount = int32(ci.NumWaiting) //nolint:gosec + // Delivered.Consumer counts deliveries (redeliveries included, + // like RabbitMQ's deliver rate). Ephemeral sources have no + // consumer identity to sample here, so their rate stays zero. + // Keyed per polling source like the publish rate above (sources + // can share a durable through a common ConsumerGroup). + stats.DeliverRate = bd.rates.observe("del/"+streamName+"/"+durable+"/"+source.GetName(), now, ci.Delivered.Consumer) + } + } + // A stream source reports on a log, not on a queue, and amqp091 says so in + // two ways this has to match. It fills a message count only for + // Source_QUEUE (getStreamOrQueueStats): a stream's readers each hold their + // own position in one retained log, so there is no single backlog that + // belongs to the source — the position is the reading, and it is already + // in LastOffset/CurrentOffset. And when the log has no offset to report it + // answers with RabbitMQ's own "Offset not found" (updateStatsForStream) + // rather than a silent zero. + if source.GetType() == pb.Source_STREAM { + stats.MessageCount = 0 + if info.State.LastSeq == 0 { + stats.Error = &pb.Error{Message: offsetNotFoundError} + } + } + return stats +} diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go new file mode 100644 index 0000000..5f09d44 --- /dev/null +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -0,0 +1,4212 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package natsjs + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "net" + "os" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + pb "github.com/sassoftware/arke/api" + "github.com/sassoftware/arke/internal/util/tracing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The connector resolves the per-client broker state from the client identifier +// in the context. For tests we override it with a fixed identifier (the real +// implementation reads it from gRPC metadata). +func init() { + GetClientIdentifier = func(context.Context) (string, error) { + return "test-client", nil + } +} + +// runJetStreamServer starts an in-process NATS server with JetStream enabled, +// backed by a temp store dir that is cleaned up with the test. +func runJetStreamServer(t *testing.T) *natsserver.Server { + t.Helper() + return runJetStreamServerAt(t, -1) // choose a free port +} + +// runJetStreamServerAt is runJetStreamServer on a caller-chosen port, so a +// test can restart "the broker" at the address a client is connected to — +// with a fresh store dir, i.e. a broker that lost its JetStream state. +func runJetStreamServerAt(t *testing.T, port int) *natsserver.Server { + t.Helper() + opts := &natsserver.Options{ + Host: "127.0.0.1", + Port: port, + JetStream: true, + StoreDir: t.TempDir(), + NoLog: true, + NoSigs: true, + } + s, err := natsserver.NewServer(opts) + require.NoError(t, err) + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("embedded nats server did not become ready") + } + t.Cleanup(s.Shutdown) + return s +} + +// connectClient builds a provider and connects a single client to the server. +func connectClient(t *testing.T, s *natsserver.Server) (*natsjsProvider, context.Context) { + t.Helper() + addr := s.Addr().(*net.TCPAddr) + p := NewNATSJetStreamProvider().(*natsjsProvider) + cfg := &pb.ConnectionConfiguration{ + Host: "127.0.0.1", + Port: int32(addr.Port), //nolint:gosec // local server port fits int32 + ClientName: "test", + } + ctx := context.Background() + if perr := p.Connect(ctx, cfg, false); perr != nil { + t.Fatalf("connect: %s", perr.GetMessage()) + } + if !p.WaitForConnect(ctx) { + t.Fatal("WaitForConnect returned false against a live server") + } + return p, ctx +} + +func queueSource(name, addr string, subjects ...string) *pb.Source { + return &pb.Source{ + Name: name, + Type: pb.Source_QUEUE, + Address: &pb.Address{Name: addr, Subjects: subjects}, + // Offset:first => deliver from the start so tests can publish before + // subscribing without racing the consumer setup. + Options: map[string]string{"Offset": "first"}, + } +} + +// recv waits for one message on out or fails after a generous timeout. +func recv(t *testing.T, out <-chan *pb.Message) *pb.Message { + t.Helper() + select { + case m := <-out: + return m + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for a message") + return nil + } +} + +func TestConnectDisconnect(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + assert.True(t, p.ClientExists("test-client")) + + p.Disconnect(ctx) + assert.False(t, p.ClientExists("test-client")) +} + +// TestSubscribeOutlivesDisconnect: a client-initiated Disconnect must not end +// a live Subscribe — ending it ends the caller's whole consume stream, but +// the amqp091 connector leaves that stream open (its subscribe loop goes +// quiet when the AMQP connection closes), so a client that disconnects and +// then acks straggler messages gets per-ack failures, not end-of-stream. +// Subscribe may only return once the subscription's own context ends. +func TestSubscribeOutlivesDisconnect(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + bd := p.getBrokerDetailsByIdentifier("test-client") + + src := queueSource("events.disc.q", "events.disc", "e") + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan *pb.Error, 1) + go func() { done <- p.Subscribe(subCtx, src, make(chan *pb.Message, 1)) }() + + require.Eventually(t, func() bool { + stream, serr := bd.js.Stream(ctx, streamNameFor("events.disc")) + if serr != nil { + return false + } + info, ierr := stream.Info(ctx) + return ierr == nil && info.State.Consumers == 1 + }, 10*time.Second, 20*time.Millisecond, "subscription never established") + + p.Disconnect(ctx) + + select { + case serr := <-done: + t.Fatalf("Subscribe ended on Disconnect (%v); it must stay open until the stream context ends", serr) + case <-time.After(2 * time.Second): + } + + cancel() + select { + case serr := <-done: + assert.Nil(t, serr, "a disconnected subscription ends cleanly with its context") + case <-time.After(10 * time.Second): + t.Fatal("Subscribe did not return after its context ended") + } +} + +// TestConcurrentConnectClosesRedundantLink: the gRPC server's "already +// connected?" check is not atomic, so two Connect calls for one client +// identifier can both reach the provider. Only one connection may survive; the +// other's nats.Conn must be closed rather than orphaned to reconnect forever +// (MaxReconnects(-1)). The proof is the server seeing exactly one live client. +func TestConcurrentConnectClosesRedundantLink(t *testing.T) { + s := runJetStreamServer(t) + addr := s.Addr().(*net.TCPAddr) + p := NewNATSJetStreamProvider().(*natsjsProvider) + cfg := &pb.ConnectionConfiguration{ + Host: "127.0.0.1", + Port: int32(addr.Port), //nolint:gosec // local server port fits int32 + ClientName: "test", + } + + ctx := context.Background() + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if perr := p.Connect(ctx, cfg, false); perr != nil { + t.Errorf("connect: %s", perr.GetMessage()) + } + }() + } + wg.Wait() + require.True(t, p.WaitForConnect(ctx)) + + // Exactly one live server-side client connection: the redundant dial was + // closed. Drain can lag the close slightly, so let it settle. + require.Eventually(t, func() bool { + return s.NumClients() == 1 + }, 5*time.Second, 50*time.Millisecond, + "a racing Connect must leave exactly one live NATS connection, not leak the loser") + + p.Disconnect(ctx) +} + +func TestSupportedSourceOptions(t *testing.T) { + p := NewNATSJetStreamProvider().(*natsjsProvider) + opts := p.SupportedSourceOptions() + for _, k := range []string{"MessageTTL", "DeadLetterAddress", "DeadLetterSubject", "Expires", "Offset", "ConsumerGroup"} { + assert.True(t, opts[k], "expected %s to be supported", k) + } + assert.False(t, opts["NotARealOption"]) +} + +func TestPublishOneAndSubscribe(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.orders", Subjects: []string{"created"}} + const n = 5 + for i := 0; i < n; i++ { + msg := &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))} + if perr := p.PublishOne(ctx, msg); perr != nil { + t.Fatalf("publish %d: %s", i, perr.GetMessage()) + } + } + + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.orders.consumer", "events.orders", "created"), out) + + for i := 0; i < n; i++ { + m := recv(t, out) + assert.NoError(t, errOf(p.Ack(ctx, m.GetUuid()))) + } + + stats := p.Stats() + require.Len(t, stats.Clients, 1) + assert.Equal(t, n, stats.Clients[0].Produced) + assert.Equal(t, n, stats.Clients[0].Consumed) +} + +func TestPublishChannel(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + in := make(chan *pb.Message) + errChan := make(chan *pb.Error) + go func() { _ = p.Publish(ctx, in, errChan) }() + + addr := &pb.Address{Name: "events.audit", Subjects: []string{"login"}} + // The server protocol sends a message then blocks reading exactly one error. + in <- &pb.Message{Address: addr, Body: []byte("hello")} + assert.Nil(t, <-errChan, "publish should report nil on success") + close(in) +} + +func TestRetrySynthesizesRetryCount(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.retry", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("work")})) + + out := make(chan *pb.Message, 2) + src := queueSource("events.retry.consumer", "events.retry", "job") + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + first := recv(t, out) + assert.Empty(t, first.GetHeaders()[retryCountHeaderName], "first delivery has no retry count") + require.Nil(t, p.Retry(ctx, src, first.GetUuid(), 1)) + + second := recv(t, out) + assert.Equal(t, "1", second.GetHeaders()[retryCountHeaderName], "redelivery carries x-retry-count") + require.Nil(t, p.Ack(ctx, second.GetUuid())) +} + +// TestRetryFailureKeepsMessageResolvable: if NakWithDelay fails, Retry must +// leave the active-message entry in place instead of deleting it up front +// (which a plain takeMessage would). The server falls back to nacking this +// same uuid when Retry returns an error (server.go, mirroring the existing +// DeadLetter fallback) — a fallback that can only resolve anything while the +// uuid stays in activeMessages. Deleting it first strands the message with +// neither a nak nor a term having reached the server, until AckWait expires +// on its own instead of redelivering promptly. +func TestRetryFailureKeepsMessageResolvable(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.retryfail", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("work")})) + + out := make(chan *pb.Message, 1) + src := queueSource("events.retryfail.consumer", "events.retryfail", "job") + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + + // Force NakWithDelay to fail: ack the underlying message directly, + // bypassing the connector's own bookkeeping, so the entry stays in + // activeMessages but the message itself is already resolved server-side. + bd := p.getBrokerDetailsByIdentifier("test-client") + mu, ok := bd.activeMessages.Get(m.GetUuid()) + require.True(t, ok) + require.NoError(t, mu.(inflightMsg).msg.Ack()) + + perr := p.Retry(ctx, src, m.GetUuid(), 1) + require.NotNil(t, perr, "Retry must surface the NakWithDelay failure") + + entry, stillThere := bd.activeMessages.Get(m.GetUuid()) + require.True(t, stillThere, + "a failed NakWithDelay must not remove the active-message entry the fallback nack needs") + assert.True(t, entry.(inflightMsg).redeliverOnNack, + "the fallback nack must put the message back, not term it, after a failed retry") +} + +// TestNackDoesNotRedeliver pins Nack to amqp091's Delivery.Nack(requeue=false): +// a nacked message is rejected, not put back. Naking it instead (JetStream's +// Nak, which means redeliver now) turns one nacked message into an unbounded +// delivery loop, because the client nacks each redelivery straight back. +func TestNackDoesNotRedeliver(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.nack", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("work")})) + + out := make(chan *pb.Message, 8) + src := queueSource("events.nack.consumer", "events.nack", "job") + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + first := recv(t, out) + require.Nil(t, p.Nack(ctx, first.GetUuid())) + + select { + case m := <-out: + t.Fatalf("nacked message was redelivered (uuid %s); Nack must not requeue", m.GetUuid()) + case <-time.After(2 * time.Second): + } + + // Resolved server-side, not merely undelivered: nothing is left ack-pending + // to come back once AckWait expires. + stats := p.SourceStats(ctx, src) + require.Nil(t, stats.GetError()) + require.EqualValues(t, 0, stats.GetMessageCount(), "nacked message still pending") +} + +func TestDeadLetter(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dl", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 1) + src := queueSource("events.dl.consumer", "events.dl", "job") + src.Options["DeadLetterAddress"] = "events.dlq" + src.Options["DeadLetterSubject"] = "failed" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + // Fail the first delivery, then dead-letter the redelivery, so the copy + // has a retry trail to carry. Retry (not Nack) is what asks for a + // redelivery — Nack rejects the message outright. + first := recv(t, out) + require.Nil(t, p.Retry(ctx, src, first.GetUuid(), 0)) + m := recv(t, out) + require.Equal(t, "1", m.GetHeaders()[retryCountHeaderName], "redelivery precondition") + require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) + + // The dead-lettered copy lands on the DLQ stream. + dlqStats := p.SourceStats(ctx, &pb.Source{ + Name: "events.dlq.consumer", + Address: &pb.Address{Name: "events.dlq"}, + }) + assert.Nil(t, dlqStats.GetError()) + assert.Equal(t, int64(1), dlqStats.GetMessageCount()) + + // The copy carries a deterministic Nats-Msg-Id, so a retried dead-letter of + // the same message dedups in the DLQ instead of duplicating — and the same + // retry count the consumer saw when it gave the message up (RabbitMQ's + // broker-side dead-lettering preserves the death trail via x-death). + bd := p.getBrokerDetailsByIdentifier("test-client") + dlqStream, serr := bd.js.Stream(ctx, streamNameFor("events.dlq")) + require.NoError(t, serr) + raw, gerr := dlqStream.GetMsg(ctx, 1) + require.NoError(t, gerr) + assert.NotEmpty(t, raw.Header.Get("Nats-Msg-Id"), "DLQ copy must carry a dedup message id") + assert.Equal(t, "1", raw.Header.Get(retryCountHeaderName), + "DLQ copy must carry the retry count at dead-letter time") +} + +// TestDeadLetterPreservesRoutingKey: when a source sets no DeadLetterSubject, +// the DLQ copy must keep the original message's routing key — RabbitMQ +// dead-letters under the original key unless x-dead-letter-routing-key +// overrides it, and DLQ consumers may bind by routing key. +func TestDeadLetterPreservesRoutingKey(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dlrk", Subjects: []string{"region.us.created"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 1) + src := queueSource("events.dlrk.consumer", "events.dlrk", "region.#") + src.Options["DeadLetterAddress"] = "events.dlrk.dlq" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) + + bd := p.getBrokerDetailsByIdentifier("test-client") + dlqStream, serr := bd.js.Stream(ctx, streamNameFor("events.dlrk.dlq")) + require.NoError(t, serr) + raw, gerr := dlqStream.GetMsg(ctx, 1) + require.NoError(t, gerr) + assert.Equal(t, "events.dlrk.dlq.~.region.us.created", raw.Subject, + "the DLQ copy must carry the original routing key under the DLA root") +} + +// TestDeadLetterPreservesRoutingKeyRoutedFromParent is the same contract for a +// message that reached the source through an address-to-address binding. +// Sourcing keeps the subject the message was published under, so it stays +// rooted at the PARENT address while the source names the child — and RabbitMQ +// preserves the original routing key through both the exchange-to-exchange +// route and the DLX move, so the DLQ copy must still carry it. +// +// Regression: the routing key was recovered by stripping the consuming +// address's own prefix, which a parent-rooted subject never matches, so every +// message routed in from a parent dead-lettered under an empty key — landing +// on the bare DLA subject where a DLQ consumer bound by routing key would +// never see it. +func TestDeadLetterPreservesRoutingKeyRoutedFromParent(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + parent := &pb.Address{Name: "events.dlrk.parent", Type: pb.Address_TOPIC} + child := &pb.Address{ + Name: "events.dlrk.child", + Type: pb.Address_TOPIC, + Subjects: []string{"region.us.created"}, + ParentAddress: parent, + } + + out := make(chan *pb.Message, 1) + src := &pb.Source{Name: "events.dlrk.child.consumer", Type: pb.Source_QUEUE, Address: child, + Options: map[string]string{ + "Offset": "first", + "DeadLetterAddress": "events.dlrk.routed.dlq", + }} + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + time.Sleep(500 * time.Millisecond) + + // Published to the PARENT: the binding sources it into the child's stream + // under its original, parent-rooted subject. + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.dlrk.parent", Subjects: []string{"region.us.created"}}, + Body: []byte("poison")})) + + m := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) + + bd := p.getBrokerDetailsByIdentifier("test-client") + dlqStream, serr := bd.js.Stream(ctx, streamNameFor("events.dlrk.routed.dlq")) + require.NoError(t, serr) + raw, gerr := dlqStream.GetMsg(ctx, 1) + require.NoError(t, gerr) + assert.Equal(t, "events.dlrk.routed.dlq.~.region.us.created", raw.Subject, + "a message routed in from a parent must dead-letter under its original routing key") +} + +// TestDeadLetterFailureKeepsMessage: when the dead-letter publish cannot happen +// (here the DLQ address's subject space is already claimed by a foreign +// stream, so ensuring its stream fails), DeadLetter must NOT Term the +// original. It returns an error and leaves the message in flight, so the +// caller's fallback nack still resolves the uuid and the message is +// redelivered — the failure mode is retry, not silent loss. +// TestLateDLQReaderSeesEarlierDeadLetters: a dead-letter reader that attaches +// after the failures still receives them. The reader is transient by +// convention (the arke integration suite's dead-letter tests use +// Source_TEMPORARY), which makes it ephemeral and starts it at the stream tail +// — behind the very messages it exists to read. amqp091 passes this because +// arke declares the DLQ queue when the *source* declares its +// DeadLetterAddress, so it accumulates from that moment; setupDeadLetter +// mirrors it with a pre-declared durable the reader adopts. +// +// Deliberately cross-connection: the reader is a different client from the +// source that declared the DLA, so no per-connection state can carry the +// position (see the connector's per-connection memos — knownStreams, +// durableFilters). Only server-side topology can. +func TestLateDLQReaderSeesEarlierDeadLetters(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.late", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 1) + src := queueSource("events.late.consumer", "events.late", "job") + src.Options["DeadLetterAddress"] = "events.late.dlq" + src.Options["DeadLetterSubject"] = "failed" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) + + // Only now does the reader attach, on its own connection, naming the + // conventional DLQ source name and asking for no Offset — it must not need + // one, exactly as it does not on RabbitMQ. + p2, ctx2 := connectClient(t, s) + readerOut := make(chan *pb.Message, 1) + reader := &pb.Source{ + Name: dlqSourceName("events.late.consumer"), + Type: pb.Source_TEMPORARY, + Address: &pb.Address{Name: "events.late.dlq", Subjects: []string{"failed"}}, + } + readCtx, readCancel := context.WithCancel(ctx2) + defer readCancel() + go p2.Subscribe(readCtx, reader, readerOut) + + got := recv(t, readerOut) + assert.Equal(t, "poison", string(got.GetBody())) +} + +// TestDLQDurableAdoptableByASecondReader: attaching to a pre-declared +// dead-letter durable re-asserts its config, and a config without the marker +// metadata would REPLACE the stored metadata and erase it — leaving the +// durable unadoptable, so the next reader of the same queue would silently +// fall back to an ephemeral at the tail. Found in the arke integration suite, +// where TestDeadLettering and TestDeadLetteringReject share a source name: +// the first test's reader wiped the marker and the second saw nothing. +func TestDLQDurableAdoptableByASecondReader(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.twice", Subjects: []string{"job"}} + out := make(chan *pb.Message, 2) + src := queueSource("events.twice.consumer", "events.twice", "job") + src.Options["DeadLetterAddress"] = "events.twice.dlq" + src.Options["DeadLetterSubject"] = "failed" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + reader := func() *pb.Source { + return &pb.Source{ + Name: dlqSourceName("events.twice.consumer"), + Type: pb.Source_TEMPORARY, + Address: &pb.Address{Name: "events.twice.dlq", Subjects: []string{"failed"}}, + } + } + + // First reader: attaches, drains one dead letter, goes away. + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("first")})) + first := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, first.GetUuid())) + + firstOut := make(chan *pb.Message, 1) + firstCtx, firstCancel := context.WithCancel(ctx) + go p.Subscribe(firstCtx, reader(), firstOut) + got := recv(t, firstOut) + require.Equal(t, "first", string(got.GetBody())) + require.Nil(t, p.Ack(ctx, got.GetUuid())) + firstCancel() + + // Second reader, after another failure: must still adopt the same durable. + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("second")})) + next := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, next.GetUuid())) + + secondOut := make(chan *pb.Message, 1) + secondCtx, secondCancel := context.WithCancel(ctx) + defer secondCancel() + go p.Subscribe(secondCtx, reader(), secondOut) + assert.Equal(t, "second", string(recv(t, secondOut).GetBody())) +} + +// TestLateDLQReaderNoDeadLetterSubject covers the other filter branch: with no +// DeadLetterSubject the dead-lettered copy keeps its ORIGINAL routing key, +// which is per-message and unknowable when the queue is pre-declared, so +// setupDeadLetter captures the whole dead-letter address instead. A reader +// binding by that original key must still find it. This is the shape a source +// setting only DeadLetterAddress produces — the common one. +func TestLateDLQReaderNoDeadLetterSubject(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.nosub", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 1) + src := queueSource("events.nosub.consumer", "events.nosub", "job") + src.Options["DeadLetterAddress"] = "events.nosub.dlq" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) + + readerOut := make(chan *pb.Message, 1) + reader := &pb.Source{ + Name: dlqSourceName("events.nosub.consumer"), + Type: pb.Source_TEMPORARY, + // Bound by the ORIGINAL routing key, which is what the copy carries. + Address: &pb.Address{Name: "events.nosub.dlq", Subjects: []string{"job"}}, + } + readCtx, readCancel := context.WithCancel(ctx) + defer readCancel() + go p.Subscribe(readCtx, reader, readerOut) + + assert.Equal(t, "poison", string(recv(t, readerOut).GetBody())) +} + +// TestSetupDeadLetterLeavesExistingDurableAlone: re-declaring a source must not +// disturb a dead-letter queue that has been accumulating. setupDeadLetter +// returns early when the durable exists precisely so it never moves a position +// — re-asserting it at DeliverNew would silently discard every dead letter +// collected since the first declare, and clients re-subscribe routinely +// (reconnects, rollouts), so this is a common path, not an edge case. +func TestSetupDeadLetterLeavesExistingDurableAlone(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.redeclare", Subjects: []string{"job"}} + out := make(chan *pb.Message, 2) + src := queueSource("events.redeclare.consumer", "events.redeclare", "job") + src.Options["DeadLetterAddress"] = "events.redeclare.dlq" + src.Options["DeadLetterSubject"] = "failed" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + m := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) + + // Re-declare the same source, as a reconnecting client does. If this moved + // the pre-declared durable's start position, the dead letter above would + // become invisible. + againCtx, againCancel := context.WithCancel(ctx) + go p.Subscribe(againCtx, src, make(chan *pb.Message, 1)) + bd := p.getBrokerDetailsByIdentifier("test-client") + require.Eventually(t, func() bool { + st, serr := bd.js.Stream(ctx, streamNameFor("events.redeclare.dlq")) + if serr != nil { + return false + } + info, ierr := st.Info(ctx) + return ierr == nil && info.State.Msgs == 1 + }, 10*time.Second, 20*time.Millisecond, "dead letter never stored") + againCancel() + + readerOut := make(chan *pb.Message, 1) + readCtx, readCancel := context.WithCancel(ctx) + defer readCancel() + go p.Subscribe(readCtx, &pb.Source{ + Name: dlqSourceName("events.redeclare.consumer"), + Type: pb.Source_TEMPORARY, + Address: &pb.Address{Name: "events.redeclare.dlq", Subjects: []string{"failed"}}, + }, readerOut) + + assert.Equal(t, "poison", string(recv(t, readerOut).GetBody())) +} + +// TestSetupDeadLetterEmptyAddressDeclaresNothing: an empty DeadLetterAddress +// must declare no topology at all. Not hypothetical — the amqp091 connector's +// own Retry builds a source with DeadLetterAddress "" to mean "expire back to +// the default exchange", and an empty address maps to the degenerate stream +// name "arke_". +func TestSetupDeadLetterEmptyAddressDeclaresNothing(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + out := make(chan *pb.Message, 1) + src := queueSource("events.emptydla.consumer", "events.emptydla", "job") + src.Options["DeadLetterAddress"] = "" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.emptydla", Subjects: []string{"job"}}, + Body: []byte("hello"), + })) + require.Equal(t, "hello", string(recv(t, out).GetBody())) + + bd := p.getBrokerDetailsByIdentifier("test-client") + _, serr := bd.js.Stream(ctx, streamNameFor("")) + assert.ErrorIs(t, serr, jetstream.ErrStreamNotFound, + "an empty DeadLetterAddress must not create a stream") +} + +func TestDLQDeclareTTLFromEnv(t *testing.T) { + t.Setenv("NATSJS_DLQ_DECLARE_TTL", "10m") + assert.Equal(t, 10*time.Minute, dlqDeclareThreshold()) + + // Unset and unparseable both fall back to the dead-letter stream's own + // retention, so the queue outlives the messages in it instead of expiring + // at some guessed duration while they are still there to read. + t.Setenv("NATSJS_DLQ_DECLARE_TTL", "bad") + assert.Equal(t, defaultStreamMaxAge, dlqDeclareThreshold()) + t.Setenv("NATSJS_STREAM_MAX_AGE", "6h") + assert.Equal(t, 6*time.Hour, dlqDeclareThreshold(), + "the pre-declared queue must track the retention actually configured") + + // A stream retaining forever gives a queue that never expires, so there is + // still no window where messages outlive the queue holding them. + t.Setenv("NATSJS_STREAM_MAX_AGE", "0") + assert.Equal(t, time.Duration(0), dlqDeclareThreshold()) + + // "0" is meaningful here, unlike the other thresholds: it opts out of + // expiry entirely, which is exact amqp091 parity. + t.Setenv("NATSJS_STREAM_MAX_AGE", "72h") + t.Setenv("NATSJS_DLQ_DECLARE_TTL", "0") + assert.Equal(t, time.Duration(0), dlqDeclareThreshold()) +} + +// TestTransientSourceDoesNotAdoptUnmarkedDurable pins the gate on adoption: a +// transient source whose name merely collides with an existing durable must +// stay ephemeral. Adopting on name alone would hand it another subscriber's +// pending messages and stored ack position. +// +// The source is named with the dead-letter suffix deliberately, so it clears +// the cheap name check in adoptableDLQDurable and actually reaches the marker +// check this test is about. Without the suffix it would short-circuit earlier +// and pass for the wrong reason. +func TestTransientSourceDoesNotAdoptUnmarkedDurable(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + // A durable with the name a transient source would look for, but none of + // arke's dead-letter marker metadata. + bd := p.getBrokerDetailsByIdentifier("test-client") + _, serr := bd.js.CreateStream(ctx, jetstream.StreamConfig{ + Name: streamNameFor("events.collide"), + Subjects: streamSubjectsFor("events.collide"), + }) + require.NoError(t, serr) + stream, serr := bd.js.Stream(ctx, streamNameFor("events.collide")) + require.NoError(t, serr) + _, cerr := stream.CreateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: streamNameFor("events.collide.reader.dlq"), + AckPolicy: jetstream.AckExplicitPolicy, + }) + require.NoError(t, cerr) + + srcAddr := &pb.Address{Name: "events.collide", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: srcAddr, Body: []byte("hello")})) + + out := make(chan *pb.Message, 1) + src := &pb.Source{ + Name: "events.collide.reader.dlq", + Type: pb.Source_TEMPORARY, + Address: srcAddr, + // Read from the start, so receipt does not race consumer setup. The + // discriminator here is the untouched durable below, not the delivery. + Options: map[string]string{"Offset": "first"}, + } + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + got := recv(t, out) + require.Equal(t, "hello", string(got.GetBody())) + + // The pre-existing durable is untouched: the subscribe went to an ephemeral + // consumer of its own rather than adopting it. + existing, ierr := stream.Consumer(ctx, streamNameFor("events.collide.reader.dlq")) + require.NoError(t, ierr) + info, ierr := existing.Info(ctx) + require.NoError(t, ierr) + assert.Zero(t, info.Delivered.Consumer, "an unmarked durable must not have been consumed from") +} + +func TestDeadLetterFailureKeepsMessage(t *testing.T) { + // The fallback nack of a failed dead-letter is paced (see + // deadLetterRetryDelay); shorten it so the test does not wait it out. + t.Setenv("NATSJS_DEADLETTER_RETRY_DELAY", "100ms") + + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + // Claim the DLQ subject space so ensureStream for the DLA fails with + // "subjects overlap with an existing stream". + bd := p.getBrokerDetailsByIdentifier("test-client") + _, serr := bd.js.CreateStream(ctx, jetstream.StreamConfig{ + Name: "squatter", + Subjects: []string{"events.dlx.~", "events.dlx.~.>"}, + }) + require.NoError(t, serr) + + addr := &pb.Address{Name: "events.dlfail", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 2) + src := queueSource("events.dlfail.consumer", "events.dlfail", "job") + src.Options["DeadLetterAddress"] = "events.dlx" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + perr := p.DeadLetter(ctx, src, m.GetUuid()) + require.NotNil(t, perr, "DeadLetter must fail when the DLQ publish cannot happen") + assert.Contains(t, perr.GetMessage(), "dead-letter") + + // Not Term'd: the same uuid is still nackable, and the nack redelivers. + require.Nil(t, p.Nack(ctx, m.GetUuid()), "a failed dead-letter leaves the message nackable") + again := recv(t, out) + assert.Equal(t, "poison", string(again.GetBody())) + require.Nil(t, p.Ack(ctx, again.GetUuid())) +} + +func TestDeadLetterEmptyAddressKeepsMessage(t *testing.T) { + // The fallback nack of a failed dead-letter is paced (see + // deadLetterRetryDelay); shorten it so the test does not wait it out. + t.Setenv("NATSJS_DEADLETTER_RETRY_DELAY", "100ms") + + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dlempty", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 2) + src := queueSource("events.dlempty.consumer", "events.dlempty", "job") + src.Options["DeadLetterAddress"] = "" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + perr := p.DeadLetter(ctx, src, m.GetUuid()) + require.NotNil(t, perr, "an empty DLA must fail instead of terming the original") + assert.Contains(t, perr.GetMessage(), "DeadLetterAddress is empty") + + require.Nil(t, p.Nack(ctx, m.GetUuid()), "a failed dead-letter leaves the message nackable") + again := recv(t, out) + assert.Equal(t, "poison", string(again.GetBody())) + require.Nil(t, p.Ack(ctx, again.GetUuid())) +} + +// TestDeadLetterFailedTermKeepsMessageResolvable: if the DLQ copy is stored but +// Term fails, DeadLetter must leave the active-message entry in place. The +// server resolves its fallback nack (server.go, on a DeadLetter error) by +// uuid, so deleting the entry before Term succeeds would make that nack a +// no-op and strand the original ack-pending until AckWait, instead of +// redelivering it promptly. +func TestDeadLetterFailedTermKeepsMessageResolvable(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dlterm", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 1) + src := queueSource("events.dlterm.consumer", "events.dlterm", "job") + src.Options["DeadLetterAddress"] = "events.dlterm.dlq" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + + // Force Term to fail without breaking the DLQ publish: ack the underlying + // message directly (marking it already-acked), so DeadLetter's DLQ publish + // still lands but its subsequent Term returns ErrMsgAlreadyAckd. + bd := p.getBrokerDetailsByIdentifier("test-client") + mu, ok := bd.activeMessages.Get(m.GetUuid()) + require.True(t, ok) + require.NoError(t, mu.(inflightMsg).msg.Ack()) + + perr := p.DeadLetter(ctx, src, m.GetUuid()) + require.NotNil(t, perr, "DeadLetter must surface the Term failure") + + // The DLQ copy was stored... + dlqStats := p.SourceStats(ctx, &pb.Source{Name: "x", Address: &pb.Address{Name: "events.dlterm.dlq"}}) + assert.Equal(t, int64(1), dlqStats.GetMessageCount(), "the DLQ copy must still have been published") + + // ...but the original stays resolvable so the server's fallback nack lands. + _, stillThere := bd.activeMessages.Get(m.GetUuid()) + assert.True(t, stillThere, + "a failed Term must not remove the active-message entry the fallback nack needs") +} + +// TestPrefetchMapsToMaxAckPending: a positive PrefetchCount becomes the +// consumer's MaxAckPending. A prefetch of 0 is the AMQP "unlimited" +// convention — the amqp091 connector honors it by leaving the channel +// default, which is unlimited — so it maps to JetStream's unlimited (-1), +// not to a one-message-at-a-time clamp. +// A stream can be deleted out from under a live connection — an operator +// reset, a storage wipe, external cleanup — without the NATS connection +// noticing, and knownStreams pins the "already ensured" answer for the +// connection's lifetime. Publish, Subscribe, and the dead-letter path must +// detect the resulting no-stream errors, drop the stale entry, and re-create +// the stream instead of failing every call until the client reconnects. +func TestPublishRecreatesDeletedStream(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.wiped", Subjects: []string{"created"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("before")})) + + bd := p.getBrokerDetailsByIdentifier("test-client") + require.NoError(t, bd.js.DeleteStream(ctx, streamNameFor("events.wiped"))) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("after")}), + "publish must re-ensure the stream instead of failing on the stale memo") + + stream, serr := bd.js.Stream(ctx, streamNameFor("events.wiped")) + require.NoError(t, serr, "the stream must have been re-created") + assert.Equal(t, uint64(1), stream.CachedInfo().State.Msgs, "the retried publish must be stored") +} + +// TestPublishOneStreamAddressRequiresDeclaredStream pins the amqp091 contract +// for unary publishes to a STREAM address: the stream must already have been +// declared by a reader (amqp091 routes these through the RabbitMQ Streams +// client, which refuses a stream nobody declared), so a typo'd address name +// errors instead of silently minting a junk stream that swallows the +// messages. +func TestPublishOneStreamAddressRequiresDeclaredStream(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.undeclared", Type: pb.Address_STREAM, Subjects: []string{"e"}} + perr := p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m")}) + require.NotNil(t, perr, "publishing to an undeclared stream must fail") + assert.Equal(t, fmt.Sprintf(streamMissingError, "events.undeclared"), perr.GetMessage()) + + bd := p.getBrokerDetailsByIdentifier("test-client") + _, serr := bd.js.Stream(ctx, streamNameFor("events.undeclared")) + assert.ErrorIs(t, serr, jetstream.ErrStreamNotFound, "the refused publish must not create the stream") + + // Once a reader has declared the stream, the same publish succeeds. + _, eerr := p.ensureStream(ctx, bd, "events.undeclared") + require.NoError(t, eerr) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m")})) +} + +// TestPublishOneStreamAddressDeletedStreamNotRecreated is the STREAM-address +// counterpart of TestPublishRecreatesDeletedStream: the self-heal that +// re-ensures an auto-created stream must not resurrect one the operator +// deleted, because for a STREAM address the stream is the reader's declared +// topology — a RabbitMQ stream publisher errors after deletion too. +func TestPublishOneStreamAddressDeletedStreamNotRecreated(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dropped", Type: pb.Address_STREAM, Subjects: []string{"e"}} + bd := p.getBrokerDetailsByIdentifier("test-client") + _, eerr := p.ensureStream(ctx, bd, "events.dropped") + require.NoError(t, eerr) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("before")})) + + require.NoError(t, bd.js.DeleteStream(ctx, streamNameFor("events.dropped"))) + + perr := p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("after")}) + require.NotNil(t, perr, "a deleted stream must fail the publish, not be resurrected") + assert.Equal(t, fmt.Sprintf(streamMissingError, "events.dropped"), perr.GetMessage()) + _, serr := bd.js.Stream(ctx, streamNameFor("events.dropped")) + assert.ErrorIs(t, serr, jetstream.ErrStreamNotFound) +} + +// TestPublishStreamingPathStillAutoCreatesStreamAddress pins the deliberate +// split: only the unary path requires a declared stream. amqp091's streaming +// Publish sends STREAM addresses over an auto-declared exchange and reports +// no error either, so the streaming path keeps auto-creating — the message is +// stored where amqp091 would drop it into an unbound exchange. +func TestPublishStreamingPathStillAutoCreatesStreamAddress(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + in := make(chan *pb.Message, 1) + errChan := make(chan *pb.Error, 1) + pubCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Publish(pubCtx, in, errChan) + in <- &pb.Message{ + Address: &pb.Address{Name: "events.streamed", Type: pb.Address_STREAM, Subjects: []string{"e"}}, + Body: []byte("m")} + select { + case e := <-errChan: + require.Nil(t, e, "the streaming publish must auto-create the stream: %v", e.GetMessage()) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the publish reply") + } + + bd := p.getBrokerDetailsByIdentifier("test-client") + stream, serr := bd.js.Stream(ctx, streamNameFor("events.streamed")) + require.NoError(t, serr, "the stream must have been auto-created") + assert.Equal(t, uint64(1), stream.CachedInfo().State.Msgs) +} + +func TestSubscribeRecreatesDeletedStream(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.wiped2", Subjects: []string{"created"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("before")})) + bd := p.getBrokerDetailsByIdentifier("test-client") + require.NoError(t, bd.js.DeleteStream(ctx, streamNameFor("events.wiped2"))) + + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.wiped2.consumer", "events.wiped2", "created"), out) + + // The subscribe must have re-created the stream and its consumer; without + // the heal it returns fatal "stream not found" and nothing ever consumes. + require.Eventually(t, func() bool { + stream, serr := bd.js.Stream(ctx, streamNameFor("events.wiped2")) + return serr == nil && stream.CachedInfo().State.Consumers > 0 + }, 10*time.Second, 50*time.Millisecond, "subscribe must re-create the deleted stream") + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("after")})) + m := recv(t, out) + assert.Equal(t, "after", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) +} + +func TestDeadLetterRecoversDeletedDLQStream(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dlsrc", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("p1")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("p2")})) + + out := make(chan *pb.Message, 2) + src := queueSource("events.dlsrc.consumer", "events.dlsrc", "job") + src.Options["DeadLetterAddress"] = "events.dlq.wiped" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + first := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, first.GetUuid()), "first dead-letter creates the DLQ stream") + + bd := p.getBrokerDetailsByIdentifier("test-client") + require.NoError(t, bd.js.DeleteStream(ctx, streamNameFor("events.dlq.wiped"))) + + second := recv(t, out) + require.NotNil(t, p.DeadLetter(ctx, src, second.GetUuid()), + "the publish into the deleted DLQ stream must fail") + // The failed call dropped the stale memo, so this retry — the message is + // still in flight and resolvable, per the dead-letter failure contract — + // re-creates the DLQ stream and succeeds. + require.Nil(t, p.DeadLetter(ctx, src, second.GetUuid()), + "the dead-letter retry must re-create the DLQ stream") +} + +// TestSubscribeEndsWhenConsumerDeleted: deleting a subscription's server-side +// consumer out from under it (an operator cleanup, a misdirected tool) used +// to leave the subscription permanently deaf — the client library treats the +// resulting consume error as terminal and silently stops delivering, while +// Subscribe kept blocking on its context. Subscribe must instead end with a +// non-fatal error so the client's re-subscribe recreates the consumer. +func TestSubscribeEndsWhenConsumerDeleted(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.zap", Subjects: []string{"k"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m1")})) + + src := queueSource("events.zap.consumer", "events.zap", "k") + out := make(chan *pb.Message, 2) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan *pb.Error, 1) + go func() { done <- p.Subscribe(subCtx, src, out) }() + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + bd := p.getBrokerDetailsByIdentifier("test-client") + stream, serr := bd.js.Stream(ctx, streamNameFor("events.zap")) + require.NoError(t, serr) + require.NoError(t, stream.DeleteConsumer(ctx, durableName(src))) + + select { + case perr := <-done: + require.NotNil(t, perr, "Subscribe must end with an error, not block forever") + assert.False(t, perr.GetIsFatal(), "consumer loss must be retriable (non-fatal)") + case <-time.After(30 * time.Second): + t.Fatal("Subscribe kept blocking after its consumer was deleted") + } + + // The re-subscribe — the client's reaction to the ended stream — + // recreates the durable and delivery resumes. (Offset "first" replays the + // retained backlog into the fresh durable, so drain until the new + // message arrives.) + out2 := make(chan *pb.Message, 4) + go p.Subscribe(subCtx, src, out2) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m2")})) + for { + m2 := recv(t, out2) + require.Nil(t, p.Ack(ctx, m2.GetUuid())) + if string(m2.GetBody()) == "m2" { + break + } + } +} + +// TestSubscribeRecoversAfterServerStateWipe: a broker restart that loses its +// JetStream state (storage wipe, factory reset) does not sever a NATS client +// the way it severs an AMQP client — the connection reconnects transparently +// and the subscription keeps pulling a consumer that no longer exists. No +// terminal error ever arrives on the consume path: pulls find no responder +// and heartbeats go missing, indefinitely. The existence probe must notice +// the consumer is gone and end the subscription (non-fatal), so the client's +// re-subscribe rebuilds stream and consumer. +func TestSubscribeRecoversAfterServerStateWipe(t *testing.T) { + s := runJetStreamServer(t) + port := s.Addr().(*net.TCPAddr).Port + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.wipe3", Subjects: []string{"k"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m1")})) + + src := queueSource("events.wipe3.consumer", "events.wipe3", "k") + out := make(chan *pb.Message, 2) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan *pb.Error, 1) + go func() { done <- p.Subscribe(subCtx, src, out) }() + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + // Restart the broker at the same address with an empty store. The client + // reconnects on its own; stream and consumer are gone server-side. + s.Shutdown() + s.WaitForShutdown() + runJetStreamServerAt(t, port) + + select { + case perr := <-done: + require.NotNil(t, perr, "Subscribe must end with an error, not starve silently") + assert.False(t, perr.GetIsFatal(), "state loss must be retriable (non-fatal)") + case <-time.After(60 * time.Second): + t.Fatal("Subscribe kept blocking after the broker lost its JetStream state") + } + + // The re-subscribe rebuilds the whole topology on the wiped broker and + // delivery resumes. + out2 := make(chan *pb.Message, 2) + go p.Subscribe(subCtx, src, out2) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m2")})) + m2 := recv(t, out2) + assert.Equal(t, "m2", string(m2.GetBody())) + require.Nil(t, p.Ack(ctx, m2.GetUuid())) +} + +func TestPrefetchMapsToMaxAckPending(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + bd := p.getBrokerDetailsByIdentifier("test-client") + maxAckPending := func(name string, prefetch int32) int { + t.Helper() + src := queueSource(name, "events.prefetch", "e") + src.PrefetchCount = prefetch + src.DeclareOnly = true + require.Nil(t, p.Subscribe(ctx, src, nil)) + stream, serr := bd.js.Stream(ctx, streamNameFor("events.prefetch")) + require.NoError(t, serr) + cons, cerr := stream.Consumer(ctx, durableName(src)) + require.NoError(t, cerr) + return cons.CachedInfo().Config.MaxAckPending + } + + assert.Equal(t, 7, maxAckPending("events.prefetch.limited", 7)) + assert.Equal(t, -1, maxAckPending("events.prefetch.unlimited", 0)) +} + +// TestExpiresSetsInactiveThreshold: the Expires option (AMQP x-expires — +// delete the queue after this many ms without consumers) must become the +// consumer's InactiveThreshold, for durables and ephemerals alike; unset +// keeps the defaults (durables never expire, transients get +// defaultInactiveThreshold). Pre-fix the value was accepted, warned about, +// and silently ignored: durables got no threshold and ephemerals always got +// the 5-minute default, so a client shortening or lengthening its queue's +// disuse lifetime diverged from amqp091 without any signal. +func TestExpiresSetsInactiveThreshold(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + bd := p.getBrokerDetailsByIdentifier("test-client") + + durableThreshold := func(name string, opts map[string]string) time.Duration { + t.Helper() + src := queueSource(name, "events.expires", "e") + for k, v := range opts { + src.Options[k] = v + } + src.DeclareOnly = true + require.Nil(t, p.Subscribe(ctx, src, nil)) + stream, serr := bd.js.Stream(ctx, streamNameFor("events.expires")) + require.NoError(t, serr) + cons, cerr := stream.Consumer(ctx, durableName(src)) + require.NoError(t, cerr) + return cons.CachedInfo().Config.InactiveThreshold + } + + assert.Equal(t, 2*time.Minute, + durableThreshold("events.expires.set", map[string]string{"Expires": "120000"}), + "a durable's Expires must become its InactiveThreshold") + assert.Zero(t, durableThreshold("events.expires.unset", nil), + "a durable without Expires never expires") + + // Ephemeral consumers are deleted eagerly when their subscription ends, + // so inspect the config while the subscription is live. + ephemeralThreshold := func(name string, opts map[string]string) time.Duration { + t.Helper() + src := &pb.Source{ + Name: name, + Type: pb.Source_QUEUE, + AutoDelete: true, // transient -> ephemeral consumer + Address: &pb.Address{Name: "events.expires.tmp." + name, Subjects: []string{"e"}}, + Options: opts, + } + require.Empty(t, durableName(src), "test source must map to an ephemeral consumer") + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, make(chan *pb.Message, 1)) + var threshold time.Duration + require.Eventually(t, func() bool { + stream, serr := bd.js.Stream(ctx, streamNameFor(src.GetAddress().GetName())) + if serr != nil { + return false + } + infos := stream.ListConsumers(ctx) + for info := range infos.Info() { + threshold = info.Config.InactiveThreshold + return true + } + return false + }, 10*time.Second, 20*time.Millisecond, "ephemeral consumer never appeared") + return threshold + } + + assert.Equal(t, time.Minute, + ephemeralThreshold("eph.set", map[string]string{"Expires": "60000"}), + "a transient source's Expires must become its InactiveThreshold") + assert.Equal(t, defaultInactiveThreshold, ephemeralThreshold("eph.unset", nil), + "a transient source without Expires keeps the default threshold") +} + +func TestSubscribeInvalidExpiresRejected(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + for _, bad := range []string{"10m", "0", "-5"} { + src := queueSource("events.expires.bad", "events.expires.bad", "e") + src.Options["Expires"] = bad + src.DeclareOnly = true // the reject must come before any topology work + serr := p.Subscribe(ctx, src, nil) + require.NotNil(t, serr, "Expires=%q must be rejected", bad) + assert.True(t, serr.GetIsFatal()) + assert.Contains(t, serr.GetMessage(), "value for Expires option must be") + } +} + +func TestDeduplication(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + // Dedup is a STREAM-address feature on both brokers — RabbitMQ gets it from + // the Streams client, which is the only thing amqp091's publish paths reach + // for a STREAM address — and a unary publish to one needs a declared stream. + bd := p.getBrokerDetailsByIdentifier("test-client") + _, eerr := p.ensureStream(ctx, bd, "events.dedup") + require.NoError(t, eerr) + + addr := &pb.Address{Name: "events.dedup", Type: pb.Address_STREAM, Subjects: []string{"e"}} + dup := func() *pb.Message { + return &pb.Message{Address: addr, Body: []byte("x"), PublisherName: "pub", PublishId: 42} + } + require.Nil(t, p.PublishOne(ctx, dup())) + require.Nil(t, p.PublishOne(ctx, dup())) + + stats := p.SourceStats(ctx, &pb.Source{ + Name: "events.dedup.consumer", + Address: &pb.Address{Name: "events.dedup"}, + }) + assert.Nil(t, stats.GetError()) + assert.Equal(t, int64(1), stats.GetMessageCount(), "duplicate publish_id within the window is collapsed") +} + +func TestPublishIDRequiresPublisherName(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + // A STREAM address: the only kind whose unary publish reaches the + // PublisherName check at all, since every other type is refused outright + // for asking for dedup (see TestPublishDedupRefusedOnEveryNonStreamAddress). + bd := p.getBrokerDetailsByIdentifier("test-client") + _, eerr := p.ensureStream(ctx, bd, "events.dedup.required") + require.NoError(t, eerr) + + perr := p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.dedup.required", Type: pb.Address_STREAM, Subjects: []string{"e"}}, + Body: []byte("x"), + PublishId: 1, + }) + require.NotNil(t, perr) + assert.Contains(t, perr.GetMessage(), "PublisherName not set") +} + +func TestPublisherNameWithoutPublishIDDoesNotDedup(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dedup.nameonly", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("a"), PublisherName: "pub"})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("b"), PublisherName: "pub"})) + + stats := p.SourceStats(ctx, &pb.Source{ + Name: "events.dedup.nameonly.consumer", + Address: &pb.Address{Name: "events.dedup.nameonly"}, + }) + assert.Nil(t, stats.GetError()) + assert.Equal(t, int64(2), stats.GetMessageCount(), "publisher_name alone must not collapse messages as pub-0") +} + +func TestHeaderFilterDropsNonMatching(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.filter", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("keep"), Headers: map[string]string{"region": "us"}})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("drop"), Headers: map[string]string{"region": "eu"}})) + + out := make(chan *pb.Message, 2) + src := queueSource("events.filter.consumer", "events.filter", "e") + src.Filters = []*pb.Filter{{ + Type: pb.Filter_ALL, + Matches: []*pb.Match{{Name: "region", Value: "us"}}, + }} + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + assert.Equal(t, "keep", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + // The non-matching message is acked and dropped proxy-side, so nothing else + // should arrive. + select { + case extra := <-out: + t.Fatalf("unexpected second delivery: %q", string(extra.GetBody())) + case <-time.After(500 * time.Millisecond): + } +} + +// TestConflictingHeaderFiltersOnDurableRejected: header filters are evaluated +// proxy-side per consumer, so two competing subscribers of one durable that +// disagree would each drop the other's messages (JetStream hands a message to +// one of them, and if it does not match that one's filter it is acked and +// lost). RabbitMQ header bindings are queue-wide and cannot diverge like this, +// so reject the conflicting declaration instead of silently losing messages. +// A second subscriber whose header filter matches is still allowed (competing +// consumers with a shared filter are legitimate). +func TestConflictingHeaderFiltersOnDurableRejected(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.hfconflict", Subjects: []string{"e"}} + mk := func(region string) *pb.Source { + src := queueSource("events.hfconflict.consumer", "events.hfconflict", "e") + src.Filters = []*pb.Filter{{ + Type: pb.Filter_ALL, + Matches: []*pb.Match{{Name: "region", Value: region}}, + }} + return src + } + + // Subscriber A (region=us) claims the durable and stays live; receiving a + // matching message proves it is past the claim. + outA := make(chan *pb.Message, 1) + ctxA, cancelA := context.WithCancel(ctx) + defer cancelA() + go p.Subscribe(ctxA, mk("us"), outA) + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, Body: []byte("k"), Headers: map[string]string{"region": "us"}})) + require.Nil(t, p.Ack(ctx, recv(t, outA).GetUuid())) + + // B tries to share the same durable with a different header filter: rejected. + perr := p.Subscribe(ctx, mk("eu"), make(chan *pb.Message, 1)) + require.NotNil(t, perr, "a conflicting header filter on a shared durable must be rejected") + assert.True(t, perr.GetIsFatal(), "the rejection is a client misconfiguration, not retriable") + assert.Contains(t, perr.GetMessage(), "header filter") + + // A matching second subscriber is accepted (does not reject). + outC := make(chan *pb.Message, 1) + ctxC, cancelC := context.WithCancel(ctx) + defer cancelC() + errC := make(chan *pb.Error, 1) + go func() { errC <- p.Subscribe(ctxC, mk("us"), outC) }() + select { + case perr := <-errC: + t.Fatalf("a matching header filter must not be rejected: %s", perr.GetMessage()) + case <-time.After(500 * time.Millisecond): + } + + // Once every subscriber of the durable ends, a fresh filter is allowed again + // (the fingerprint is forgotten when the last reference drops). + cancelA() + cancelC() + require.Eventually(t, func() bool { + return p.Subscribe(context.Background(), func() *pb.Source { + s := mk("eu") + s.DeclareOnly = true + return s + }(), nil) == nil + }, 5*time.Second, 50*time.Millisecond, + "a new filter must be accepted after the durable's subscribers all leave") +} + +func TestDurableConsumerResumesBacklog(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.durable", Subjects: []string{"e"}} + src := queueSource("events.durable.consumer", "events.durable", "e") + + // Establish the durable consumer, then stop consuming (client "outage"). + declCtx, declCancel := context.WithCancel(ctx) + go p.Subscribe(declCtx, src, make(chan *pb.Message, 1)) + time.Sleep(200 * time.Millisecond) + declCancel() + time.Sleep(100 * time.Millisecond) + + // Publish a backlog while no consumer is attached. + const n = 3 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("b%d", i))})) + } + + // Reattach the same durable; the backlog is redelivered. + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + for i := 0; i < n; i++ { + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } +} + +func TestDeclareOnly(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := queueSource("events.declare.consumer", "events.declare", "e") + src.DeclareOnly = true + + // DeclareOnly establishes topology and returns without blocking. + done := make(chan *pb.Error, 1) + go func() { done <- p.Subscribe(ctx, src, make(chan *pb.Message)) }() + select { + case perr := <-done: + assert.Nil(t, perr) + case <-time.After(10 * time.Second): + t.Fatal("DeclareOnly Subscribe did not return") + } + + // The stream exists afterwards. + stats := p.SourceStats(ctx, &pb.Source{Name: "x", Address: &pb.Address{Name: "events.declare"}}) + assert.Nil(t, stats.GetError()) +} + +func TestAckUnknownUUID(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + perr := p.Ack(ctx, "does-not-exist") + require.NotNil(t, perr) + // Worded exactly as amqp091 words it: a client matching on the text must + // not have to know which connector answered. + assert.Equal(t, "No message with uuid does-not-exist", perr.GetMessage()) +} + +func TestStreamConfigFromEnv(t *testing.T) { + t.Setenv("NATSJS_STREAM_REPLICAS", "5") + assert.Equal(t, 5, streamReplicas()) + t.Setenv("NATSJS_STREAM_REPLICAS", "bad") + assert.Equal(t, 1, streamReplicas()) + + t.Setenv("NATSJS_STREAM_MAX_AGE", "24h") + assert.Equal(t, 24*time.Hour, streamMaxAge()) + t.Setenv("NATSJS_STREAM_MAX_AGE", "0") + assert.Equal(t, time.Duration(0), streamMaxAge()) + + t.Setenv("NATSJS_STREAM_MAX_BYTES", "1048576") + assert.Equal(t, int64(1048576), streamMaxBytes()) + t.Setenv("NATSJS_STREAM_MAX_BYTES", "0") + assert.Equal(t, int64(0), streamMaxBytes()) +} + +func TestJSAPITimeoutFromEnv(t *testing.T) { + t.Setenv("NATSJS_API_TIMEOUT", "45s") + assert.Equal(t, 45*time.Second, jsAPITimeout()) + t.Setenv("NATSJS_API_TIMEOUT", "bad") + assert.Equal(t, defaultJSAPITimeout, jsAPITimeout()) + t.Setenv("NATSJS_API_TIMEOUT", "-1s") + assert.Equal(t, defaultJSAPITimeout, jsAPITimeout()) +} + +func TestAckWaitFromEnv(t *testing.T) { + t.Setenv("NATSJS_ACK_WAIT", "5m") + assert.Equal(t, 5*time.Minute, ackWait()) + t.Setenv("NATSJS_ACK_WAIT", "bad") + assert.Equal(t, defaultAckWait, ackWait()) + t.Setenv("NATSJS_ACK_WAIT", "0") + assert.Equal(t, defaultAckWait, ackWait()) +} + +func TestSACPinnedTTLFromEnv(t *testing.T) { + t.Setenv("NATSJS_SAC_PINNED_TTL", "10s") + assert.Equal(t, 10*time.Second, sacPinnedTTL()) + t.Setenv("NATSJS_SAC_PINNED_TTL", "bad") + assert.Equal(t, defaultSACPinnedTTL, sacPinnedTTL()) + t.Setenv("NATSJS_SAC_PINNED_TTL", "0") + assert.Equal(t, defaultSACPinnedTTL, sacPinnedTTL()) +} + +func TestStreamRegistryCollapsesConcurrentCalls(t *testing.T) { + r := newStreamRegistry() + release := make(chan struct{}) + leaderStarted := make(chan struct{}) + const key = "broker:4222/arke_events" + + // One call gets in flight and blocks... + leaderErr := make(chan error, 1) + go func() { + leaderErr <- r.ensure(context.Background(), key, func() error { + close(leaderStarted) + <-release + return nil + }) + }() + <-leaderStarted + + // ...then every caller that arrives while it is in flight must piggyback + // on it instead of running its own create. + var extraCalls int32 + const n = 8 + errs := make(chan error, n) + for i := 0; i < n; i++ { + go func() { + errs <- r.ensure(context.Background(), key, func() error { + atomic.AddInt32(&extraCalls, 1) + return nil + }) + }() + } + time.Sleep(250 * time.Millisecond) // let the followers reach ensure + close(release) + + assert.NoError(t, <-leaderErr) + for i := 0; i < n; i++ { + assert.NoError(t, <-errs) + } + assert.Equal(t, int32(0), atomic.LoadInt32(&extraCalls), "followers share the in-flight result") +} + +func TestStreamRegistryDoesNotCacheFailures(t *testing.T) { + r := newStreamRegistry() + boom := fmt.Errorf("create failed") + assert.ErrorIs(t, r.ensure(context.Background(), "k", func() error { return boom }), boom) + + calls := 0 + assert.NoError(t, r.ensure(context.Background(), "k", func() error { calls++; return nil })) + assert.Equal(t, 1, calls, "a failed create is retried by the next caller") +} + +func TestStreamRegistryFollowerHonorsContext(t *testing.T) { + r := newStreamRegistry() + release := make(chan struct{}) + started := make(chan struct{}) + go func() { + _ = r.ensure(context.Background(), "k", func() error { + close(started) + <-release + return nil + }) + }() + <-started + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := r.ensure(ctx, "k", func() error { + t.Error("follower must not run create") + return nil + }) + assert.ErrorIs(t, err, context.Canceled) + close(release) +} + +// TestStreamEnsureKeyScopedToCredentials: the registry must never coalesce +// two connections that share a broker endpoint but authenticate as different +// users — on a multi-account server those are different accounts with +// disjoint JetStream state and permissions, so sharing one +// CreateOrUpdateStream call would hand one account the other's outcome +// (success in the wrong account, or a permission failure that is not its +// own). +func TestStreamEnsureKeyScopedToCredentials(t *testing.T) { + cfg := func(user string) *pb.ConnectionConfiguration { + c := &pb.ConnectionConfiguration{Host: "nats", Port: 4222} + if user != "" { + c.Credentials = &pb.Credentials{Username: user, Password: "s"} + } + return c + } + assert.Equal(t, streamEnsureKey(cfg("a"), "arke_x"), streamEnsureKey(cfg("a"), "arke_x"), + "same endpoint, account, and stream must coalesce") + assert.NotEqual(t, streamEnsureKey(cfg("a"), "arke_x"), streamEnsureKey(cfg("b"), "arke_x"), + "different accounts must not share an ensure result") + assert.NotEqual(t, streamEnsureKey(cfg(""), "arke_x"), streamEnsureKey(cfg("a"), "arke_x"), + "anonymous and authenticated connections must not share") + assert.NotEqual(t, streamEnsureKey(cfg("a"), "arke_x"), streamEnsureKey(cfg("a"), "arke_y"), + "different streams must not share") + other := cfg("a") + other.Port = 4223 + assert.NotEqual(t, streamEnsureKey(cfg("a"), "arke_x"), streamEnsureKey(other, "arke_x"), + "different endpoints must not share") +} + +func TestDeliverPolicyFor(t *testing.T) { + mk := func(off string) *pb.Source { return &pb.Source{Options: map[string]string{"Offset": off}} } + check := func(off string, wantPol jetstream.DeliverPolicy, wantSeq uint64) { + t.Helper() + pol, seq, err := deliverPolicyFor(mk(off)) + assert.Nil(t, err, "error for Offset=%q", off) + assert.Equal(t, wantPol, pol, "policy for Offset=%q", off) + assert.Equal(t, wantSeq, seq, "start seq for Offset=%q", off) + } + check("first", jetstream.DeliverAllPolicy, 0) + check("First", jetstream.DeliverAllPolicy, 0) // case-insensitive, like amqp091 + check("continue", jetstream.DeliverAllPolicy, 0) + check("last", jetstream.DeliverLastPolicy, 0) + check("next", jetstream.DeliverNewPolicy, 0) + check("", jetstream.DeliverNewPolicy, 0) + // A numeric offset counts from 0 like a RabbitMQ Stream's, so it is one + // less than the JetStream sequence it starts the consumer at. + check("100", jetstream.DeliverByStartSequencePolicy, 101) + check("0", jetstream.DeliverAllPolicy, 0) // offset 0 == from the beginning + + pol, seq, err := deliverPolicyFor(&pb.Source{}) + assert.Nil(t, err) + assert.Equal(t, jetstream.DeliverNewPolicy, pol) + assert.Equal(t, uint64(0), seq) + + // An unrecognized offset is rejected, like amqp091's toStreamOffset — + // starting at a silently different position would lose or replay data. + for _, bad := range []string{"garbage", "-1", "1.5"} { + _, _, err := deliverPolicyFor(mk(bad)) + assert.ErrorContains(t, err, "invalid offset", "Offset=%q", bad) + } +} + +func TestFirstSubject(t *testing.T) { + assert.Equal(t, "created", firstSubject(&pb.Address{Subjects: []string{"created", "ignored"}})) + assert.Equal(t, "", firstSubject(&pb.Address{})) +} + +func TestWaitForConnectNoClient(t *testing.T) { + p := NewNATSJetStreamProvider().(*natsjsProvider) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + assert.False(t, p.WaitForConnect(ctx)) +} + +func TestSourceStatsMissingStream(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + stats := p.SourceStats(ctx, &pb.Source{Name: "x", Address: &pb.Address{Name: "never.created"}}) + assert.NotNil(t, stats.GetError()) +} + +// TestClientIdentifierFailurePropagates drives the error branch every method +// shares: when the client identifier cannot be resolved, each returns an error +// rather than touching the (nil) connection. +func TestClientIdentifierFailurePropagates(t *testing.T) { + saved := GetClientIdentifier + defer func() { GetClientIdentifier = saved }() + GetClientIdentifier = func(context.Context) (string, error) { + return "", fmt.Errorf("no client identifier") + } + + p := NewNATSJetStreamProvider().(*natsjsProvider) + ctx := context.Background() + src := queueSource("c", "events.x", "e") + msg := &pb.Message{Address: &pb.Address{Name: "events.x"}} + + assert.NotNil(t, p.Connect(ctx, &pb.ConnectionConfiguration{Host: "127.0.0.1", Port: 1}, false)) + assert.NotNil(t, p.PublishOne(ctx, msg)) + assert.NotNil(t, p.Publish(ctx, make(chan *pb.Message), make(chan *pb.Error))) + assert.NotNil(t, p.Subscribe(ctx, src, make(chan *pb.Message))) + assert.NotNil(t, p.Ack(ctx, "x")) + assert.NotNil(t, p.Nack(ctx, "x")) + assert.NotNil(t, p.Retry(ctx, src, "x", 1)) + assert.NotNil(t, p.DeadLetter(ctx, src, "x")) + assert.False(t, p.WaitForConnect(ctx)) + assert.NotNil(t, p.SourceStats(ctx, src).GetError()) + p.Disconnect(ctx) // exercises the early-return path; must not panic +} + +// errOf adapts a *pb.Error into a Go error for assert.NoError-style checks. +func errOf(e *pb.Error) error { + if e == nil { + return nil + } + return fmt.Errorf("%s", e.GetMessage()) +} + +// TestPrefixAddressesCoexist is the end-to-end regression test for addresses +// in a dotted-prefix relationship (e.g. "events.jobs" and +// "events.jobs.filter"). Before the "~" delimiter both addresses mapped to +// overlapping stream subjects, so whichever stream was created first won and +// every ensure of the other failed with "subjects overlap with an existing +// stream" (err 10065). Both creation orders are exercised, and traffic on one +// address must not leak into the other even when a routing key spells out the +// other address's name. +func TestPrefixAddressesCoexist(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + // parent first, then child + parent := &pb.Address{Name: "events.jobs", Subjects: []string{"created"}} + child := &pb.Address{Name: "events.jobs.filter", Subjects: []string{"created"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: parent, Body: []byte("to-parent")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: child, Body: []byte("to-child")}), + "creating the child stream after the parent must not collide") + + // child first, then parent + child2 := &pb.Address{Name: "events.audit.trail", Subjects: []string{"e"}} + parent2 := &pb.Address{Name: "events.audit", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: child2, Body: []byte("x")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: parent2, Body: []byte("x")}), + "creating the parent stream after the child must not collide") + + // a routing key on the parent that spells the child's name stays on the + // parent's stream + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.jobs", Subjects: []string{"filter.created"}}, + Body: []byte("still-to-parent"), + })) + + out := make(chan *pb.Message, 4) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.jobs.filter.consumer", "events.jobs.filter", "created"), out) + + m := recv(t, out) + assert.Equal(t, "to-child", string(m.GetBody()), + "the child consumer sees only the child's message") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("unexpected cross-address delivery: %q", extra.GetBody()) + case <-time.After(500 * time.Millisecond): + } + + // the parent consumer sees both parent messages + pout := make(chan *pb.Message, 4) + go p.Subscribe(subCtx, queueSource("events.jobs.consumer", "events.jobs", "#"), pout) + got := map[string]bool{} + for i := 0; i < 2; i++ { + m := recv(t, pout) + got[string(m.GetBody())] = true + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + assert.True(t, got["to-parent"] && got["still-to-parent"], "got %v", got) +} + +// TestUnderscoreAddressesCoexist: address names "events.a" and "events_a" are +// distinct AMQP exchanges, but a naive dots-to-underscores stream name maps +// both onto one JetStream stream, whose config each address's re-ensure then +// flips to its own subjects — each breaking the other. The name mapping must +// keep them on separate streams. +func TestUnderscoreAddressesCoexist(t *testing.T) { //nolint:dupl // deliberately parallel to TestEscapedAddressesCoexist + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + dotted := &pb.Address{Name: "events.a", Subjects: []string{"k"}} + scored := &pb.Address{Name: "events_a", Subjects: []string{"k"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: dotted, Body: []byte("to-dotted")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: scored, Body: []byte("to-scored")}), + "publishing to events_a after events.a must not hijack its stream") + + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + for addr, want := range map[string]string{"events.a": "to-dotted", "events_a": "to-scored"} { + out := make(chan *pb.Message, 2) + go p.Subscribe(subCtx, queueSource(addr+".consumer", addr, "k"), out) + m := recv(t, out) + assert.Equal(t, want, string(m.GetBody()), "consumer on %q sees its own message", addr) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("unexpected cross-address delivery on %q: %q", addr, extra.GetBody()) + case <-time.After(500 * time.Millisecond): + } + } +} + +// TestEscapedAddressesCoexist: address names "evt.~.b" and "evt._.b" are +// distinct AMQP exchanges, but a lossy token sanitizer mapped both onto the +// root "evt._.b" — one shared subject space and one shared stream, so each +// address received the other's messages and each ensure reconfigured the +// other's stream. Escaped tokens must keep them fully apart. +func TestEscapedAddressesCoexist(t *testing.T) { //nolint:dupl // deliberately parallel to TestUnderscoreAddressesCoexist + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + tilde := &pb.Address{Name: "evt.~.b", Subjects: []string{"k"}} + scored := &pb.Address{Name: "evt._.b", Subjects: []string{"k"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: tilde, Body: []byte("to-tilde")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: scored, Body: []byte("to-scored")}), + "publishing to evt._.b after evt.~.b must not land on its stream") + + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + for addr, want := range map[string]string{"evt.~.b": "to-tilde", "evt._.b": "to-scored"} { + out := make(chan *pb.Message, 2) + go p.Subscribe(subCtx, queueSource(addr+".consumer", addr, "k"), out) + m := recv(t, out) + assert.Equal(t, want, string(m.GetBody()), "consumer on %q sees its own message", addr) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("unexpected cross-address delivery on %q: %q", addr, extra.GetBody()) + case <-time.After(500 * time.Millisecond): + } + } +} + +// TestEscapedRoutingKeysStayDistinct: routing keys "a b" and "a_b" are +// distinct on a direct exchange, but the lossy sanitizer merged both onto +// "a_b", so a queue bound to "a_b" also received every "a b" message. +func TestEscapedRoutingKeysStayDistinct(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + pub := func(rk, body string) { + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "evt.direct", Subjects: []string{rk}}, + Body: []byte(body), + })) + } + pub("a b", "spaced") + pub("a_b", "scored") + + src := queueSource("evt.direct.consumer", "evt.direct", "a_b") + src.Address.Type = pb.Address_QUEUE + out := make(chan *pb.Message, 2) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + assert.Equal(t, "scored", string(m.GetBody()), `binding "a_b" must match only routing key "a_b"`) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("binding %q wrongly matched routing key %q message", "a_b", string(extra.GetBody())) + case <-time.After(500 * time.Millisecond): + } +} + +// TestUnderscoreSourceNamesStayIndependent: two queues named "grp.a" and +// "grp_a" bound to the same address are independent queues — each receives +// every message — but a naive name mapping collapses them onto one durable, +// turning them into competing consumers that split the traffic. +func TestUnderscoreSourceNamesStayIndependent(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.split", Subjects: []string{"k"}} + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + outDot := make(chan *pb.Message, 2) + outScore := make(chan *pb.Message, 2) + go p.Subscribe(subCtx, queueSource("grp.a", "events.split", "k"), outDot) + go p.Subscribe(subCtx, queueSource("grp_a", "events.split", "k"), outScore) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("fan-out")})) + + for name, out := range map[string]<-chan *pb.Message{"grp.a": outDot, "grp_a": outScore} { + m := recv(t, out) + assert.Equal(t, "fan-out", string(m.GetBody()), "queue %q gets its own copy", name) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } +} + +// TestSameSourceNameSubscriptionsTrackedSeparately: consumer groups share one +// source name and differ only by ConsumerGroup, so a single connection can +// hold several live subscriptions for the same name. Each must keep its own +// bookkeeping entry: keyed by name alone, the second Add overwrites the +// first, Stats undercounts the connection's streams, and the first teardown +// deletes the survivor's entry (so Disconnect would no longer Stop it). +func TestSameSourceNameSubscriptionsTrackedSeparately(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := func(cg string) *pb.Source { + return &pb.Source{ + Name: "events.shared.consumer", + Type: pb.Source_STREAM, + Address: &pb.Address{Name: "events.shared", Subjects: []string{"e"}}, + Options: map[string]string{"Offset": "first", "ConsumerGroup": cg}, + } + } + + outA := make(chan *pb.Message, 2) + outB := make(chan *pb.Message, 2) + ctxA, cancelA := context.WithCancel(ctx) + ctxB, cancelB := context.WithCancel(ctx) + defer cancelB() + errA := make(chan *pb.Error, 1) + go func() { errA <- p.Subscribe(ctxA, src("grp.a"), outA) }() + go p.Subscribe(ctxB, src("grp.b"), outB) + + // Each group's durable receives the message, proving both subscriptions + // are live and consuming. + addr := &pb.Address{Name: "events.shared", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("x")})) + require.Nil(t, p.Ack(ctx, recv(t, outA).GetUuid())) + require.Nil(t, p.Ack(ctx, recv(t, outB).GetUuid())) + + require.Eventually(t, func() bool { + stats := p.Stats() + return len(stats.Clients) == 1 && stats.Clients[0].Streams == 2 + }, 5*time.Second, 20*time.Millisecond, + "two live subscriptions sharing a source name must both be tracked") + + // Ending one subscription removes only its own entry. + cancelA() + require.Nil(t, <-errA) + stats := p.Stats() + require.Len(t, stats.Clients, 1) + assert.Equal(t, 1, stats.Clients[0].Streams, + "the sibling's teardown must not drop the survivor's entry") +} + +// streamSource builds a stream-typed source (Source_STREAM) positioned at the +// given RabbitMQ Streams offset ("first"/"next"/...). It carries a +// ConsumerGroup, derived from the source name, because the group is a stream +// source's durable identity — each test consumer keeps its own durable. See +// TestStreamSourceWithoutGroupReadsIndependently for the group-less +// (ephemeral) form. +func streamSource(name, addr, offset string, subjects ...string) *pb.Source { + return &pb.Source{ + Name: name, + Type: pb.Source_STREAM, + Address: &pb.Address{Name: addr, Subjects: subjects}, + Options: map[string]string{"Offset": offset, "ConsumerGroup": name + ".grp"}, + } +} + +// TestStreamSourceWithoutGroupReadsIndependently: a STREAM source with no +// ConsumerGroup has no durable identity, so every subscriber is an +// independent ephemeral reader positioned by its own Offset and sees every +// message. That is deliberate RabbitMQ parity — stream consumers read a +// shared log and never compete (unlike queue consumers) — distinct from the +// transient-source fan-out documented as a known limitation. +func TestStreamSourceWithoutGroupReadsIndependently(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.reader", Subjects: []string{"e"}} + const n = 3 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + + src := func() *pb.Source { + return &pb.Source{ + Name: "events.reader.consumer", + Type: pb.Source_STREAM, + Address: &pb.Address{Name: "events.reader", Subjects: []string{"e"}}, + Options: map[string]string{"Offset": "first"}, + } + } + + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + outA := make(chan *pb.Message, n) + outB := make(chan *pb.Message, n) + go p.Subscribe(subCtx, src(), outA) + go p.Subscribe(subCtx, src(), outB) + for _, out := range []<-chan *pb.Message{outA, outB} { + for i := 0; i < n; i++ { + m := recv(t, out) + assert.Equal(t, fmt.Sprintf("m%d", i), string(m.GetBody()), + "each reader replays the full log in order") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + } +} + +// TestStreamOffsetReplay is the RabbitMQ-Streams parity test. A natsjs address +// is a retained JetStream log (LimitsPolicy, acked messages are NOT deleted), +// so it behaves like a RabbitMQ Stream rather than a classic/quorum work +// queue: independent consumers each read from their own offset, and a fully +// acked backlog is still replayable by a fresh consumer. +func TestStreamOffsetReplay(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.stream", Subjects: []string{"e"}} + const n = 4 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + + drain := func(t *testing.T, name string) int { + t.Helper() + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, streamSource(name, "events.stream", "first", "e"), out) + for i := 0; i < n; i++ { + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + return n + } + + // Consumer A reads and acks the whole backlog from offset "first". + t.Run("first_reads_full_backlog", func(t *testing.T) { + assert.Equal(t, n, drain(t, "events.stream.a")) + }) + + // The defining Streams property: a second, independent consumer with its + // own durable/offset still replays all n messages even though consumer A + // already acked them. A RabbitMQ classic/quorum queue would have nothing + // left to deliver here — a Stream does. + t.Run("independent_consumer_replays_after_acks", func(t *testing.T) { + assert.Equal(t, n, drain(t, "events.stream.b"), + "an independent stream consumer replays the retained log") + }) +} + +// TestStreamOffsetContinueResumes: offset "continue" resumes a group-less +// stream source where its last subscription stopped, instead of replaying the +// log from the start like "first". amqp091 answers "continue" from RabbitMQ +// Streams' server-side offset tracking (QueryOffset by consumer name, +1); +// here the source's durable holds the position, so a re-subscribe under the +// same name picks up after the messages it already acked. +func TestStreamOffsetContinueResumes(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.cont", Subjects: []string{"e"}} + publish := func(t *testing.T, n int, tag string) { + t.Helper() + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, + &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("%s%d", tag, i))})) + } + } + src := func() *pb.Source { + return &pb.Source{ + Name: "events.cont.consumer", + Type: pb.Source_STREAM, + Address: &pb.Address{Name: "events.cont", Subjects: []string{"e"}}, + Options: map[string]string{"Offset": "continue"}, + } + } + + // With nothing read yet there is no stored position, so "continue" starts + // at the beginning — RabbitMQ's QueryOffset finds no offset and falls back + // to offset 0 the same way. + publish(t, 2, "first") + drain := func(t *testing.T, want int) []string { + t.Helper() + out := make(chan *pb.Message, want+4) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src(), out) + got := make([]string, 0, want) + for i := 0; i < want; i++ { + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + got = append(got, string(m.GetBody())) + } + // Nothing beyond what was asked for: a replay would show up here. + select { + case m := <-out: + t.Fatalf("unexpected extra message %q; continue replayed the log", m.GetBody()) + case <-time.After(time.Second): + } + return got + } + require.Equal(t, []string{"first0", "first1"}, drain(t, 2)) + + // The second subscription must see only what was published after the + // first one stopped. + publish(t, 3, "second") + require.Equal(t, []string{"second0", "second1", "second2"}, drain(t, 3), + "continue replayed already-acked messages instead of resuming") +} + +// TestStreamOffsetNextSkipsBacklog: offset "next" (and the default) positions a +// new consumer at the tail — it sees only messages published after it was +// created, matching RabbitMQ Streams "next". +func TestStreamOffsetNextSkipsBacklog(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.tail", Subjects: []string{"e"}} + // Backlog published before the consumer exists. + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("old")})) + + out := make(chan *pb.Message, 2) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, streamSource("events.tail.consumer", "events.tail", "next", "e"), out) + // Let the durable form at the tail before publishing "new", so DeliverNew's + // start point is unambiguously after "old". + time.Sleep(500 * time.Millisecond) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("new")})) + + m := recv(t, out) + assert.Equal(t, "new", string(m.GetBody()), "next offset skips the pre-existing backlog") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("unexpected extra delivery (backlog leaked past 'next'): %q", extra.GetBody()) + case <-time.After(500 * time.Millisecond): + } +} + +// TestStreamOffsetLastAndNumeric is the behavioral counterpart to +// TestDeliverPolicyFor: it drives the two offset positions that amqp091's +// toStreamOffset supports and that natsjs now honors — "last" (final message +// only) and an absolute numeric stream sequence. +func TestStreamOffsetLastAndNumeric(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.offset", Subjects: []string{"e"}} + const n = 4 // seq: m0=1, m1=2, m2=3, m3=4 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + + t.Run("last_delivers_only_final", func(t *testing.T) { + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, streamSource("events.offset.last", "events.offset", "last", "e"), out) + + m := recv(t, out) + assert.Equal(t, "m3", string(m.GetBody()), "last offset starts at the final message") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("last should deliver only the final message, got extra %q", extra.GetBody()) + case <-time.After(500 * time.Millisecond): + } + }) + + t.Run("numeric_offset_starts_at_offset", func(t *testing.T) { + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + // Offsets count from 0 like a RabbitMQ Stream's, so offset 2 is the + // third message, m2 — not JetStream's sequence 2, which is m1. + go p.Subscribe(subCtx, streamSource("events.offset.num", "events.offset", "2", "e"), out) + + got := []string{} + for i := 0; i < 2; i++ { + m := recv(t, out) + got = append(got, string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + assert.Equal(t, []string{"m2", "m3"}, got, "numeric offset starts at the named message") + }) + + // The offset vocabulary has to round-trip: an offset read from SourceStats + // and handed back as a source's Offset must name the same message. A + // connector reporting JetStream's 1-based sequence but accepting it back + // as a 0-based offset would quietly skip one message per hop. + t.Run("offset_from_stats_round_trips", func(t *testing.T) { + stats := p.SourceStats(ctx, streamSource("events.offset.rt", "events.offset", "first", "e")) + require.Nil(t, stats.GetError()) + require.EqualValues(t, n-1, stats.GetLastOffset(), "last offset names the final message") + + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + last := strconv.FormatInt(stats.GetLastOffset(), 10) + go p.Subscribe(subCtx, streamSource("events.offset.rt2", "events.offset", last, "e"), out) + + m := recv(t, out) + assert.Equal(t, "m3", string(m.GetBody()), "LastOffset fed back must start at the last message") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + }) +} + +// TestSubscribeInvalidOffsetRejected: an offset outside the shared vocabulary +// fails the subscribe (mirroring amqp091's toStreamOffset) instead of silently +// starting the consumer at "next". +func TestSubscribeInvalidOffsetRejected(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + out := make(chan *pb.Message, 1) + perr := p.Subscribe(ctx, streamSource("events.bad.consumer", "events.bad", "latest", "e"), out) + require.NotNil(t, perr) + assert.Contains(t, perr.GetMessage(), "invalid offset") + assert.True(t, perr.GetIsFatal()) +} + +// TestDurableOffsetPinnedAtCreation: a durable's start position is fixed when +// the consumer is first created — JetStream rejects DeliverPolicy/OptStartSeq +// updates (err 10012) — so a re-subscribe that asks for a different Offset must +// not fail; it attaches to the existing durable and resumes from its stored ack +// position, per the documented contract. +func TestDurableOffsetPinnedAtCreation(t *testing.T) { + // The first subscription reads ahead, so it can pull m1 into flight before + // this test cancels it. releaseInFlight naks such messages precisely so + // they redeliver promptly, but that nak is best-effort: if it loses its + // race with the unsubscribe, the server holds m1 until AckWait instead. + // The production default (30s) outlasts recv's own 10s timeout, so when + // both of those happen at once the test failed spuriously. Pin an AckWait + // shorter than recv's timeout — the subject here is the pinned start + // position, not redelivery latency. + t.Setenv("NATSJS_ACK_WAIT", "2s") + + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.pinned", Subjects: []string{"e"}} + const n = 4 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + + src := func(offset string) *pb.Source { + s := streamSource("events.pinned.consumer", "events.pinned", offset, "e") + // One message in flight at a time, so m1..m3 stay undelivered while + // the first subscription is up and are immediately deliverable to the + // re-attach below (unlimited prefetch would leave them ack-pending + // until AckWait). + s.PrefetchCount = 1 + return s + } + + // Create the durable at "first" and ack only m0, leaving m1..m3 pending. + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + errCh := make(chan *pb.Error, 1) + go func() { errCh <- p.Subscribe(subCtx, src("first"), out) }() + m := recv(t, out) + assert.Equal(t, "m0", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + cancel() + require.Nil(t, <-errCh) + + // Re-subscribe the same durable asking for sequence 4 (m3). The request + // conflicts with the pinned start position; the consumer resumes from its + // stored ack floor instead, so the next delivery is m1, not m3. + out2 := make(chan *pb.Message, n) + subCtx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + errCh2 := make(chan *pb.Error, 1) + go func() { errCh2 <- p.Subscribe(subCtx2, src("4"), out2) }() + m2 := recv(t, out2) + assert.Equal(t, "m1", string(m2.GetBody()), + "existing durable resumes from its stored position; the conflicting offset is ignored") + require.Nil(t, p.Ack(ctx, m2.GetUuid())) + cancel2() + require.Nil(t, <-errCh2, "conflicting offset on an existing durable must not fail the subscribe") +} + +// TestIsStartPositionConflict pins down which consumer-create errors the +// durable fallback absorbs: only the server's refusals to move a start +// position. Everything else — other immutable-field refusals sharing err code +// 10012, other API errors, non-API errors — stays fatal. +func TestIsStartPositionConflict(t *testing.T) { + mk := func(code jetstream.ErrorCode, desc string) error { + return &jetstream.APIError{ErrorCode: code, Description: desc} + } + assert.True(t, isStartPositionConflict(mk(jetstream.JSErrCodeConsumerCreate, "deliver policy can not be updated"))) + assert.True(t, isStartPositionConflict(mk(jetstream.JSErrCodeConsumerCreate, "start sequence can not be updated"))) + assert.True(t, isStartPositionConflict(mk(jetstream.JSErrCodeConsumerCreate, "start time can not be updated"))) + + // Same error code, different refusal: not a start-position conflict. + assert.False(t, isStartPositionConflict(mk(jetstream.JSErrCodeConsumerCreate, "replay policy can not be updated"))) + assert.False(t, isStartPositionConflict(mk(jetstream.JSErrCodeConsumerCreate, "invalid subject"))) + // Different code, or not an API error at all. + assert.False(t, isStartPositionConflict(mk(jetstream.JSErrCodeStreamNotFound, "stream not found"))) + assert.False(t, isStartPositionConflict(fmt.Errorf("deliver policy can not be updated"))) +} + +// TestDurableConflictBeyondStartPositionStaysFatal: the attach-unchanged +// fallback exists for start-position conflicts only. If the existing durable +// differs in any other way — here an incompatible ReplayPolicy — resuming it +// unchanged would silently consume with a configuration the source never +// asked for, so the subscribe must fail instead. +func TestDurableConflictBeyondStartPositionStaysFatal(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.replay", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("x")})) + + src := queueSource("events.replay.consumer", "events.replay", "e") + + // A consumer with the connector's durable name already exists, matching on + // start position but differing in another immutable field. + bd := p.getBrokerDetailsByIdentifier("test-client") + stream, serr := bd.js.Stream(ctx, streamNameFor("events.replay")) + require.NoError(t, serr) + _, cerr := stream.CreateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: durableName(src), + FilterSubjects: filterSubjectsFor(src), + AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: jetstream.DeliverAllPolicy, // matches Offset "first" + ReplayPolicy: jetstream.ReplayOriginalPolicy, + }) + require.NoError(t, cerr) + + // Bound the subscribe so a regression (wrongly attaching and blocking) + // fails the test instead of hanging it. + subCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + perr := p.Subscribe(subCtx, src, make(chan *pb.Message, 1)) + require.NotNil(t, perr, "a non-start-position config conflict must fail the subscribe") + assert.True(t, perr.GetIsFatal()) + assert.Contains(t, perr.GetMessage(), "create consumer") +} + +// TestPublishContractRefusals pins the two publish combinations amqp091 +// refuses, so a client cannot write code against natsjs that silently means +// something else on RabbitMQ. Both error strings are amqp091's, verbatim. +func TestPublishContractRefusals(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + // Deduplication on a queue address: RabbitMQ gets dedup from streams, so + // amqp091 cannot offer it on a queue at all. + perr := p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.q", Type: pb.Address_QUEUE, Subjects: []string{"k"}}, + PublisherName: "pub", PublishId: 1, Body: []byte("x")}) + require.NotNil(t, perr) + assert.Equal(t, queueDedupError, perr.GetMessage()) + + // ...and equally on a topic address. amqp091's PublishOne sends every + // non-STREAM address through publishOneQueue, which is where that refusal + // lives, so TOPIC and FILTER are refused on exactly the same grounds. + require.NotNil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.t", Type: pb.Address_TOPIC, Subjects: []string{"k"}}, + PublisherName: "pub", PublishId: 1, Body: []byte("x")})) + + // A message wrong twice over — dedup contract violated AND aimed at an + // undeclared stream — reports the contract error: amqp091 refuses both + // of these before it reaches for the stream, so the refusals precede the + // stream lookup here too. + perr = p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.tcr.undeclared", Type: pb.Address_STREAM, Subjects: []string{"k"}}, + PublishId: 7, Body: []byte("x")}) + require.NotNil(t, perr) + assert.Contains(t, perr.GetMessage(), "PublisherName not set on message") + + // Confirm on the fire-and-forget streaming publish. + in := make(chan *pb.Message, 1) + errChan := make(chan *pb.Error, 2) + pubCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Publish(pubCtx, in, errChan) + in <- &pb.Message{Address: &pb.Address{Name: "events.t", Subjects: []string{"k"}}, + Body: []byte("x"), Confirm: true} + select { + case e := <-errChan: + require.NotNil(t, e) + assert.Equal(t, unsupportedConfirmError, e.GetMessage()) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the publish error") + } + // Exactly one reply per message: the server does an unconditional receive + // after each send, so a second value here would answer the *next* message. + select { + case e := <-errChan: + t.Fatalf("second reply for one message: %v", e) + case <-time.After(time.Second): + } +} + +// TestParentAddressBindingRoutes covers address-to-address binding: a child +// address bound to a parent receives what is published to the parent under the +// bound keys, still receives what is published to it directly, and does not +// receive what the parent routes elsewhere. +func TestParentAddressBindingRoutes(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + parent := &pb.Address{Name: "events.parent", Type: pb.Address_TOPIC} + child := &pb.Address{ + Name: "events.child", + Type: pb.Address_FILTER, + Subjects: []string{"bound.one", "bound.two"}, + ParentAddress: parent, + } + + out := make(chan *pb.Message, 8) + src := &pb.Source{Name: "events.child.consumer", Type: pb.Source_QUEUE, Address: child, + Options: map[string]string{"Offset": "first"}} + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + time.Sleep(500 * time.Millisecond) + + pub := func(t *testing.T, addrName, key, body string) { + t.Helper() + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: addrName, Subjects: []string{key}}, + Body: []byte(body)})) + } + // Routed in from the parent under a bound key... + pub(t, "events.parent", "bound.one", "routed") + // ...not routed: the parent has no binding to the child for this key. + pub(t, "events.parent", "unbound.key", "not-routed") + // ...and published straight to the child. + pub(t, "events.child", "bound.two", "direct") + + got := map[string]bool{} + deadline := time.After(6 * time.Second) + for len(got) < 2 { + select { + case m := <-out: + got[string(m.GetBody())] = true + require.Nil(t, p.Ack(ctx, m.GetUuid())) + case <-deadline: + t.Fatalf("timed out; received %v", got) + } + } + assert.True(t, got["routed"], "a message published to the parent under a bound key must reach the child") + assert.True(t, got["direct"], "a direct publish to the child must still arrive") + + select { + case m := <-out: + t.Fatalf("child received %q, which the parent does not route to it", m.GetBody()) + case <-time.After(2 * time.Second): + } +} + +// TestParentAddressBindingsAccumulate: a second address-to-address binding on +// the same address adds to the first rather than replacing it, the way +// declaring a binding on RabbitMQ never removes another. +func TestParentAddressBindingsAccumulate(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + parent := &pb.Address{Name: "events.acc.parent", Type: pb.Address_TOPIC} + childWith := func(subjects ...string) *pb.Address { + return &pb.Address{Name: "events.acc.child", Type: pb.Address_TOPIC, + Subjects: subjects, ParentAddress: parent} + } + sub := func(t *testing.T, name string, addr *pb.Address) chan *pb.Message { + t.Helper() + out := make(chan *pb.Message, 8) + subCtx, cancel := context.WithCancel(ctx) + t.Cleanup(cancel) + go p.Subscribe(subCtx, &pb.Source{Name: name, Type: pb.Source_QUEUE, Address: addr, + Options: map[string]string{"Offset": "first"}}, out) + time.Sleep(500 * time.Millisecond) + return out + } + + first := sub(t, "events.acc.first", childWith("key.one")) + // The second subscriber declares a different binding on the same address. + second := sub(t, "events.acc.second", childWith("key.two")) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.acc.parent", Subjects: []string{"key.one"}}, + Body: []byte("one")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.acc.parent", Subjects: []string{"key.two"}}, + Body: []byte("two")})) + + // The first subscriber's binding must have survived the second's declare. + m := recv(t, first) + assert.Equal(t, "one", string(m.GetBody()), "the earlier binding was dropped by a later one") + m = recv(t, second) + assert.Equal(t, "two", string(m.GetBody())) +} + +// TestEmptyRoutingKeyDelivers covers fanout-style publishes that carry no +// routing key: they map to the bare ".~" subject, which the stream +// captures and a consumer selects by binding the empty routing key. The +// binding has to be declared — a source with no bindings receives nothing, +// whatever the routing key (see TestNoBindingsReceiveNothing). +func TestEmptyRoutingKeyDelivers(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.fanout"} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("no-key")})) + + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.fanout.consumer", "events.fanout", ""), out) + + m := recv(t, out) + assert.Equal(t, "no-key", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) +} + +// TestNoBindingsReceiveNothing pins the AMQP rule that a source with no +// routing-key bindings receives nothing. amqp091 declares no binding at all +// for such a source (declareBinding iterates an empty subject list), so +// selecting the whole address instead — which a pattern-less consumer used to +// do — hands the source every message published to it. +func TestNoBindingsReceiveNothing(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + out := make(chan *pb.Message, 4) + src := queueSource("events.unbound.consumer", "events.unbound") + src.Address.Subjects = nil + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + // Let the consumer establish before publishing, so a delivery would race + // in rather than be missed. + time.Sleep(500 * time.Millisecond) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.unbound"}, Body: []byte("no-key")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.unbound", Subjects: []string{"some.key"}}, + Body: []byte("keyed")})) + + select { + case m := <-out: + t.Fatalf("source with no bindings received %q", m.GetBody()) + case <-time.After(2 * time.Second): + } +} + +func TestDirectAddressBindingIsExact(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := &pb.Source{ + Name: "events.direct.consumer", + Address: &pb.Address{ + Name: "events.direct", + Type: pb.Address_QUEUE, + Subjects: []string{"#"}, + }, + Options: map[string]string{"Offset": "first"}, + } + out := make(chan *pb.Message, 2) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.direct", Type: pb.Address_QUEUE, Subjects: []string{"actual"}}, + Body: []byte("wildcard-leak"), + })) + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.direct", Type: pb.Address_QUEUE, Subjects: []string{"#"}}, + Body: []byte("literal-match"), + })) + + m := recv(t, out) + assert.Equal(t, "literal-match", string(m.GetBody()), "direct bindings must not treat # as a topic wildcard") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("unexpected direct-address wildcard delivery: %q", string(extra.GetBody())) + case <-time.After(500 * time.Millisecond): + } +} + +// TestSourceStatsReportsConsumerBacklog: for a durable source the message +// count is the consumer's backlog, not the stream depth — the stream retains +// acked messages under its retention limits, so its count never shrinks as +// consumers catch up. +func TestSourceStatsReportsConsumerBacklog(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.backlog", Subjects: []string{"e"}} + for i := 0; i < 3; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("x")})) + } + + // Declare the durable consumer without consuming: its backlog is 3. + src := queueSource("events.backlog.consumer", "events.backlog", "e") + src.DeclareOnly = true + require.Nil(t, p.Subscribe(ctx, src, nil)) + + stats := p.SourceStats(ctx, src) + require.Nil(t, stats.GetError()) + assert.Equal(t, int64(3), stats.GetMessageCount(), "durable backlog") + + // Drain and ack everything; backlog returns to 0 even though the stream + // still retains the acked messages. + src.DeclareOnly = false + out := make(chan *pb.Message, 3) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + for i := 0; i < 3; i++ { + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + require.Eventually(t, func() bool { + st := p.SourceStats(ctx, src) + return st.GetError() == nil && st.GetMessageCount() == 0 + }, 10*time.Second, 200*time.Millisecond, "backlog drains to 0 after acks") +} + +// TestEphemeralConsumerDeletedOnTeardown verifies a transient source's +// ephemeral consumer is removed from the server as soon as its subscription +// ends, instead of counting against the stream's consumer limit until the +// inactivity threshold expires it. +// Streams are shared per address root, so the stream-wide consumer count +// spans every source on the address. A durable source's ConsumerCount must +// instead reflect the clients attached to ITS consumer — RabbitMQ reports the +// queue's own consumer count — or two queues on one exchange each report the +// other's consumers. +func TestSourceStatsConsumerCountIsPerSource(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + srcA := queueSource("events.cc.a", "events.cc", "created") + srcB := queueSource("events.cc.b", "events.cc", "created") + outA := make(chan *pb.Message, 1) + outB := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, srcA, outA) + go p.Subscribe(subCtx, srcB, outB) + + // Wait for both subscriptions' pulls to register, then check that each + // source counts only its own attached client. Pre-fix both report the + // stream-wide count of 2. + require.Eventually(t, func() bool { + return p.SourceStats(ctx, srcA).GetConsumerCount() == 1 && + p.SourceStats(ctx, srcB).GetConsumerCount() == 1 + }, 10*time.Second, 100*time.Millisecond, + "each durable source must report exactly its own attached client") +} + +func TestEphemeralConsumerDeletedOnTeardown(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := &pb.Source{ + Name: "events.tmp.listener", + Type: pb.Source_QUEUE, + AutoDelete: true, // transient -> ephemeral consumer + Address: &pb.Address{Name: "events.tmp", Subjects: []string{"a"}}, + } + require.Empty(t, durableName(src), "test source must map to an ephemeral consumer") + + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + _ = p.Subscribe(subCtx, src, out) + close(done) + }() + + bd := p.getBrokerDetailsByIdentifier("test-client") + var stream jetstream.Stream + require.Eventually(t, func() bool { + st, err := bd.js.Stream(ctx, streamNameFor("events.tmp")) + if err != nil { + return false + } + stream = st + info, ierr := st.Info(ctx) + return ierr == nil && info.State.Consumers == 1 + }, 10*time.Second, 20*time.Millisecond, "ephemeral consumer never appeared") + + cancel() + <-done + + assert.Eventually(t, func() bool { + info, err := stream.Info(ctx) + return err == nil && info.State.Consumers == 0 + }, 10*time.Second, 20*time.Millisecond, "ephemeral consumer was not deleted on teardown") +} + +// TestTeardownReleasesInFlightMessages: when a subscription ends, its +// delivered-but-unresolved messages are removed from the active-message map +// (their acks can only arrive on the consume stream that just closed, so they +// could never be resolved again — a leak for the life of the connection) and +// nak'd, so a durable redelivers them promptly instead of after the full +// AckWait, matching RabbitMQ's immediate requeue of a closed channel's +// unacked deliveries. +func TestTeardownReleasesInFlightMessages(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.inflight", Subjects: []string{"e"}} + const n = 3 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + + // First subscription receives the backlog but never resolves it. + src := queueSource("events.inflight.consumer", "events.inflight", "e") + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + errCh := make(chan *pb.Error, 1) + go func() { errCh <- p.Subscribe(subCtx, src, out) }() + for i := 0; i < n; i++ { + recv(t, out) // deliberately left unacked + } + cancel() + require.Nil(t, <-errCh) + + // The abandoned deliveries are no longer tracked... + stats := p.Stats() + require.Len(t, stats.Clients, 1) + assert.Zero(t, stats.Clients[0].ActiveMessages, + "teardown must release the subscription's in-flight messages") + + // ...and a re-subscribe gets them back well before AckWait (30s) because + // teardown nak'd them (recv's 10s timeout is the proof of promptness). + out2 := make(chan *pb.Message, n) + subCtx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + go p.Subscribe(subCtx2, src, out2) + for i := 0; i < n; i++ { + m := recv(t, out2) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } +} + +// TestTeardownReleasesOnlyItsOwnInFlight: two subscriptions with the same +// (durable) source name attach to one server-side consumer and compete, like +// two service instances on a shared queue. When the idle sibling ends, teardown +// must release only its own deliveries — not nak and drop the message the other +// subscription is still holding, which would fail that subscription's ack and +// redeliver the message as a duplicate. +func TestTeardownReleasesOnlyItsOwnInFlight(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.siblings", Subjects: []string{"e"}} + src := queueSource("events.siblings.consumer", "events.siblings", "e") + + // Subscription A takes one message and holds it unacked. + outA := make(chan *pb.Message, 1) + ctxA, cancelA := context.WithCancel(ctx) + defer cancelA() + errA := make(chan *pb.Error, 1) + go func() { errA <- p.Subscribe(ctxA, src, outA) }() + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("held")})) + held := recv(t, outA) // delivered to A, deliberately left unacked + + // Subscription B joins the same durable as a competing sibling, then ends + // without ever receiving anything of its own. + outB := make(chan *pb.Message, 1) + ctxB, cancelB := context.WithCancel(ctx) + errB := make(chan *pb.Error, 1) + go func() { errB <- p.Subscribe(ctxB, src, outB) }() + time.Sleep(400 * time.Millisecond) // let B's pull reach the server + cancelB() + require.Nil(t, <-errB) + + // A's held message must still resolve — the sibling teardown owned none of it. + require.Nil(t, p.Ack(ctx, held.GetUuid()), + "a sibling subscription's teardown must not release this subscription's in-flight message") + + // And it must not have been redelivered (which a wrongful nak would cause). + select { + case dup := <-outA: + t.Fatalf("message redelivered after a sibling teardown: %q", dup.GetBody()) + case <-time.After(300 * time.Millisecond): + } +} + +// TestRedundantBindingsCoexist: an AMQP binding set may be redundant — a +// wildcard binding alongside a specific key it already covers — and RabbitMQ +// routes each message to the queue once no matter how many bindings match. +// JetStream rejects a consumer whose filter subjects overlap, so the +// connector must collapse the redundant filters instead of failing the +// subscribe (and each message still arrives exactly once). +func TestRedundantBindingsCoexist(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + pub := func(key, body string) { + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.redundant", Subjects: []string{key}}, + Body: []byte(body), + })) + } + pub("orders.created", "covered-by-both") + pub("other.thing", "covered-by-wildcard") + + out := make(chan *pb.Message, 4) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + errCh := make(chan *pb.Error, 1) + go func() { + errCh <- p.Subscribe(subCtx, + queueSource("events.redundant.consumer", "events.redundant", "#", "orders.created"), out) + }() + + got := map[string]int{} + for i := 0; i < 2; i++ { + m := recv(t, out) + got[string(m.GetBody())]++ + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + assert.Equal(t, map[string]int{"covered-by-both": 1, "covered-by-wildcard": 1}, got) + + select { + case extra := <-out: + t.Fatalf("message delivered twice: %q", extra.GetBody()) + case perr := <-errCh: + t.Fatalf("subscribe failed on redundant bindings: %s", perr.GetMessage()) + case <-time.After(500 * time.Millisecond): + } +} + +// sacSource is queueSource with SingleActiveConsumer set. +func sacSource(name, addr string, subjects ...string) *pb.Source { + s := queueSource(name, addr, subjects...) + s.SingleActiveConsumer = true + return s +} + +// TestSingleActiveConsumerPinsOneClient: a SingleActiveConsumer source maps to +// a pinned-client priority group, so with two live subscribers exactly one +// receives (RabbitMQ SAC semantics — ordered processing across competing +// instances) and the standby takes over after the active one goes away and +// its pin expires. +func TestSingleActiveConsumerPinsOneClient(t *testing.T) { + t.Setenv("NATSJS_SAC_PINNED_TTL", "1s") // fast failover for the test + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.sac", Subjects: []string{"e"}} + src := sacSource("events.sac.consumer", "events.sac", "e") + + // The first subscriber pulls alone, so it takes the pin. + outA := make(chan *pb.Message, 8) + ctxA, cancelA := context.WithCancel(ctx) + errA := make(chan *pb.Error, 1) + go func() { errA <- p.Subscribe(ctxA, src, outA) }() + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m0")})) + m := recv(t, outA) + assert.Equal(t, "m0", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + // A standby joins. Everything published while the pin holder is alive + // must keep going to the pin holder only. + outB := make(chan *pb.Message, 8) + ctxB, cancelB := context.WithCancel(ctx) + defer cancelB() + go p.Subscribe(ctxB, src, outB) + time.Sleep(500 * time.Millisecond) // let the standby's pull reach the server + + for i := 1; i <= 3; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + for i := 1; i <= 3; i++ { + m := recv(t, outA) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + select { + case got := <-outB: + t.Fatalf("standby received %q while another consumer held the pin", got.GetBody()) + case <-time.After(500 * time.Millisecond): + } + + // The active subscriber goes away; once its pin expires (1s here) the + // standby is pinned and receives. + cancelA() + require.Nil(t, <-errA) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m4")})) + m = recv(t, outB) + assert.Equal(t, "m4", string(m.GetBody()), "standby takes over after the active consumer goes away") + require.Nil(t, p.Ack(ctx, m.GetUuid())) +} + +// TestSingleActiveConsumerUpgradesExistingDurable: priority-group config is +// update-mutable, so a durable created before SingleActiveConsumer was set on +// the source is upgraded in place — same name, same ack position — instead of +// failing the subscribe. +func TestSingleActiveConsumerUpgradesExistingDurable(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.sacup", Subjects: []string{"e"}} + plain := queueSource("events.sacup.consumer", "events.sacup", "e") + + // Create the durable without single-active (an older deployment). + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + errCh := make(chan *pb.Error, 1) + go func() { errCh <- p.Subscribe(subCtx, plain, out) }() + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m0")})) + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + cancel() + require.Nil(t, <-errCh) + + // Re-subscribe with SingleActiveConsumer set: the consumer is updated in + // place and keeps delivering from its stored position. + sac := sacSource("events.sacup.consumer", "events.sacup", "e") + out2 := make(chan *pb.Message, 1) + subCtx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + go p.Subscribe(subCtx2, sac, out2) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m1")})) + m = recv(t, out2) + assert.Equal(t, "m1", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + // The effective config now carries the pinned priority group. + bd := p.getBrokerDetailsByIdentifier("test-client") + stream, serr := bd.js.Stream(ctx, streamNameFor("events.sacup")) + require.NoError(t, serr) + cons, cerr := stream.Consumer(ctx, durableName(sac)) + require.NoError(t, cerr) + assert.Equal(t, []string{sacPriorityGroup}, cons.CachedInfo().Config.PriorityGroups) + assert.Equal(t, jetstream.PriorityPolicyPinned, cons.CachedInfo().Config.PriorityPolicy) +} + +// TestSingleActiveConsumerGroupsAreIndependent: the ConsumerGroup option is +// the identity single-active instances coordinate through (amqp091 uses it as +// the consumer reference the same way), so two groups of a source — same +// source name, different ConsumerGroup — must get independent pinned durables +// that each receive the whole stream. Naming the durable after the source +// name instead would collapse every group onto one pinned consumer and starve +// all groups but the pin holder's. +func TestSingleActiveConsumerGroupsAreIndependent(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.sacgroups", Subjects: []string{"e"}} + mkSrc := func(group string) *pb.Source { + return &pb.Source{ + Name: "events.sacgroups.consumer", // shared across groups + Type: pb.Source_STREAM, + SingleActiveConsumer: true, + Address: &pb.Address{Name: "events.sacgroups", Subjects: []string{"e"}}, + Options: map[string]string{"Offset": "first", "ConsumerGroup": "grp-" + group}, + } + } + + const n = 2 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + + outA := make(chan *pb.Message, n) + outB := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, mkSrc("a"), outA) + go p.Subscribe(subCtx, mkSrc("b"), outB) + + // Each group replays the full backlog independently. + for i := 0; i < n; i++ { + require.Nil(t, p.Ack(ctx, recv(t, outA).GetUuid())) + } + for i := 0; i < n; i++ { + require.Nil(t, p.Ack(ctx, recv(t, outB).GetUuid())) + } +} + +// TestSingleActiveStreamSourceRequiresConsumerGroup: a single-active stream +// source names no ConsumerGroup to coordinate through — amqp091 rejects the +// combination, and so does natsjs, instead of inventing a per-subscriber +// durable that would make the instances compete. +func TestSingleActiveStreamSourceRequiresConsumerGroup(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := &pb.Source{ + Name: "events.sacnogroup.consumer", + Type: pb.Source_STREAM, + SingleActiveConsumer: true, + Address: &pb.Address{Name: "events.sacnogroup", Subjects: []string{"e"}}, + } + perr := p.Subscribe(ctx, src, make(chan *pb.Message, 1)) + require.NotNil(t, perr) + assert.Contains(t, perr.GetMessage(), "no ConsumerGroup option set") + assert.True(t, perr.GetIsFatal()) +} + +// TestDeclareOnlyEphemeralConsumerCleanedUp: DeclareOnly on a transient source +// validates topology, but the ephemeral consumer it creates can never be +// attached to afterwards (its server-generated name is not surfaced), so it is +// deleted before Subscribe returns instead of lingering until the inactivity +// threshold. +func TestDeclareOnlyEphemeralConsumerCleanedUp(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := &pb.Source{ + Name: "events.tmpdecl.listener", + Type: pb.Source_QUEUE, + AutoDelete: true, // transient -> ephemeral consumer + DeclareOnly: true, + Address: &pb.Address{Name: "events.tmpdecl", Subjects: []string{"a"}}, + } + require.Empty(t, durableName(src), "test source must map to an ephemeral consumer") + require.Nil(t, p.Subscribe(ctx, src, nil)) + + bd := p.getBrokerDetailsByIdentifier("test-client") + stream, serr := bd.js.Stream(ctx, streamNameFor("events.tmpdecl")) + require.NoError(t, serr) + info, ierr := stream.Info(ctx) + require.NoError(t, ierr) + assert.Zero(t, info.State.Consumers, "declare-only ephemeral consumer must not linger") +} + +func TestRateTracker(t *testing.T) { + rt := newRateTracker() + t0 := time.Now() + + // First observation only establishes the baseline. + assert.Zero(t, rt.observe("k", t0, 100)) + // 10 messages over 2 seconds. + assert.InDelta(t, 5.0, rt.observe("k", t0.Add(2*time.Second), 110), 1e-6) + // A counter that moved backwards (stream recreated) re-baselines at zero. + assert.Zero(t, rt.observe("k", t0.Add(3*time.Second), 4)) + assert.InDelta(t, 6.0, rt.observe("k", t0.Add(4*time.Second), 10), 1e-6) + // A non-advancing clock cannot divide by zero. + assert.Zero(t, rt.observe("k", t0.Add(4*time.Second), 20)) + // Keys are independent. + assert.Zero(t, rt.observe("other", t0, 1)) +} + +// TestSourceStatsStreamSourceReportsOffsetsNotDepth pins the two ways a +// Source_STREAM's stats differ from a queue's on amqp091: no message count +// (the readers of a shared log have no common backlog — the offsets are the +// reading), and RabbitMQ's "Offset not found" when the log holds no offset +// yet, rather than a silent zero. +func TestSourceStatsStreamSourceReportsOffsetsNotDepth(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := &pb.Source{ + Name: "events.log.consumer", + Type: pb.Source_STREAM, + Address: &pb.Address{Name: "events.log", Type: pb.Address_STREAM, Subjects: []string{"e"}}, + } + + // An empty log has no offset to report. + bd, bderr := p.getBrokerDetails(ctx) + require.NoError(t, bderr) + _, serr := p.ensureStream(ctx, bd, "events.log") + require.Nil(t, serr) + empty := p.SourceStats(ctx, src) + require.NotNil(t, empty.GetError()) + assert.Equal(t, offsetNotFoundError, empty.GetError().GetMessage()) + + addr := &pb.Address{Name: "events.log", Subjects: []string{"e"}} + const n = 5 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m")})) + } + + stats := p.SourceStats(ctx, src) + require.Nil(t, stats.GetError()) + assert.EqualValues(t, n-1, stats.GetLastOffset(), "offsets count from 0, so the fifth message is offset 4") + assert.Zero(t, stats.GetMessageCount(), "a stream source reports no backlog, like amqp091") +} + +// TestSourceStatsRates verifies publish/deliver rates are sampled between +// SourceStats calls: the first call baselines at zero, and a later call after +// publishing and consuming reports positive rates. +func TestSourceStatsRates(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.rates", Subjects: []string{"e"}} + src := queueSource("events.rates.consumer", "events.rates", "e") + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("x")})) + + out := make(chan *pb.Message, 8) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + require.Nil(t, p.Ack(ctx, recv(t, out).GetUuid())) + + // Baseline scrape: no previous observation, rates are zero. + stats := p.SourceStats(ctx, src) + require.Nil(t, stats.GetError()) + assert.Zero(t, stats.GetPublishRate()) + assert.Zero(t, stats.GetDeliverRate()) + + for i := 0; i < 5; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("y")})) + require.Nil(t, p.Ack(ctx, recv(t, out).GetUuid())) + } + time.Sleep(50 * time.Millisecond) // guarantee a measurable interval + + stats = p.SourceStats(ctx, src) + require.Nil(t, stats.GetError()) + assert.Positive(t, stats.GetPublishRate(), "5 publishes since the last scrape") + assert.Positive(t, stats.GetDeliverRate(), "5 deliveries since the last scrape") +} + +// TestSourceStatsRatesIndependentPerSource: several sources commonly share +// one address (many queues bound to one exchange), and each polls its stats +// on its own cadence. The rate baselines must therefore be per source: with +// a shared per-stream key, any other source's poll resets the window, and a +// poll right after it reports the rate over that tiny gap — here, zero — +// instead of over the poller's own interval. +func TestSourceStatsRatesIndependentPerSource(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.ratesplit", Subjects: []string{"e"}} + srcA := queueSource("events.ratesplit.a", "events.ratesplit", "e") + srcB := queueSource("events.ratesplit.b", "events.ratesplit", "e") + + // Create the stream and baseline source A. + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("x")})) + require.Nil(t, p.SourceStats(ctx, srcA).GetError()) + + for i := 0; i < 5; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("y")})) + } + + // Source B polls (its own baseline) between A's polls. A's next poll must + // still see the 5 publishes inside its own window. + require.Nil(t, p.SourceStats(ctx, srcB).GetError()) + time.Sleep(20 * time.Millisecond) + + stats := p.SourceStats(ctx, srcA) + require.Nil(t, stats.GetError()) + assert.Positive(t, stats.GetPublishRate(), + "another source's poll must not reset this source's rate window") +} + +// TestSourceStatsRatesIndependentPerGroup: consumer groups share one source +// name and differ only by ConsumerGroup, so a name-keyed publish-rate sample +// makes the groups reset each other's baseline window exactly like the +// per-source case above — the sample key must carry the durable identity too. +func TestSourceStatsRatesIndependentPerGroup(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.grpsplit", Subjects: []string{"e"}} + src := func(cg string) *pb.Source { + return &pb.Source{ + Name: "events.grpsplit.consumer", + Type: pb.Source_STREAM, + Address: &pb.Address{Name: "events.grpsplit", Subjects: []string{"e"}}, + Options: map[string]string{"ConsumerGroup": cg}, + } + } + srcA, srcB := src("grp.a"), src("grp.b") + + // Create the stream and baseline group A. + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("x")})) + require.Nil(t, p.SourceStats(ctx, srcA).GetError()) + + for i := 0; i < 5; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("y")})) + } + + // Group B polls (its own baseline) between A's polls. A's next poll must + // still see the 5 publishes inside its own window. + require.Nil(t, p.SourceStats(ctx, srcB).GetError()) + time.Sleep(20 * time.Millisecond) + + stats := p.SourceStats(ctx, srcA) + require.Nil(t, stats.GetError()) + assert.Positive(t, stats.GetPublishRate(), + "another group's poll must not reset this group's rate window") +} + +func TestUnappliedSourceOptions(t *testing.T) { + src := func(opts map[string]string) *pb.Source { return &pb.Source{Options: opts} } + + assert.Empty(t, unappliedSourceOptions(src(nil))) + assert.Empty(t, unappliedSourceOptions(src(map[string]string{"Offset": "first", "MessageTTL": ""}))) + assert.Equal(t, []string{"MessageTTL"}, unappliedSourceOptions(src(map[string]string{"MessageTTL": "300000"}))) + // Expires is applied (consumer InactiveThreshold), so it must not be + // reported — and warned about — as unapplied. + assert.Equal(t, []string{"MessageTTL"}, + unappliedSourceOptions(src(map[string]string{"MessageTTL": "300000", "Expires": "600000"}))) +} + +func TestExpiresThreshold(t *testing.T) { + src := func(opts map[string]string) *pb.Source { return &pb.Source{Options: opts} } + + d, err := expiresThreshold(src(nil)) + require.NoError(t, err) + assert.Zero(t, d, "unset means the kind's default applies") + + d, err = expiresThreshold(src(map[string]string{"Expires": "600000"})) + require.NoError(t, err) + assert.Equal(t, 10*time.Minute, d) + + _, err = expiresThreshold(src(map[string]string{"Expires": "10m"})) + require.EqualError(t, err, "value for Expires option must be a quoted integer", + "error string mirrors the amqp091 connector") + _, err = expiresThreshold(src(map[string]string{"Expires": "0"})) + require.EqualError(t, err, "value for Expires option must be a positive integer") + _, err = expiresThreshold(src(map[string]string{"Expires": "-5"})) + require.EqualError(t, err, "value for Expires option must be a positive integer") +} + +// connectSecondClient attaches a second, independently-identified client to the +// same server, so a test can exercise what one connection's state (the +// knownStreams memo, say) hides from another. +func connectSecondClient(t *testing.T, s *natsserver.Server, id string) (*natsjsProvider, context.Context) { + t.Helper() + saved := GetClientIdentifier + t.Cleanup(func() { GetClientIdentifier = saved }) + GetClientIdentifier = func(context.Context) (string, error) { return id, nil } + + addr := s.Addr().(*net.TCPAddr) + p := NewNATSJetStreamProvider().(*natsjsProvider) + ctx := context.Background() + if perr := p.Connect(ctx, &pb.ConnectionConfiguration{ + Host: "127.0.0.1", Port: int32(addr.Port), //nolint:gosec // local server port fits int32 + ClientName: id}, false); perr != nil { + t.Fatalf("connect %s: %s", id, perr.GetMessage()) + } + if !p.WaitForConnect(ctx) { + t.Fatalf("WaitForConnect returned false for %s", id) + } + return p, ctx +} + +// TestBindingSurvivesBindinglessDeclare: asserting a stream that has +// address-to-address bindings must carry them forward even when the asserting +// call declares none of its own. CreateOrUpdateStream replaces the source set +// wholesale, so a config that omits it silently unbinds the address — and the +// call that omits it is the most ordinary one there is: a publisher naming the +// bound address directly, which carries no ParentAddress. The per-connection +// knownStreams memo hides this from the connection that declared the binding, +// so the regression only shows up from a second one. +func TestBindingSurvivesBindinglessDeclare(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + parent := &pb.Address{Name: "events.bsurv.parent", Type: pb.Address_TOPIC} + child := &pb.Address{ + Name: "events.bsurv.child", + Type: pb.Address_TOPIC, + Subjects: []string{"key.one"}, + ParentAddress: parent, + } + + out := make(chan *pb.Message, 8) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, &pb.Source{Name: "events.bsurv.consumer", Type: pb.Source_QUEUE, + Address: child, Options: map[string]string{"Offset": "first"}}, out) + time.Sleep(500 * time.Millisecond) + + pubToParent := func(t *testing.T, body string) { + t.Helper() + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.bsurv.parent", Subjects: []string{"key.one"}}, + Body: []byte(body)})) + } + + pubToParent(t, "before") + m := recv(t, out) + require.Equal(t, "before", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + // A second client publishes straight to the bound address, naming no + // parent. Its connection has never asserted this stream, so it takes the + // create-or-update path with no bindings of its own to declare. + p2, ctx2 := connectSecondClient(t, s, "other-client") + require.Nil(t, p2.PublishOne(ctx2, &pb.Message{ + Address: &pb.Address{Name: "events.bsurv.child", Subjects: []string{"direct"}}, + Body: []byte("direct")})) + + // The binding the first client declared must still route. + GetClientIdentifier = func(context.Context) (string, error) { return "test-client", nil } + pubToParent(t, "after") + + deadline := time.After(10 * time.Second) + for { + select { + case m := <-out: + require.Nil(t, p.Ack(ctx, m.GetUuid())) + if string(m.GetBody()) == "after" { + return + } + case <-deadline: + t.Fatal("the address-to-address binding did not survive a bindingless declare") + } + } +} + +// TestFailedDeadLetterRedeliveryIsPaced: a dead-letter that fails leaves the +// message in flight so the server's fallback nack retries it (see +// TestDeadLetterFailureKeepsMessage). That retry runs the same attempt against +// the same configuration, so when the failure is permanent — here an empty +// DeadLetterAddress — an immediate nak spins the message between server and +// client as fast as the client can nack, and MaxDeliver is deliberately unset +// so nothing bounds it. The redelivery must be paced instead. +func TestFailedDeadLetterRedeliveryIsPaced(t *testing.T) { + t.Setenv("NATSJS_DEADLETTER_RETRY_DELAY", "500ms") + + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dlpace", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 64) + src := queueSource("events.dlpace.consumer", "events.dlpace", "job") + src.Options["DeadLetterAddress"] = "" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + // Replay the gRPC server's nack handling for one poison message: it calls + // DeadLetter whenever the DeadLetterAddress key is present (whatever its + // value) and falls back to Nack when that errors. + deliveries := 0 + const window = 3 * time.Second + deadline := time.After(window) + for { + select { + case m := <-out: + deliveries++ + if deliveries > 40 { + t.Fatalf("redelivery storm: %d deliveries of one message in under %s", deliveries, window) + } + require.NotNil(t, p.DeadLetter(ctx, src, m.GetUuid())) + require.Nil(t, p.Nack(ctx, m.GetUuid())) + case <-deadline: + // Paced, but still retrying: dropping the message instead would + // lose it while it is also absent from any DLQ. + assert.GreaterOrEqual(t, deliveries, 2, "a failed dead-letter must still be retried") + assert.LessOrEqual(t, deliveries, 20, + "redelivery of a permanently-failing dead-letter must be paced, not immediate") + return + } + } +} + +// TestSubscribeInvalidOffsetCreatesNoStream: an unrecognized Offset is a +// contract refusal, and a refused subscribe must not leave topology behind for +// an address the client only ever named by mistake. +func TestSubscribeInvalidOffsetCreatesNoStream(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + out := make(chan *pb.Message, 1) + perr := p.Subscribe(ctx, streamSource("events.nostream.consumer", "events.nostream", "latest", "e"), out) + require.NotNil(t, perr) + assert.Contains(t, perr.GetMessage(), "invalid offset") + + bd := p.getBrokerDetailsByIdentifier("test-client") + _, serr := bd.js.Stream(ctx, streamNameFor("events.nostream")) + assert.ErrorIs(t, serr, jetstream.ErrStreamNotFound, + "a subscribe refused on its options must not have created the address's stream") +} + +// TestConcurrentResolveAndMarkLeavesNoOrphans: resolving a message and marking +// one for redelivery are both read-modify-writes over activeMessages. Run +// concurrently without a lock between them, a mark landing between a resolve's +// read and its delete reinserts an entry nothing can ever resolve, leaking it +// for the life of the connection and inflating the active-message stat. +func TestConcurrentResolveAndMarkLeavesNoOrphans(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + bd := p.getBrokerDetailsByIdentifier("test-client") + + const n = 300 + uuids := make([]string, 0, n) + for i := 0; i < n; i++ { + uuid := fmt.Sprintf("uuid-%d", i) + uuids = append(uuids, uuid) + bd.activeMessages.Add(uuid, inflightMsg{subKey: "sub#1"}) + } + + var wg sync.WaitGroup + for _, uuid := range uuids { + wg.Add(2) + go func() { defer wg.Done(); _, _ = p.takeMessage(ctx, uuid) }() + go func() { defer wg.Done(); markForRedelivery(bd, uuid) }() + } + wg.Wait() + + assert.Equal(t, 0, bd.activeMessages.Length(), + "marking a message for redelivery must not reinsert one that was resolved concurrently") +} + +// TestPublishDedupRefusedOnEveryNonStreamAddress pins the address types a +// unary publish may ask for deduplication on. amqp091's PublishOne routes only +// Address_STREAM to publishOneStream (the RabbitMQ Streams client, the one +// thing that can deduplicate); every other type — TOPIC, QUEUE and FILTER — +// goes to publishOneQueue, which refuses PublishId outright (amqp091.go:1501). +// TOPIC is the case that matters most: it is the proto zero value, so an +// address that never sets a type lands here. +// +// Regression: the refusal used to test == Address_QUEUE, so a TOPIC or FILTER +// address silently accepted the id and deduplicated — giving a client a +// guarantee that turns into a hard error the moment it runs against RabbitMQ. +func TestPublishDedupRefusedOnEveryNonStreamAddress(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + for _, tc := range []struct { + name string + typ pb.Address_TargetType + }{ + {"topic (the proto zero value)", pb.Address_TOPIC}, + {"queue", pb.Address_QUEUE}, + {"filter", pb.Address_FILTER}, + } { + t.Run(tc.name, func(t *testing.T) { + perr := p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.dedup.scope." + tc.name, Type: tc.typ, Subjects: []string{"k"}}, + PublisherName: "pub", PublishId: 1, Body: []byte("x")}) + require.NotNil(t, perr, "a unary publish asking for dedup on a non-STREAM address must be refused") + assert.Equal(t, queueDedupError, perr.GetMessage()) + }) + } + + // A STREAM address is the one that may: declare it, then dedup applies. + bd := p.getBrokerDetailsByIdentifier("test-client") + _, eerr := p.ensureStream(ctx, bd, "events.dedup.scope.stream") + require.NoError(t, eerr) + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.dedup.scope.stream", Type: pb.Address_STREAM, Subjects: []string{"k"}}, + PublisherName: "pub", PublishId: 1, Body: []byte("x")})) +} + +// TestStreamingPublishIgnoresPublishID pins the other half of the dedup +// contract: the whole publish-id contract belongs to the unary path only. +// amqp091's streaming Publish calls prepareAndSend directly for EVERY address +// type — the plain channel publish, which reads neither PublishId nor +// PublisherName — so natsjs must not refuse the id there (that would be +// stricter than RabbitMQ, breaking a publisher that works there today) and +// must not honour it there either (that would be a guarantee RabbitMQ does not +// give, and honouring it discards a message RabbitMQ stores). +// +// Regression: the address type alone decided this, so a STREAM address was +// deduplicated on the streaming path too — silently dropping the second +// publish, where amqp091 never reaches the Streams client that deduplicates. +func TestStreamingPublishIgnoresPublishID(t *testing.T) { + for _, tc := range []struct { + name string + typ pb.Address_TargetType + }{ + {"topic (the proto zero value)", pb.Address_TOPIC}, + {"queue", pb.Address_QUEUE}, + // The one that regressed: STREAM is the type the UNARY path + // deduplicates on, but the streaming path never routes to the client + // that does the deduplicating, so the id has to be dropped here too. + {"stream", pb.Address_STREAM}, + } { + t.Run(tc.name, func(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + in := make(chan *pb.Message, 2) + errChan := make(chan *pb.Error, 2) + pubCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Publish(pubCtx, in, errChan) + + addr := &pb.Address{Name: "events.stream.dedupignored", Type: tc.typ, Subjects: []string{"k"}} + for i := 0; i < 2; i++ { + in <- &pb.Message{Address: addr, Body: []byte("x"), PublisherName: "pub", PublishId: 42} + select { + case e := <-errChan: + require.Nil(t, e, "the streaming path must not refuse a publish id: %v", e.GetMessage()) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the publish reply") + } + } + + stats := p.SourceStats(ctx, &pb.Source{ + Name: "events.stream.dedupignored.consumer", + Address: &pb.Address{Name: "events.stream.dedupignored"}, + }) + assert.Nil(t, stats.GetError()) + assert.Equal(t, int64(2), stats.GetMessageCount(), + "the id must be ignored on the streaming path, not deduplicated on — RabbitMQ drops it on the floor") + }) + } +} + +// TestStreamingPublishAcceptsPublishIDWithoutPublisherName pins the companion +// refusal to the unary path as well. amqp091 requires a PublisherName +// alongside a PublishID only in publishOneStream (amqp091.go:1544), where the +// RabbitMQ Streams client needs a named publisher to deduplicate against. The +// streaming Publish never routes there and reads neither field, so this +// publish succeeds on RabbitMQ and must succeed here. +func TestStreamingPublishAcceptsPublishIDWithoutPublisherName(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + in := make(chan *pb.Message, 1) + errChan := make(chan *pb.Error, 1) + pubCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Publish(pubCtx, in, errChan) + + in <- &pb.Message{ + Address: &pb.Address{Name: "events.stream.noPubName", Type: pb.Address_TOPIC, Subjects: []string{"k"}}, + Body: []byte("x"), + PublishId: 42, + } + select { + case e := <-errChan: + require.Nil(t, e, "the streaming path reads neither PublishId nor PublisherName: %v", e.GetMessage()) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the publish reply") + } +} + +// TestHeadersAddressIgnoresRoutingKey covers a headers address that also +// carries subjects. RabbitMQ's headers exchange never looks at the routing key +// — rabbit_exchange_type_headers matches every binding of the exchange on its +// header arguments alone — so the subjects amqp091 passes to QueueBind decide +// nothing, and a message published under any key reaches the consumer. +// +// Regression: subjects on a FILTER address used to become NATS consumer filter +// subjects, so a message whose routing key matched no subject was dropped — +// silently delivering strictly less than RabbitMQ would. +func TestHeadersAddressIgnoresRoutingKey(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.headers", Type: pb.Address_FILTER, Subjects: []string{"bound.one"}} + out := make(chan *pb.Message, 8) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, &pb.Source{Name: "events.headers.consumer", Type: pb.Source_QUEUE, + Address: addr, Options: map[string]string{"Offset": "first"}}, out) + time.Sleep(500 * time.Millisecond) + + for _, key := range []string{"bound.one", "some.other.key"} { + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.headers", Type: pb.Address_FILTER, Subjects: []string{key}}, + Body: []byte(key)})) + } + + got := map[string]bool{} + for len(got) < 2 { + m := recv(t, out) + got[string(m.GetBody())] = true + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + assert.True(t, got["bound.one"], "the message under the declared key must arrive") + assert.True(t, got["some.other.key"], + "a headers exchange ignores routing keys, so a message under any key must arrive") +} + +// TestHeadersAddressStillAppliesHeaderFilters is the guard on the test above: +// selecting the whole address must hand the decision to the header filters, +// not deliver everything unconditionally. +func TestHeadersAddressStillAppliesHeaderFilters(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.headers.filtered", Type: pb.Address_FILTER, Subjects: []string{"bound.one"}} + out := make(chan *pb.Message, 8) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, &pb.Source{ + Name: "events.headers.filtered.consumer", Type: pb.Source_QUEUE, Address: addr, + Filters: []*pb.Filter{{Type: pb.Filter_ALL, Matches: []*pb.Match{{Name: "tenant", Value: "acme"}}}}, + Options: map[string]string{"Offset": "first"}, + }, out) + time.Sleep(500 * time.Millisecond) + + // Matching headers under a key no subject names: delivered (headers decide). + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.headers.filtered", Type: pb.Address_FILTER, Subjects: []string{"any.key"}}, + Headers: map[string]string{"tenant": "acme"}, Body: []byte("match")})) + // Non-matching headers under the declared key: filtered out. + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.headers.filtered", Type: pb.Address_FILTER, Subjects: []string{"bound.one"}}, + Headers: map[string]string{"tenant": "other"}, Body: []byte("nomatch")})) + + m := recv(t, out) + assert.Equal(t, "match", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("header filters must still gate a headers address; got %q", string(extra.GetBody())) + case <-time.After(2 * time.Second): + } +} + +// TestHeaderFilterMatchesWireEscapedHeader exercises the intersection of two +// features added at different times: the wire-format header escaping +// (pbToNatsHeader/natsToPbHeader — a name or value the NATS wire would drop or +// trim is base64-wrapped under X-Arke-Esc-) and proxy-side headers-exchange +// matching (evaluateFilters). A FILTER source that matches on a header whose +// NAME is not an RFC 7230 token (spaces, a colon) and whose VALUE would not +// survive http.Header.Write (leading/trailing spaces) must still match exactly +// the messages RabbitMQ's headers exchange would — the filter is evaluated on +// the DECODED header map, so the escape has to round-trip transparently before +// the match runs. If it did not, a headers source keyed on any non-token header +// would silently drop every message. Neither the escaping tests +// (TestHeadersSurviveARealBrokerRoundTrip) nor the filter tests +// (TestHeaderFilterDropsNonMatching) cover this composition on their own. +func TestHeaderFilterMatchesWireEscapedHeader(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + // Name has a space and a colon (neither is a tchar); value has leading and + // trailing spaces (trimmed by the wire) — so both halves force escaping. + const hdrName, hdrVal = "x tenant:id", " acme " + addr := &pb.Address{Name: "events.escfilter", Type: pb.Address_FILTER, Subjects: []string{"k"}} + out := make(chan *pb.Message, 8) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, &pb.Source{ + Name: "events.escfilter.consumer", Type: pb.Source_QUEUE, Address: addr, + Filters: []*pb.Filter{{Type: pb.Filter_ALL, Matches: []*pb.Match{{Name: hdrName, Value: hdrVal}}}}, + Options: map[string]string{"Offset": "first"}, + }, out) + time.Sleep(500 * time.Millisecond) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, Headers: map[string]string{hdrName: hdrVal}, Body: []byte("match")})) + // Same escaped name, a value that differs only in the spaces the wire would + // have trimmed — proving the value round-trips byte-for-byte into the match. + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, Headers: map[string]string{hdrName: "acme"}, Body: []byte("nomatch")})) + + m := recv(t, out) + assert.Equal(t, "match", string(m.GetBody())) + assert.Equal(t, hdrVal, m.GetHeaders()[hdrName], + "the escaped header must decode byte-for-byte for the consumer, not just for the filter") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("a value differing only by wire-trimmed spaces must not match; got %q", string(extra.GetBody())) + case <-time.After(2 * time.Second): + } +} + +// TestBareStreamAssertionIssuesNoUpdate pins that asserting a stream that +// already satisfies the requested config writes nothing. +// +// Carrying the existing binding set forward makes ensureStreamFor a +// read-modify-write, and p.bindMu serializes that only inside one process — +// Arke runs more than one replica. Skipping the write when nothing would +// change takes the overwhelmingly common case (every publisher's first touch +// of an existing stream) out of that cross-process race, so a bare assertion +// can no longer be the thing that drops a binding another replica just added. +func TestBareStreamAssertionIssuesNoUpdate(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + bd := p.getBrokerDetailsByIdentifier("test-client") + + streamName := streamNameFor("events.noupdate") + _, eerr := p.ensureStream(ctx, bd, "events.noupdate") + require.NoError(t, eerr) + + // Watch the JetStream management API for updates to this stream. A plain + // subscription is an observer: the server's own responder still receives + // the request, so this changes nothing about the call itself. + updates := make(chan struct{}, 8) + sub, serr := bd.nc.Subscribe("$JS.API.STREAM.UPDATE."+streamName, func(*nats.Msg) { + updates <- struct{}{} + }) + require.NoError(t, serr) + defer sub.Unsubscribe() //nolint:errcheck + require.NoError(t, bd.nc.Flush()) + + // Drop the per-connection memo so the assertion really re-runs against the + // server, exactly as a fresh connection's first touch would. + bd.knownStreams.Delete(streamName) + _, eerr = p.ensureStream(ctx, bd, "events.noupdate") + require.NoError(t, eerr) + require.NoError(t, bd.nc.Flush()) + + select { + case <-updates: + t.Fatal("a bare assertion of an already-satisfying stream must not issue a stream update") + case <-time.After(500 * time.Millisecond): + } + + // ...but an assertion that genuinely changes something still writes. A + // binding is the case that matters: it must survive the skip logic. + child := &pb.Address{ + Name: "events.noupdate", + Type: pb.Address_TOPIC, + Subjects: []string{"routed.key"}, + ParentAddress: &pb.Address{Name: "events.noupdate.parent", Type: pb.Address_TOPIC}, + } + _, eerr = p.ensureStreamFor(ctx, bd, child) + require.NoError(t, eerr) + require.NoError(t, bd.nc.Flush()) + select { + case <-updates: + case <-time.After(2 * time.Second): + t.Fatal("declaring a new binding must still update the stream") + } + + // And the binding really landed — the skip must not have swallowed it. + st, ferr := bd.js.Stream(ctx, streamName) + require.NoError(t, ferr) + sources := st.CachedInfo().Config.Sources + require.Len(t, sources, 1) + assert.Equal(t, streamNameFor("events.noupdate.parent"), sources[0].Name) + require.Len(t, sources[0].SubjectTransforms, 1) + assert.Equal(t, publishSubjectFor("events.noupdate.parent", "routed.key"), + sources[0].SubjectTransforms[0].Source) + + // Re-asserting that same binding is now itself a no-op. + bd.knownStreams.Delete(streamName) + _, eerr = p.ensureStreamFor(ctx, bd, child) + require.NoError(t, eerr) + require.NoError(t, bd.nc.Flush()) + select { + case <-updates: + t.Fatal("re-declaring an existing binding must not issue a stream update") + case <-time.After(500 * time.Millisecond): + } + + // A SECOND binding onto a parent already in the source set must still + // write. This is the case the skip gets wrong if the config being built + // shares its StreamSource objects with the snapshot it is compared + // against: merging appends to the transforms in place, so both sides move + // together and a genuinely new binding reads as no change at all. A first + // binding hides that — it appends a whole element, so the lengths differ + // regardless. + sibling := &pb.Address{ + Name: "events.noupdate", + Type: pb.Address_TOPIC, + Subjects: []string{"second.key"}, + ParentAddress: &pb.Address{Name: "events.noupdate.parent", Type: pb.Address_TOPIC}, + } + _, eerr = p.ensureStreamFor(ctx, bd, sibling) + require.NoError(t, eerr) + require.NoError(t, bd.nc.Flush()) + select { + case <-updates: + case <-time.After(2 * time.Second): + t.Fatal("a second binding onto an already-bound parent must still update the stream") + } + + st, ferr = bd.js.Stream(ctx, streamName) + require.NoError(t, ferr) + sources = st.CachedInfo().Config.Sources + require.Len(t, sources, 1, "both bindings share one parent, so one source carries both") + assert.Len(t, sources[0].SubjectTransforms, 2, "the earlier binding must survive the later one") +} + +// TestStreamConfigSatisfied covers the comparison that lets ensureStreamFor +// skip an assertion which would change nothing (see +// TestBareStreamAssertionIssuesNoUpdate for why that matters). It must answer +// "satisfied" only when every field the connector manages already matches — +// a false positive there silently drops a real config change. +func TestStreamConfigSatisfied(t *testing.T) { + base := func() *jetstream.StreamConfig { + return &jetstream.StreamConfig{ + Name: "arke_events_x", + Subjects: []string{"events.x.~", "events.x.~.>"}, + Storage: jetstream.FileStorage, + Retention: jetstream.LimitsPolicy, + Duplicates: defaultDedupWindow, + Replicas: 1, + MaxAge: defaultStreamMaxAge, + } + } + assert.True(t, streamConfigSatisfied(base(), base())) + + // Subject order is the server's to choose, not a difference. + reordered := base() + reordered.Subjects = []string{"events.x.~.>", "events.x.~"} + assert.True(t, streamConfigSatisfied(reordered, base())) + + // Each managed field, differing. + for name, mutate := range map[string]func(*jetstream.StreamConfig){ + "subjects": func(c *jetstream.StreamConfig) { c.Subjects = []string{"events.x.~"} }, + "max age": func(c *jetstream.StreamConfig) { c.MaxAge = time.Hour }, + "max bytes": func(c *jetstream.StreamConfig) { c.MaxBytes = 1 << 20 }, + "replicas": func(c *jetstream.StreamConfig) { c.Replicas = 3 }, + "duplicates": func(c *jetstream.StreamConfig) { c.Duplicates = time.Minute }, + "storage": func(c *jetstream.StreamConfig) { c.Storage = jetstream.MemoryStorage }, + "retention": func(c *jetstream.StreamConfig) { c.Retention = jetstream.WorkQueuePolicy }, + } { + t.Run(name+" differs", func(t *testing.T) { + live := base() + mutate(live) + assert.False(t, streamConfigSatisfied(live, base()), + "a differing %s must not read as satisfied", name) + }) + } + + // A stream with no bindings does not satisfy a config that declares one... + want := base() + want.Sources = []*jetstream.StreamSource{{Name: "arke_events_p", + SubjectTransforms: []jetstream.SubjectTransformConfig{{Source: "events.p.~.k", Destination: "events.p.~.k"}}}} + assert.False(t, streamConfigSatisfied(base(), want)) + + // ...and one that already carries exactly that binding does. + live := base() + live.Sources = []*jetstream.StreamSource{{Name: "arke_events_p", + SubjectTransforms: []jetstream.SubjectTransformConfig{{Source: "events.p.~.k", Destination: "events.p.~.k"}}}} + assert.True(t, streamConfigSatisfied(live, want)) + + // An extra binding on the live stream is a difference in the other + // direction: the merged config would have carried it forward, so a + // mismatch here means something else changed it concurrently. + extra := base() + extra.Sources = append(copySources(live.Sources), &jetstream.StreamSource{Name: "arke_events_q"}) + assert.False(t, streamConfigSatisfied(extra, want)) +} + +// TestCopySourcesIsDeep guards the trap that makes the skip logic safe: +// mergeBoundSources appends to a StreamSource's transforms in place, so the +// snapshot ensureStreamFor compares against must not share those objects with +// the config it merges into — or every merge would compare equal to itself and +// a newly declared binding would be skipped instead of written. +func TestCopySourcesIsDeep(t *testing.T) { + orig := []*jetstream.StreamSource{{Name: "arke_events_p", + SubjectTransforms: []jetstream.SubjectTransformConfig{{Source: "a", Destination: "a"}}}} + snapshot := copySources(orig) + + merged := mergeBoundSources(orig, "arke_events_p", []string{"b"}) + require.Len(t, merged[0].SubjectTransforms, 2) + assert.Len(t, snapshot[0].SubjectTransforms, 1, + "the snapshot must not see a transform appended after it was taken") + assert.False(t, sameSources(snapshot, merged), + "a merge that added a binding must read as a difference") +} + +// TestStreamConfigSatisfiedUnlimitedSentinel: JetStream normalizes an unset +// size limit to its -1 "unlimited" sentinel and echoes that back, so a config +// asking for 0 describes the same stream. Comparing the two literally made +// every assertion look like a change, defeating the skip entirely. +func TestStreamConfigSatisfiedUnlimitedSentinel(t *testing.T) { + live := &jetstream.StreamConfig{Subjects: []string{"a"}, Replicas: 1, MaxBytes: -1} + want := &jetstream.StreamConfig{Subjects: []string{"a"}, Replicas: 1, MaxBytes: 0} + assert.True(t, streamConfigSatisfied(live, want), + "an unset max-bytes and the server's -1 sentinel describe the same stream") + + // A real cap is still a difference in both directions. + capped := &jetstream.StreamConfig{Subjects: []string{"a"}, Replicas: 1, MaxBytes: 1 << 20} + assert.False(t, streamConfigSatisfied(live, capped)) + assert.False(t, streamConfigSatisfied(capped, want)) +} + +// TestHeadersSurviveARealBrokerRoundTrip is the end-to-end half of the header +// escaping: the helper tests prove the encoding is reversible and that it +// survives net/http's writer, but only a real publish and consume proves the +// connector actually routes headers through that encoding on both sides. +// +// Every name here round-trips through RabbitMQ unchanged. Without escaping the +// NATS wire format drops the punctuation and non-ASCII names outright and trims +// the padded values, silently — measured against a live broker of each kind as +// 15 of 38 names delivered. +func TestHeadersSurviveARealBrokerRoundTrip(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + sent := map[string]string{ + "Content-Type": "application/vnd.example.event+json", + "__namespace": "tenant-a", + "x-event-address": "events.orders", + "traceparent": "00-abc-def-01", + "has space": "spacekey", + "has:colon": "colonvalue", + "has/slash": "slashkey", + "unicode-vàlue": "héllo wörld", + "中文": "cjk", + "padded-value": " keep my spaces ", + "tabbed-value": "\tkeep\ttabs\t", + "empty-value": "", + } + + addr := &pb.Address{Name: "events.hdrroundtrip", Subjects: []string{"probe"}} + out := make(chan *pb.Message, 4) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, &pb.Source{ + Name: "events.hdrroundtrip.consumer", Type: pb.Source_QUEUE, Address: addr, + Options: map[string]string{"Offset": "first"}, + }, out) + time.Sleep(500 * time.Millisecond) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, Headers: sent, Body: []byte("probe")})) + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + got := m.GetHeaders() + for k, want := range sent { + v, ok := got[k] + assert.True(t, ok, "header %q must be delivered, not silently dropped", k) + assert.Equal(t, want, v, "header %q must arrive byte-for-byte", k) + } +} + +// TestTimestampHeaderIsSynthesized: on RabbitMQ the broker's own ingress +// interceptor stamps timestamp_in_ms on every incoming message, so an amqp091 +// consumer always sees one. NATS has no interceptor, so the connector has to +// synthesize it from the JetStream store time or clients that read it see +// nothing. +// +// This was long recorded as a capability gap ("NATS has no concept of it") and +// parked in the per-broker skip list. That was wrong: JetStream carries a +// per-message timestamp in its metadata. +func TestTimestampHeaderIsSynthesized(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.timestamped", Subjects: []string{"probe"}} + out := make(chan *pb.Message, 4) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, &pb.Source{ + Name: "events.timestamped.consumer", Type: pb.Source_QUEUE, Address: addr, + Options: map[string]string{"Offset": "first"}, + }, out) + time.Sleep(500 * time.Millisecond) + + before := time.Now().UnixMilli() + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, + // A publisher-supplied value must be replaced, not kept: the RabbitMQ + // interceptor this stands in for runs with overwrite = true. + Headers: map[string]string{timestampHeaderName: "1"}, + Body: []byte("probe")})) + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + raw, ok := m.GetHeaders()[timestampHeaderName] + require.True(t, ok, "every delivery must carry %s, as it does on RabbitMQ", timestampHeaderName) + ms, err := strconv.ParseInt(raw, 10, 64) + require.NoError(t, err, "%s must be integer milliseconds", timestampHeaderName) + assert.GreaterOrEqual(t, ms, before, "must be the broker store time, not the publisher's value") + assert.LessOrEqual(t, ms, time.Now().UnixMilli()+1000) +} + +// TestSingleActiveConsumerDowngradeReleasesPinPromptly: a durable that drops +// SingleActiveConsumer must resume ordinary (non-pinned) delivery promptly. +// Without releaseStalePins, CreateOrUpdateConsumer clears the priority-group +// config immediately but the server's own pin persists as runtime state +// independent of that config (verified against the embedded server: unpinning +// AFTER the config is cleared errors "priority group does not exist for this +// consumer") — so a plain re-subscribe would silently receive nothing until +// NATSJS_SAC_PINNED_TTL (default 1m) elapsed, with no error anywhere. This +// pins the fix at the connector's default TTL, not a shortened test-only one, +// so it actually proves the wait is gone rather than just faster. +func TestSingleActiveConsumerDowngradeReleasesPinPromptly(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.sacdowngrade", Subjects: []string{"e"}} + sac := sacSource("events.sacdowngrade.consumer", "events.sacdowngrade", "e") + + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + errCh := make(chan *pb.Error, 1) + go func() { errCh <- p.Subscribe(subCtx, sac, out) }() + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m0")})) + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + cancel() + require.Nil(t, <-errCh) + + // Re-subscribe to the same durable without SingleActiveConsumer. + plain := queueSource("events.sacdowngrade.consumer", "events.sacdowngrade", "e") + out2 := make(chan *pb.Message, 1) + subCtx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + go p.Subscribe(subCtx2, plain, out2) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m1")})) + select { + case m := <-out2: + require.Nil(t, p.Ack(ctx, m.GetUuid())) + assert.Equal(t, "m1", string(m.GetBody())) + case <-time.After(5 * time.Second): + t.Fatal("plain re-subscribe never received a message: the prior single-active pin was not released") + } + + bd := p.getBrokerDetailsByIdentifier("test-client") + stream, serr := bd.js.Stream(ctx, streamNameFor("events.sacdowngrade")) + require.NoError(t, serr) + cons, cerr := stream.Consumer(ctx, durableName(plain)) + require.NoError(t, cerr) + assert.Empty(t, cons.CachedInfo().Config.PriorityGroups, "the downgraded consumer must carry no priority group") +} + +// TestStreamDeliveryStampsCurrentOffset: amqp091's streamSubscribe stamps +// x-current-offset on every STREAM-source delivery from the RabbitMQ Streams +// consumer's own offset (amqp091.go:1279). The JetStream analogue is the +// message's own stream sequence in the Offset vocabulary SourceStats already +// uses, so a value read here and handed back as a source's Offset resumes at +// the same message. +func TestStreamDeliveryStampsCurrentOffset(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.curoffset", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m0")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m1")})) + + out := make(chan *pb.Message, 4) + src := streamSource("events.curoffset.consumer", "events.curoffset", "first", "e") + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m0 := recv(t, out) + require.Nil(t, p.Ack(ctx, m0.GetUuid())) + assert.Equal(t, "0", m0.GetHeaders()[currentOffsetHeaderName], + "the first message in a stream must report offset 0, matching the RabbitMQ-Streams vocabulary") + + m1 := recv(t, out) + require.Nil(t, p.Ack(ctx, m1.GetUuid())) + assert.Equal(t, "1", m1.GetHeaders()[currentOffsetHeaderName]) +} + +// TestQueueDeliveryOmitsCurrentOffset: amqp091 only stamps x-current-offset on +// the RabbitMQ-Streams delivery path (streamSubscribe) — queueSubscribe never +// sets it — so a QUEUE source must not carry it either. +func TestQueueDeliveryOmitsCurrentOffset(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.noqcuroffset", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m")})) + + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.noqcuroffset.consumer", "events.noqcuroffset", "e"), out) + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + _, ok := m.GetHeaders()[currentOffsetHeaderName] + assert.False(t, ok, "x-current-offset is a STREAM-only header on amqp091 too; a queue source must not carry it") +} + +// TestStreamGzipBodyIsDecompressed: amqp091's streamSubscribe decompresses a +// gzip-tagged body and strips the header (amqp091.go:1286-1292) — a proxy-side +// step needed because arke's own STREAM publish path transparently compresses +// bodies over the RabbitMQ-Streams client's ~1MiB framing limit, but applied +// to any body carrying the header regardless of who compressed it. Without +// this, a natsjs STREAM consumer sees the raw compressed bytes: body +// corruption, not just a missing header. +func TestStreamGzipBodyIsDecompressed(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.gzstream", Subjects: []string{"e"}} + plain := []byte("the quick brown fox jumps over the lazy dog - repeated for compressibility - the quick brown fox") + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + _, err := gw.Write(plain) + require.NoError(t, err) + require.NoError(t, gw.Close()) + + src := streamSource("events.gzstream.consumer", "events.gzstream", "first", "e") + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, + Headers: map[string]string{transferEncodingHeaderName: "gzip"}, + Body: buf.Bytes(), + })) + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + assert.Equal(t, plain, m.GetBody(), "a STREAM consumer must see the decompressed body, matching amqp091's streamSubscribe") + _, ok := m.GetHeaders()[transferEncodingHeaderName] + assert.False(t, ok, "the Transfer-Encoding header must be stripped once the body is decompressed, matching amqp091") +} + +// TestQueueGzipBodyPassesThroughUntouched: amqp091 only decompresses on the +// STREAM path; a QUEUE source's body and header must pass through exactly as +// published, on both connectors. +func TestQueueGzipBodyPassesThroughUntouched(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.gzqueue", Subjects: []string{"e"}} + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + _, err := gw.Write([]byte("hello")) + require.NoError(t, err) + require.NoError(t, gw.Close()) + gzipped := append([]byte(nil), buf.Bytes()...) + + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.gzqueue.consumer", "events.gzqueue", "e"), out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, + Headers: map[string]string{transferEncodingHeaderName: "gzip"}, + Body: gzipped, + })) + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + assert.Equal(t, gzipped, m.GetBody()) + assert.Equal(t, "gzip", m.GetHeaders()[transferEncodingHeaderName]) +} + +// TestStreamGzipDecompressFailureKeepsRawBody: a STREAM body tagged +// Transfer-Encoding: gzip that is not in fact valid gzip must be delivered +// untouched — raw body and the header both preserved — rather than dropped or +// blanked. This matches amqp091's streamSubscribe, which on a decompress error +// logs at debug and forwards the original data with the header intact +// (amqp091.go decompress branch). natsjs never compresses on publish, so this +// failure path only ever fires for a foreign publisher's mislabelled body; the +// success path (TestStreamGzipBodyIsDecompressed) left it uncovered. +func TestStreamGzipDecompressFailureKeepsRawBody(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.gzbad", Subjects: []string{"e"}} + notGzip := []byte("this is plainly not gzip-framed data") + + src := streamSource("events.gzbad.consumer", "events.gzbad", "first", "e") + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, + Headers: map[string]string{transferEncodingHeaderName: "gzip"}, + Body: notGzip, + })) + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + assert.Equal(t, notGzip, m.GetBody(), + "a body that fails to decompress must be forwarded raw, matching amqp091") + assert.Equal(t, "gzip", m.GetHeaders()[transferEncodingHeaderName], + "the Transfer-Encoding header must be kept when decompression fails, matching amqp091") +} + +// TestDeliveryInjectsTraceHeadersWhenTracingEnabled: mirrors amqp091's +// queueSubscribe, which starts (or continues) a span for every delivery and, +// when tracing is enabled, writes its W3C trace context back into the +// consumed message's headers — so a distributed trace shows arke's own +// "received from broker" hop even when the publisher set no trace headers at +// all. tracing.Enabled() is a process-global flag flipped by +// InitTracerProvider, so this test brackets it carefully and restores the +// disabled state before returning, since other tests in this package assert +// exact header sets and would break under a leaked global. +func TestDeliveryInjectsTraceHeadersWhenTracingEnabled(t *testing.T) { + require.NoError(t, os.Setenv(tracing.EnvTelemetryExporter, "stdout")) + require.NoError(t, os.Setenv(tracing.EnvOtelSdkDisabled, "false")) + _, err := tracing.InitTracerProvider() + require.NoError(t, err) + defer func() { + _ = os.Setenv(tracing.EnvOtelSdkDisabled, "true") + _, _ = tracing.InitTracerProvider() + }() + require.True(t, tracing.Enabled()) + + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.traced", Subjects: []string{"e"}} + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.traced.consumer", "events.traced", "e"), out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m")})) + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + assert.NotEmpty(t, m.GetHeaders()[tracing.HeaderTraceParent], + "a consumed message must carry a traceparent when tracing is enabled, matching amqp091's queueSubscribe") + assert.Contains(t, m.GetHeaders()[tracing.HeaderTraceParent], "-", "traceparent must be the W3C dashed format") +} diff --git a/internal/server/server.go b/internal/server/server.go index 9c8ccf1..4d545be 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -344,6 +344,9 @@ consumeLoop: ackerr = &pb.Error{Message: "Uuid not set when acking/nacking"} } else if ackmsg.GetNack() && ackmsg.GetRequeueDelay() > 0 { // delayed retry ackerr = prov.Retry(ctx, source, ackmsg.GetUuid(), ackmsg.GetRequeueDelay()) + if ackerr != nil { + ackerr = prov.Nack(ctx, ackmsg.GetUuid()) + } } else if ackmsg.GetNack() { // Nack // dead letter if enabled, else Nack opts := source.GetOptions() diff --git a/tests/integration/go.mod b/tests/integration/go.mod index e856be6..98cc1e1 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -50,8 +50,8 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/term v0.43.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect diff --git a/tests/integration/go.sum b/tests/integration/go.sum index 1278f48..ad97764 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -127,10 +127,10 @@ golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7 golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 1ee0480..b8fe19a 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -21,6 +21,7 @@ import ( "github.com/google/uuid" pb "github.com/sassoftware/arke/api" + "github.com/sassoftware/arke/test/config" mf "github.com/sassoftware/arke/test/messagefunctions" "github.com/stretchr/testify/assert" "google.golang.org/grpc" @@ -821,13 +822,22 @@ func TestConsumeContinueOffset(t *testing.T) { c.Disconnect(ctx, &pb.Empty{}) expectedMessageCount = 10 - c2 := pb.NewConsumerClient(consumerConnection) + // The second consumer needs its own gRPC connection and its own done + // channel. Arke identifies a client by name and peer address, so a second + // ConsumerClient over the same connection is the *same* Arke connection, + // and the first consumer's deferred Disconnect would tear this one down. + // consumeMessages also signals its own completion on done, so sharing that + // channel lets the first consumer's exit break this loop. + consumerConnection2 := connect() + defer consumerConnection2.Close() + done2 := make(chan bool) + c2 := pb.NewConsumerClient(consumerConnection2) ctx2, cancel2 := context.WithTimeout(context.Background(), 15*time.Second) defer cancel2() defer c2.Disconnect(ctx2, &pb.Empty{}) messages2 := make(chan *pb.Message) - go consumeMessages(consumerConnection, c2, ctx2, messages2, done, clientConnected, source, defaultHandler, t) + go consumeMessages(consumerConnection2, c2, ctx2, messages2, done2, clientConnected, source, defaultHandler, t) <-clientConnected err = mf.ProduceMessagesUnaryWOConnect(pctx, pc, expectedMessageCount, message, false) @@ -839,7 +849,7 @@ func TestConsumeContinueOffset(t *testing.T) { select { case <-messages2: msgCount2++ - case <-done: + case <-done2: breakLoop = true case <-time.After(5 * time.Second): breakLoop = true @@ -3684,7 +3694,7 @@ func Test_SourceStatsNoStream(t *testing.T) { defer c.Disconnect(ctx, &pb.Empty{}) connResp, err := c.Connect(ctx, connConfig) - assert.Nil(t, err) + assert.Nil(t, err, "got an error: %+v", err) assert.NotNil(t, connResp) assert.True(t, connResp.GetSuccess()) @@ -3695,7 +3705,13 @@ func Test_SourceStatsNoStream(t *testing.T) { assert.Nil(t, ssErr) assert.NotNil(t, stats) assert.NotNil(t, stats.Error) - assert.Contains(t, stats.Error.GetMessage(), "request returned a 404") + brokerResponses := map[string]string{ + "amqp091": "request returned a 404", + "natsjs": "stream not found", + } + notFoundResponse := brokerResponses[connConfig.GetProvider()] + + assert.Contains(t, stats.Error.GetMessage(), notFoundResponse) // verified 404 when no created yet, now create it and get stats. // we should get an "Offset not found" error in the stats.Error @@ -3712,7 +3728,7 @@ func Test_SourceStatsNoStream(t *testing.T) { assert.Nil(t, ssErr) assert.NotNil(t, stats) // emsg := stats.Error.GetMessage() - if strings.Contains(stats.GetError().GetMessage(), "request returned a 404") { + if strings.Contains(stats.GetError().GetMessage(), notFoundResponse) { time.Sleep(1 * time.Second) continue } @@ -3721,6 +3737,10 @@ func Test_SourceStatsNoStream(t *testing.T) { } } func TestStreamHeaderReceivedTimeEqualsTimestampInMs(t *testing.T) { + config := config.ConnectionConfigurationFromEnv() + if config.GetProvider() != "amqp091" { + t.Skip("Skipping test because it is only applicable to RabbitMQ broker") + } producerConnection := connect() defer producerConnection.Close() expectedMessageCount := 1