Complete centralized TSO runtime operations

Makefile

@@ -29,35 +29,40 @@ TLA_LIB      := ../lib
 
 .PHONY: tla-check tla-tools
 
-tla-tools: $(TLA_JAR)
-
-$(TLA_JAR):
+tla-tools:
 	@mkdir -p $(dir $(TLA_JAR))
-	@echo "Downloading tla2tools.jar $(TLA_VERSION)..."
-	@curl -fsSL -o $(TLA_JAR).tmp $(TLA_URL)
-	@# Prefer sha256sum (GNU coreutils, universal on Linux); fall back to
-	@# shasum -a 256 (default on macOS).  Either yields the same hex
-	@# digest in the first whitespace-delimited field.
-	@if command -v sha256sum >/dev/null 2>&1; then \
-		actual=$$(sha256sum $(TLA_JAR).tmp | awk '{print $$1}'); \
-	elif command -v shasum >/dev/null 2>&1; then \
-		actual=$$(shasum -a 256 $(TLA_JAR).tmp | awk '{print $$1}'); \
-	else \
-		echo "ERROR: neither sha256sum nor shasum is available."; \
-		rm -f $(TLA_JAR).tmp; \
-		exit 1; \
+	@set -eu; \
+	sha256_file() { \
+		if command -v sha256sum >/dev/null 2>&1; then \
+			sha256sum "$$1" | awk '{print $$1}'; \
+		elif command -v shasum >/dev/null 2>&1; then \
+			shasum -a 256 "$$1" | awk '{print $$1}'; \
+		else \
+			echo "ERROR: neither sha256sum nor shasum is available." >&2; \
+			return 1; \
+		fi; \
+	}; \
+	actual=""; \
+	if [ -f "$(TLA_JAR)" ]; then actual=$$(sha256_file "$(TLA_JAR)"); fi; \
+	if [ "$$actual" = "$(TLA_SHA256)" ]; then \
+		echo "tla2tools.jar ready at $(TLA_JAR) (SHA-256 $(TLA_SHA256))"; \
+		exit 0; \
 	fi; \
+	echo "Downloading tla2tools.jar $(TLA_VERSION)..."; \
+	trap 'rm -f "$(TLA_JAR).tmp"' EXIT HUP INT TERM; \
+	curl -fsSL -o "$(TLA_JAR).tmp" "$(TLA_URL)"; \
+	actual=$$(sha256_file "$(TLA_JAR).tmp"); \
 	if [ "$$actual" != "$(TLA_SHA256)" ]; then \
 		echo "ERROR: tla2tools.jar SHA-256 mismatch."; \
 		echo "  expected: $(TLA_SHA256)"; \
 		echo "  actual:   $$actual"; \
-		rm -f $(TLA_JAR).tmp; \
 		exit 1; \
-	fi
-	@mv $(TLA_JAR).tmp $(TLA_JAR)
-	@echo "tla2tools.jar ready at $(TLA_JAR) (SHA-256 $(TLA_SHA256))"
+	fi; \
+	mv "$(TLA_JAR).tmp" "$(TLA_JAR)"; \
+	trap - EXIT HUP INT TERM; \
+	echo "tla2tools.jar ready at $(TLA_JAR) (SHA-256 $(TLA_SHA256))"
 
-tla-check: $(TLA_JAR)
+tla-check: tla-tools
 	@# Per-module orchestration lives in scripts/tla-check.sh so adding
 	@# M3..M5 only needs an entry in that script's TLA_MODULES array
 	@# and a `case` line for the gap-invariant string.  The script does

adapter/internal.go

@@ -101,11 +101,17 @@ func (i *Internal) stampTimestamps(ctx context.Context, req *pb.ForwardRequest)
 func (i *Internal) nextTimestamp(ctx context.Context, label string) (uint64, error) {
 	if i.tsAllocator != nil {
 		ts, err := i.tsAllocator.Next(ctx)
-		if err != nil {
+		if err == nil {
+			return ts, nil
+		}
+		if !errors.Is(err, kv.ErrTSOAllocatorRequired) {
 			return 0, errors.Wrap(err, label)
 		}
-		return ts, nil
 	}
+	return i.nextLegacyTimestamp(label)
+}
+
+func (i *Internal) nextLegacyTimestamp(label string) (uint64, error) {
 	if i.clock == nil {
 		return 1, nil
 	}
@@ -121,15 +127,23 @@ func (i *Internal) nextTimestampAfter(ctx context.Context, min uint64, label str
 		return 0, errors.Wrap(kv.ErrTxnCommitTSRequired, label)
 	}
 	if i.tsAllocator != nil {
-		return i.nextTimestampAfterFromAllocator(ctx, min, label)
+		ts, err := i.nextTimestampAfterFromAllocator(ctx, min, label)
+		if err == nil {
+			return ts, nil
+		}
+		if !errors.Is(err, kv.ErrTSOAllocatorRequired) {
+			return 0, err
+		}
 	}
+	return i.nextLegacyTimestampAfter(min, label)
+}
+
+func (i *Internal) nextLegacyTimestampAfter(min uint64, label string) (uint64, error) {
 	if i.clock == nil {
 		return min + 1, nil
 	}
-	if i.clock != nil {
-		i.clock.Observe(min)
-	}
-	ts, err := i.nextTimestamp(ctx, label)
+	i.clock.Observe(min)
+	ts, err := i.nextLegacyTimestamp(label)
 	if err != nil {
 		return 0, err
 	}

adapter/internal_test.go

@@ -4,9 +4,11 @@ import (
 	"context"
 	"encoding/binary"
 	"testing"
+	"time"
 
 	"github.com/bootjp/elastickv/kv"
 	pb "github.com/bootjp/elastickv/proto"
+	"github.com/cockroachdb/errors"
 	"github.com/stretchr/testify/require"
 )
 
@@ -83,6 +85,35 @@ func TestFillForwardedTxnCommitTS_AssignsCommitTS(t *testing.T) {
 	require.Equal(t, meta.CommitTS, commitTS)
 }
 
+func TestFillForwardedTxnCommitTS_FallsBackWhenRuntimeAllocatorLegacy(t *testing.T) {
+	t.Parallel()
+
+	clock := kv.NewHLC()
+	clock.SetPhysicalCeiling(time.Now().Add(time.Minute).UnixMilli())
+	i := &Internal{
+		clock:       clock,
+		tsAllocator: internalLegacyRuntimeAllocator{},
+	}
+	startTS := uint64(10)
+	reqs := []*pb.Request{
+		{
+			IsTxn: true,
+			Phase: pb.Phase_COMMIT,
+			Mutations: []*pb.Mutation{
+				{
+					Op:    pb.Op_PUT,
+					Key:   []byte(kv.TxnMetaPrefix),
+					Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("k"), CommitTS: 0}),
+				},
+			},
+		},
+	}
+
+	commitTS, err := i.fillForwardedTxnCommitTS(context.Background(), reqs, startTS)
+	require.NoError(t, err)
+	require.Greater(t, commitTS, startTS)
+}
+
 func TestFillForwardedTxnCommitTS_PreservesExistingCommitTS(t *testing.T) {
 	t.Parallel()
 
@@ -175,6 +206,21 @@ func TestFillForwardedTxnCommitTS_StampsCommitTSValueOffset(t *testing.T) {
 	require.Zero(t, reqs[0].Mutations[1].CommitTsValueOffset)
 }
 
+func TestStampRawTimestamps_FallsBackWhenRuntimeAllocatorLegacy(t *testing.T) {
+	t.Parallel()
+
+	clock := kv.NewHLC()
+	clock.SetPhysicalCeiling(time.Now().Add(time.Minute).UnixMilli())
+	i := &Internal{
+		clock:       clock,
+		tsAllocator: internalLegacyRuntimeAllocator{},
+	}
+	reqs := []*pb.Request{{Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v")}}}}
+
+	require.NoError(t, i.stampRawTimestamps(context.Background(), reqs))
+	require.NotZero(t, reqs[0].Ts)
+}
+
 func TestFillForwardedTxnCommitTS_PrepareAllowsAlreadyStampedOffsets(t *testing.T) {
 	t.Parallel()
 
@@ -241,3 +287,9 @@ func TestStampTxnTimestamps_UsesSingleTxnStartTS(t *testing.T) {
 	require.Greater(t, meta.CommitTS, uint64(9))
 	require.Equal(t, meta.CommitTS, commitTS)
 }
+
+type internalLegacyRuntimeAllocator struct{}
+
+func (internalLegacyRuntimeAllocator) Next(context.Context) (uint64, error) {
+	return 0, errors.WithStack(kv.ErrTSOAllocatorRequired)
+}

docs/centralized_tso_operations.md

@@ -0,0 +1,115 @@
+# Centralized TSO Operations
+
+This runbook covers runtime mode changes, production signals, and the rollout
+gates for the dedicated group-0 timestamp oracle. The implementation design is
+in `docs/design/2026_04_16_implemented_centralized_tso.md`.
+
+## Runtime configuration
+
+Start every node with the same atomically replaceable mode file:
+
+```text
+--tsoModeFile=/etc/elastickv/tso-mode
+--tsoModeReloadInterval=5s
+--tsoBatchSize=256
+```
+
+The file contains exactly one mode: `legacy`, `shadow`, `cutover`, or
+`phase-d`. `--tsoModeFile` cannot be combined with `--tsoEnabled`,
+`--tsoShadowEnabled`, or `--tsoPhaseDEnabled`; those startup-only flags remain
+for backward compatibility. Runtime mode reload requires the dedicated group 0.
+
+Replace the file atomically so a poll cannot observe a truncated value:
+
+```sh
+printf '%s\n' shadow > /etc/elastickv/tso-mode.tmp
+mv /etc/elastickv/tso-mode.tmp /etc/elastickv/tso-mode
+```
+
+An unreadable file, unknown value, backward transition, or skipped phase is
+rejected without changing the live allocator. Runtime transitions must be
+adjacent and one-way:
+
+```text
+legacy -> shadow -> cutover -> phase-d
+```
+
+The durable group-0 markers override stale local configuration. Once cutover
+or Phase D is committed, a node cannot return to legacy or shadow issuance even
+if its mode file moves backward. An allocation already in flight during a
+reload may finish under the preceding process mode, but cutover-side requests
+still use group 0 and the durable markers never move backward.
+
+## Rollout gates
+
+1. Deploy the binary and group 0 while every mode file remains `legacy`.
+2. Change every node to `shadow`. Confirm all nodes expose
+   `elastickv_tso_mode{mode="shadow"} == 1`.
+3. Hold shadow mode until `legacy_overlap` remains zero for a full 15-minute
+   window, allocation errors remain below 1 percent, and allocation p99 remains
+   below 50 ms.
+4. Change nodes to `cutover`. The first production reservation commits the
+   one-way cutover marker. A node still in shadow observes that marker and
+   returns the dedicated TSO value.
+5. Confirm every binary supports Phase D and every node is on the dedicated
+   path. Then change nodes to `phase-d`. The first Phase-D reservation commits
+   its floor marker before returning a new window.
+
+After the cutover marker commits, rollback means restoring service around the
+dedicated TSO path; it never means returning to legacy issuance. Phase D has the
+same one-way rule and additionally keeps data-group HLC renewal retired.
+
+## Metrics and alerts
+
+| Signal | Meaning | Gate or threshold |
+|---|---|---|
+| `elastickv_tso_request_duration_seconds` | Local/remote reserve and validation attempt latency, split by outcome | Warning at reserve p99 > 50 ms for 10m; critical at > 200 ms for 5m |
+| `elastickv_tso_shadow_comparisons_total` | Accepted candidates, overlap discards, cutover bypasses, and errors | `legacy_overlap` must be zero for 15m before cutover |
+| `elastickv_tso_shadow_divergence_timestamps` | Absolute candidate-to-floor distance for accepted and discarded shadow comparisons | Investigate sustained growth before cutover |
+| `elastickv_tso_mode` | One-hot process-local mode | More than one mode for 15m warns about a stalled rollout |
+| `elastickv_tso_mode_reload_total` | Applied and rejected reloads plus file read/parse failures | Any failure in 10m warns |
+| `elastickv_tso_durable_state` | Applied `cutover` and `phase_d` markers | Phase D without cutover is critical |
+
+The checked rules are in `monitoring/prometheus/rules/tso-alerts.yml`. Allocation
+errors warn above 1 percent for 10 minutes and become critical above 10 percent
+for 5 minutes. The expressions require at least 0.1 reserve attempts/second so
+an idle cluster does not alert on an empty denominator.
+
+## Write-fanout benchmark
+
+Run the modeled remote-TSO benchmark with:
+
+```sh
+go test ./kv -run '^$' -bench '^BenchmarkTSOWriteFanout$' -benchmem -benchtime=1s -count=3
+```
+
+The harness uses 16 concurrent write coordinators sharing a `BatchAllocator`.
+Each operation stamps 1, 3, or 8 fan-out legs, and each refill pays a modeled
+1 ms remote-leader plus Raft-commit delay. The following medians were measured
+on Go 1.26.5, darwin/arm64, Apple M1 Max:
+
+| Fan-out | Batch | Wall time/op | p99 | Refills/op | Timestamp writes/s |
+|---:|---:|---:|---:|---:|---:|
+| 1 | 1 | 1.33 ms | 1,249 ms | 1.000 | 753 |
+| 1 | 64 | 24.81 us | 1.323 ms | 0.01563 | 40,309 |
+| 1 | 256 | 8.92 us | 0.000292 ms | 0.00391 | 112,081 |
+| 3 | 1 | 5.70 ms | 1,233 ms | 3.000 | 526 |
+| 3 | 64 | 105.35 us | 3.212 ms | 0.04690 | 28,478 |
+| 3 | 256 | 17.30 us | 1.272 ms | 0.01172 | 173,453 |
+| 8 | 1 | 10.43 ms | 1,104 ms | 8.000 | 767 |
+| 8 | 64 | 157.49 us | 1.316 ms | 0.1251 | 50,796 |
+| 8 | 256 | 39.43 us | 1.302 ms | 0.03127 | 202,870 |
+
+Batch size 1 is intentionally retained as the unamortized baseline and shows
+severe waiter tails under contention. The production default of 256 keeps the
+modeled fan-out p99 below 4 ms in this harness, including the refill-contention
+tail at fan-out 8, and below the 50 ms warning threshold. These are controlled
+local measurements, not a substitute for observing the production histograms
+during rollout.
+
+## Intentional non-goals
+
+This closure does not automate cluster-wide phase orchestration, choose a
+dedicated subset of TSO members, or change the HLC ceiling formula. Operators
+still advance the shared mode file deployment-by-deployment, and the existing
+group membership and clock-floor decisions remain independent future work.