From fa93cb17a6a499529c4ef7c076e3d5f8e9ad7284 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 25 Jun 2026 15:16:17 -0400 Subject: [PATCH 01/55] =?UTF-8?q?streaming(fullhistory):=20Phase=202=20lay?= =?UTF-8?q?er=201=20=E2=80=94=20hot=20store=20+=20lifecycle=20(#816)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased onto the reworked Phase 1 (#819), which flattened the streaming package into the fullhistory root + a backfill subpackage and made the cold pipeline source-blind (ingest.WriteColdChunk over a ledgerbackend.LedgerStream). The hot tier + lifecycle machinery is re-carved onto that layout: - hot key schema -> geometry (HotState/HotReady/HotTransient, HotChunkKey/ ParseHotChunkKey); hot catalog methods -> catalog (HotState, PutHotTransient, FlipHotReady, DeleteHotKey, {Ready,}HotChunkKeys) - HotProbe/HotChunk + ErrHotVolumeLost live in package backfill (consumed by backfillSource's hot branch, which now yields a ledgerbackend.LedgerStream + a closer); ProcessConfig gains a nil-safe HotProbe field - the freeze source opens the hot DB READ-ONLY (the design's openRocksDBReadOnly): rocksdb.Config gains a ReadOnly mode (read-only open, no create, no flush-on-close) and hotchunk.OpenReadOnly composes the facades over it, so a freeze never mutates the hot DB it reads - the per-chunk hotchunk DB (single shared multi-CF: ledgers + events + tx-hash; transient/ready state machine) + ingest.HotService rebuilt over hotchunk.DB (one atomic synced WriteBatch, decision (a) — no errgroup fan-out) - progress hot refinement: lastCommittedLedger(cat[, probe]) maxes a hot term (positional or one refineWithHotDB read), loss detected lazily on the open - freeze/discard/prune + progress + retention live in a new fullhistory/lifecycle/ subpackage (the scoped part of #824); the freeze stage reuses backfill.RunBackfill (the same path catch-up uses); the live ingestion loop stays in the daemon root - daemon wires the cold-only catch-up's HotProbe (NewRocksHotProbe) - folded-in cleanups: deleted the now-dead RunHot/HotStores orchestration AND the per-type hot ingesters it was the last caller of (the hotchunk.DB HotService supersedes them); aggressively trimmed the phase-2 doc comments Verification: go build + go vet + go test -short green on ./cmd/stellar-rpc/internal/fullhistory/... (cgo RocksDB toolchain). Stack: streaming-phase2-lifecycle -> streaming-phase1-daemon --- .../fullhistory/backfill/hot_fakes_test.go | 53 ++ .../internal/fullhistory/backfill/process.go | 120 +++- .../fullhistory/backfill/process_test.go | 17 +- .../fullhistory/backfill/recorder_test.go | 5 + .../internal/fullhistory/catalog/artifacts.go | 5 +- .../internal/fullhistory/catalog/catalog.go | 120 ++-- .../fullhistory/catalog/catalog_protocol.go | 99 +-- .../fullhistory/catalog/catalog_test.go | 23 +- .../internal/fullhistory/daemon.go | 5 +- .../internal/fullhistory/geometry/keys.go | 73 +- .../fullhistory/geometry/keys_test.go | 9 + .../internal/fullhistory/geometry/paths.go | 82 +-- .../internal/fullhistory/helpers_test.go | 20 +- .../internal/fullhistory/hotsource.go | 134 ++++ .../internal/fullhistory/ingest.go | 236 +++++++ .../internal/fullhistory/ingest/doc.go | 93 +-- .../internal/fullhistory/ingest/driver.go | 182 +---- .../internal/fullhistory/ingest/events.go | 47 -- .../fullhistory/ingest/ingest_test.go | 390 +---------- .../internal/fullhistory/ingest/ingester.go | 46 +- .../internal/fullhistory/ingest/ledgers.go | 36 - .../internal/fullhistory/ingest/metrics.go | 32 - .../internal/fullhistory/ingest/service.go | 128 ++-- .../internal/fullhistory/ingest/txhash.go | 45 -- .../internal/fullhistory/ingest_test.go | 369 ++++++++++ .../internal/fullhistory/lifecycle/discard.go | 43 ++ .../fullhistory/lifecycle/discard_test.go | 30 + .../fullhistory/lifecycle/eligibility.go | 177 +++++ .../fullhistory/lifecycle/helpers_test.go | 262 +++++++ .../fullhistory/lifecycle/hot_fakes_test.go | 101 +++ .../fullhistory/lifecycle/lifecycle.go | 271 ++++++++ .../lifecycle/lifecycle_arith_test.go | 118 ++++ .../lifecycle/lifecycle_helpers_test.go | 194 ++++++ .../lifecycle/lifecycle_loop_test.go | 122 ++++ .../fullhistory/lifecycle/lifecycle_test.go | 229 +++++++ .../fullhistory/lifecycle/progress.go | 203 ++++++ .../lifecycle/progress_realdb_test.go | 104 +++ .../lifecycle/progress_shim_test.go | 24 + .../fullhistory/lifecycle/progress_test.go | 328 +++++++++ .../fullhistory/lifecycle/retention.go | 42 ++ .../{ => lifecycle}/retention_test.go | 78 ++- .../observability/observability.go | 118 +++- .../fullhistory/pkg/rocksdb/rocksdb.go | 39 +- .../pkg/stores/eventstore/hot_store.go | 645 ++++++++---------- .../pkg/stores/hotchunk/hotchunk.go | 235 +++++++ .../pkg/stores/hotchunk/hotchunk_test.go | 468 +++++++++++++ .../pkg/stores/ledger/hot_store.go | 156 +++-- .../pkg/stores/txhash/hot_store.go | 120 ++-- .../internal/fullhistory/progress.go | 121 ---- .../internal/fullhistory/progress_test.go | 105 --- .../internal/fullhistory/retention.go | 48 -- .../internal/fullhistory/startup.go | 29 +- 52 files changed, 4951 insertions(+), 1828 deletions(-) create mode 100644 cmd/stellar-rpc/internal/fullhistory/backfill/hot_fakes_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/hotsource.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/ingest.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/ingest_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/discard_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/hot_fakes_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_arith_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_shim_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/retention.go rename cmd/stellar-rpc/internal/fullhistory/{ => lifecycle}/retention_test.go (56%) create mode 100644 cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go delete mode 100644 cmd/stellar-rpc/internal/fullhistory/progress.go delete mode 100644 cmd/stellar-rpc/internal/fullhistory/progress_test.go delete mode 100644 cmd/stellar-rpc/internal/fullhistory/retention.go diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/hot_fakes_test.go b/cmd/stellar-rpc/internal/fullhistory/backfill/hot_fakes_test.go new file mode 100644 index 000000000..f71311836 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/hot_fakes_test.go @@ -0,0 +1,53 @@ +package backfill + +import ( + "sync/atomic" + + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// fakeHotChunk is a test HotChunk: a hand-set MaxCommittedSeq + an injectable +// LedgerStream source, counting closes when closedTo is non-nil. +type fakeHotChunk struct { + maxSeq uint32 + present bool + maxErr error + source ledgerbackend.LedgerStream + closedTo *atomic.Int32 +} + +func (h *fakeHotChunk) MaxCommittedSeq() (uint32, bool, error) { + return h.maxSeq, h.present, h.maxErr +} +func (h *fakeHotChunk) Source() ledgerbackend.LedgerStream { return h.source } +func (h *fakeHotChunk) Close() error { + if h.closedTo != nil { + h.closedTo.Add(1) + } + return nil +} + +// fakeHotProbe is a test HotProbe: returns its fake chunk when ok, an error when +// openErr is set, or (nil,false,nil) for "no ready hot DB". Counts opens via +// openedTo when non-nil. +type fakeHotProbe struct { + chunk *fakeHotChunk + ok bool + openErr error + openedTo *atomic.Int32 +} + +func (p *fakeHotProbe) OpenHotChunk(chunk.ID) (HotChunk, bool, error) { + if p.openedTo != nil { + p.openedTo.Add(1) + } + if p.openErr != nil { + return nil, false, p.openErr + } + if !p.ok { + return nil, false, nil + } + return p.chunk, true, nil +} diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go index d688737bc..8e0465f02 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go @@ -23,12 +23,45 @@ import ( // ErrBackendCoverageTimeout is returned when the bulk backend's tip never reaches the chunk in time. var ErrBackendCoverageTimeout = errors.New("backend never covered chunk within deadline") +// ErrHotVolumeLost is the case-4 fatal: a "ready" hot:chunk key whose DB is +// missing/unopenable — unrecoverable loss (the hot DB is the sole copy of the +// chunk's recently-ingested ledgers), never auto-healed. Detected LAZILY on the +// open that needs the DB. A sentinel so the daemon's top loop owns the fatal. +var ErrHotVolumeLost = errors.New("hot storage lost; run surgical recovery (case 4)") + +// HotProbe answers backfillSource's hot branch: is the hot tier COMPLETE for this +// chunk (decision (a): maxCommittedSeq >= last ledger), and if so hand back a +// LedgerStream over its ledgers CF so the just-closed chunk freezes without a +// refetch. Injected: production wires NewRocksHotProbe, tests pass a fake. +type HotProbe interface { + // OpenHotChunk borrows the chunk's hot DB for a freeze. ok==false / error under + // a "ready" key (absent or unopenable dir) is case-4 loss. + OpenHotChunk(chunkID chunk.ID) (HotChunk, bool, error) +} + +// HotChunk is one chunk's opened hot tier: the single DB's completeness gate plus +// an LCM source over the ledgers CF. +type HotChunk interface { + // MaxCommittedSeq is the single authoritative watermark (decision (a)); + // ok=false on an empty DB (so the chunk cannot be complete). + MaxCommittedSeq() (seq uint32, ok bool, err error) + // Source yields the chunk's LCMs from the ledgers CF as a LedgerStream the cold + // writer (WriteColdChunk) drains. + Source() ledgerbackend.LedgerStream + // Close releases the shared hot DB. + Close() error +} + // ProcessConfig is what processChunk/backfillSource need for a freeze pass. type ProcessConfig struct { Catalog *catalog.Catalog Logger *supportlog.Entry Sink ingest.MetricSink + // HotProbe opens the hot tier for backfillSource's hot branch. Nil (cold-only + // catch-up or a hot-less test) skips that branch — pack/backend sources only. + HotProbe HotProbe + // Backend is the bulk source for a chunk with no local copy (BSB now, captive // core later — see the Backend interface). It carries its own frontier Tip, so // the coverage wait needs no separate waiter. May be nil for frontfill-only; @@ -84,11 +117,12 @@ func processChunk(ctx context.Context, chunkID chunk.ID, artifacts catalog.Artif // Choose the source before marking "freezing": a source error (a missing pack // or a coverage timeout) must not leave "freezing" debris for a chunk we then - // refuse to produce. - src, err := backfillSource(ctx, chunkID, artifacts, cfg) + // refuse to produce. closeSource releases any opened hot DB after the pass. + src, closeSource, err := backfillSource(ctx, chunkID, artifacts, cfg) if err != nil { return err } + defer func() { _ = closeSource() }() // The one-write protocol, straight-line (see catalog_protocol.go header). The // // one-write: labels keep the four steps greppable without a wrapper. @@ -130,37 +164,62 @@ func processChunk(ctx context.Context, chunkID chunk.ID, artifacts catalog.Artif return nil } -// backfillSource picks a chunk's ledger source as a bare ledgerbackend.LedgerStream: -// 1. the frozen local .pack, unless ledgers is itself requested (circular); -// 2. the bulk backend (cfg.Backend), gated by a bounded waitForCoverage on its Tip. -// -// The local pack needs no coverage wait (it is complete) and no close (its reader -// is opened and closed per RawLedgers call). The bulk backend is caller-owned (the -// daemon Closes it), so backfillSource returns no closer either. +// backfillSource picks a chunk's ledger source (+ a closer for an opened hot DB; +// no-op otherwise), in preference order: +// 1. a ready, COMPLETE hot tier (decision (a): maxCommittedSeq >= last ledger) — +// only if a HotProbe is wired; incomplete-but-present is staleness that falls +// through (re-derivation recovers it), LOSS is fatal (ErrHotVolumeLost); +// 2. the frozen local .pack, unless ledgers is itself requested (circular); +// 3. the bulk backend, gated by a bounded waitForCoverage on its Tip. func backfillSource( ctx context.Context, chunkID chunk.ID, artifacts catalog.ArtifactSet, cfg ProcessConfig, -) (ledgerbackend.LedgerStream, error) { +) (ledgerbackend.LedgerStream, func() error, error) { + noClose := func() error { return nil } cat := cfg.Catalog layout := cat.Layout() + // (1) Hot branch: only when a HotProbe is wired and the hot key is "ready". A + // "transient" key (mid-op or recovery-demoted) is not a read source. + if cfg.HotProbe != nil { + hotState, err := cat.HotState(chunkID) + if err != nil { + return nil, noClose, fmt.Errorf("read hot state chunk %s: %w", chunkID, err) + } + if hotState == geometry.HotReady { + src, closer, used, herr := tryHotSource(chunkID, cfg) + if herr != nil { + return nil, noClose, herr // case-4 loss is fatal + } + if used { + cfg.Logger.Debugf("backfillSource: chunk %s from complete hot tier", chunkID) + return src, closer, nil + } + // Present but incomplete: legitimate staleness — fall through. + cfg.Logger.Debugf("backfillSource: chunk %s hot tier present but incomplete; falling through", chunkID) + } + } + + // (2) Frozen local .pack, only when ledgers is not requested (producing ledgers + // from the pack we'd write would be circular). ledgersState, err := cat.State(chunkID, geometry.KindLedgers) if err != nil { - return nil, fmt.Errorf("read ledgers state chunk %s: %w", chunkID, err) + return nil, noClose, fmt.Errorf("read ledgers state chunk %s: %w", chunkID, err) } if ledgersState == geometry.StateFrozen && !artifacts.Has(geometry.KindLedgers) { packPath := layout.LedgerPackPath(chunkID) if _, serr := os.Stat(packPath); serr == nil { cfg.Logger.Debugf("backfillSource: chunk %s re-derived from frozen .pack", chunkID) - return ledger.NewPackStream(packPath), nil + return ledger.NewPackStream(packPath), noClose, nil } // frozen ⇒ file exists; a missing pack is a bug, not a re-download trigger. - return nil, fmt.Errorf( + return nil, noClose, fmt.Errorf( "chunk %s ledgers is %q but pack file is missing at %s", chunkID, geometry.StateFrozen, packPath) } + // (3) Bulk backend — the only source for a chunk with no local copy. if cfg.Backend == nil { - return nil, fmt.Errorf( + return nil, noClose, fmt.Errorf( "chunk %s has no local copy and no bulk backend is configured", chunkID) } // The coverage wait is mandatory before reading the bulk backend: the freeze @@ -169,8 +228,37 @@ func backfillSource( if werr := waitForCoverage( ctx, cfg.Backend, chunkID.LastLedger(), defaultCoveragePollInterval, defaultCoverageTimeout, ); werr != nil { - return nil, werr + return nil, noClose, werr } cfg.Logger.Debugf("backfillSource: chunk %s from bulk backend", chunkID) - return cfg.Backend, nil + return cfg.Backend, noClose, nil +} + +// tryHotSource handles the hot branch under a "ready" key: used=true when present +// AND complete; used=false when present-but-incomplete (staleness, caller falls +// through); err only for case-4 LOSS (ErrHotVolumeLost), detected lazily on the open. +func tryHotSource(chunkID chunk.ID, cfg ProcessConfig) (ledgerbackend.LedgerStream, func() error, bool, error) { + hot, ok, err := cfg.HotProbe.OpenHotChunk(chunkID) + if err != nil { + // "ready" key but the DB cannot be opened — hot-volume loss. + return nil, nil, false, fmt.Errorf("%w: chunk %s: %w", ErrHotVolumeLost, chunkID, err) + } + if !ok { + // "ready" key but the dir is absent — hot-volume loss. + return nil, nil, false, fmt.Errorf("%w: chunk %s: hot directory absent", ErrHotVolumeLost, chunkID) + } + maxSeq, present, merr := hot.MaxCommittedSeq() + if merr != nil { + _ = hot.Close() + // A read error against an opened DB is loss, not staleness: the DB opened + // but cannot answer its own progress. + return nil, nil, false, fmt.Errorf("%w: chunk %s: max committed seq: %w", ErrHotVolumeLost, chunkID, merr) + } + // decision (a): complete iff the single DB's maxCommittedSeq reaches the chunk's + // last ledger. An empty DB (present==false) cannot be complete. + if present && maxSeq >= chunkID.LastLedger() { + return hot.Source(), hot.Close, true, nil + } + _ = hot.Close() + return nil, nil, false, nil } diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/process_test.go b/cmd/stellar-rpc/internal/fullhistory/backfill/process_test.go index 78cf6540c..c21109249 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/process_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/process_test.go @@ -123,9 +123,10 @@ func zeroTxBackend(t *testing.T) *fakeBackend { func testProcessConfig(t *testing.T, cat *catalog.Catalog) ProcessConfig { t.Helper() return ProcessConfig{ - Catalog: cat, - Logger: silentLogger(), - Sink: ingest.NopSink{}, + Catalog: cat, + Logger: silentLogger(), + Sink: ingest.NopSink{}, + HotProbe: &fakeHotProbe{}, // not "ready" by default; tests override } } @@ -330,8 +331,9 @@ func TestBackfillSource_PrefersFrozenPackWhenLFSNotRequested(t *testing.T) { cfg.Backend = bulk set := catalog.NewArtifactSet(geometry.KindEvents, geometry.KindTxHash) // ledgers NOT requested - src, err := backfillSource(context.Background(), chunkID, set, cfg) + src, closeSrc, err := backfillSource(context.Background(), chunkID, set, cfg) require.NoError(t, err) + defer func() { require.NoError(t, closeSrc()) }() // It is a pack stream (re-derivation without download); the bulk backend was // not consulted. require.IsType(t, ledger.NewPackStream(""), src) @@ -354,8 +356,9 @@ func TestBackfillSource_DoesNotUsePackWhenLFSRequested(t *testing.T) { // ledgers IS requested — the pack branch is skipped (circular), so it goes to // the bulk backend (whose tip covers the chunk, so the wait passes). - src, err := backfillSource(context.Background(), chunkID, catalog.AllArtifacts(), cfg) + src, closeSrc, err := backfillSource(context.Background(), chunkID, catalog.AllArtifacts(), cfg) require.NoError(t, err) + defer func() { require.NoError(t, closeSrc()) }() require.Same(t, bulk, src) } @@ -369,7 +372,7 @@ func TestBackfillSource_BulkCoverageErrorAborts(t *testing.T) { chunkID := chunk.ID(0) cfg.Backend = &fakeBackend{t: t, gen: zeroTxLCMBytes, tipErr: errors.New("boom")} - _, err := backfillSource(context.Background(), chunkID, catalog.AllArtifacts(), cfg) + _, _, err := backfillSource(context.Background(), chunkID, catalog.AllArtifacts(), cfg) require.Error(t, err) require.Contains(t, err.Error(), "backend tip query") } @@ -379,7 +382,7 @@ func TestBackfillSource_NoBackendConfigured(t *testing.T) { cfg := testProcessConfig(t, cat) cfg.Backend = nil - _, err := backfillSource(context.Background(), chunk.ID(0), catalog.AllArtifacts(), cfg) + _, _, err := backfillSource(context.Background(), chunk.ID(0), catalog.AllArtifacts(), cfg) require.Error(t, err) require.Contains(t, err.Error(), "no bulk backend") } diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go b/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go index 23d440123..0a194dd2a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go @@ -42,6 +42,11 @@ func (r *recordingMetrics) Prune(count int, d time.Duration) { } func (*recordingMetrics) LastCommitted(uint32, uint32) {} +func (*recordingMetrics) LedgerCommitted(uint32) {} +func (*recordingMetrics) ChunkBoundary(uint32) {} func (*recordingMetrics) BackfillPass(time.Duration) {} +func (*recordingMetrics) LiveHotChunks(int) {} +func (*recordingMetrics) ColdTierBytes(int64) {} +func (*recordingMetrics) Discard(int, time.Duration) {} var _ observability.Metrics = (*recordingMetrics)(nil) diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/artifacts.go b/cmd/stellar-rpc/internal/fullhistory/catalog/artifacts.go index 9d8b89681..f63d5608a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/artifacts.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/artifacts.go @@ -12,8 +12,9 @@ import ( // processChunk narrows it by dropping already-frozen kinds (per-kind // idempotency). // -// Backed by a fixed-width bitmask over allKinds' canonical order, so Kinds() -// yields that order (matching buildColdIngesters) and membership is alloc-free. +// The representation is a fixed-width bitmask over allKinds' canonical order, so +// Kinds() yields kinds in that order (the canonical ledgers→txhash→events order +// the cold ingesters build in) and membership tests are allocation-free. type ArtifactSet struct { mask uint8 } diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go index cd63ac64c..6df51bbaf 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go @@ -3,36 +3,30 @@ package catalog import ( "errors" "fmt" + "slices" "strconv" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/metastore" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" ) // Catalog is the streaming daemon's view of durable state. It WRAPS -// metastore.Store — the merged RocksDB KV store with sync Put/Delete, atomic -// Batch, and PrefixScan — never reaching around it to RocksDB directly. On top -// of the geometry package (the key schema + its bijection to disk paths, the -// tx-hash-index arithmetic, and the fsync helpers) it adds the one-write -// protocol (catalog_protocol.go) and the key-driven sweeps (catalog_sweep.go). -// -// Every key names a file/dir state or a config pin; progress is derived, never -// stored. -// -// The read-then-act sequences in the write protocol and the sweeps carry no -// concurrency guard: the design's Concurrency model guarantees one writer per -// key (see the header note in catalog_protocol.go). +// metastore.Store (never reaching around it to RocksDB) and, on top of geometry +// (key schema, key↔path bijection, index arithmetic, fsync helpers), adds the +// one-write protocol (catalog_protocol.go) and the key-driven sweeps +// (catalog_sweep.go). Every key names a file/dir state or a config pin; progress +// is derived, never stored. The read-then-act sequences carry no concurrency +// guard — the design guarantees one writer per key (see catalog_protocol.go). type Catalog struct { store *metastore.Store layout geometry.Layout txhashIndex geometry.TxHashIndexLayout } -// NewCatalog binds a catalog to an open metastore.Store, the on-disk layout, -// and the tx-hash-index arithmetic. The store is caller-owned; the catalog never -// closes it. +// NewCatalog binds a catalog to an open (caller-owned) metastore.Store, the +// layout, and the index arithmetic. The catalog never closes the store. func NewCatalog(store *metastore.Store, layout geometry.Layout, txhashIndex geometry.TxHashIndexLayout) *Catalog { return &Catalog{store: store, layout: layout, txhashIndex: txhashIndex} } @@ -41,12 +35,10 @@ func (c *Catalog) Layout() geometry.Layout { return c.layout } func (c *Catalog) TxHashIndexLayout() geometry.TxHashIndexLayout { return c.txhashIndex } -// --------------------------------------------------------------------------- -// Typed artifact-state accessors. -// --------------------------------------------------------------------------- +// --- Typed artifact-state accessors --- // State returns the lifecycle State of a per-chunk artifact key, or the empty -// State when the key is absent — neither file nor in-progress write exists. +// State when the key is absent. func (c *Catalog) State(chunkID chunk.ID, kind geometry.Kind) (geometry.State, error) { v, ok, err := c.get(geometry.ChunkKey(chunkID, kind)) if err != nil || !ok { @@ -55,11 +47,19 @@ func (c *Catalog) State(chunkID chunk.ID, kind geometry.Kind) (geometry.State, e return geometry.State(v), nil } -// --------------------------------------------------------------------------- -// Scans. Every "find work" operation iterates keys via PrefixScan; nothing -// lists a directory. Results are returned sorted so callers need no second -// pass. -// --------------------------------------------------------------------------- +// HotState returns the HotState of a chunk's hot-DB key, or empty (key absent). +// The key's mere existence (any value) marks the chunk as owned by ingestion; +// only the watermark derivation cares which value (see ReadyHotChunkKeys). +func (c *Catalog) HotState(chunkID chunk.ID) (geometry.HotState, error) { + v, ok, err := c.get(geometry.HotChunkKey(chunkID)) + if err != nil || !ok { + return "", err + } + return geometry.HotState(v), nil +} + +// --- Scans. Every "find work" iterates keys via PrefixScan (never lists a +// directory); results are returned sorted so callers need no second pass. --- // ChunkArtifactKeys returns every per-chunk artifact key with its value, sorted // by key — the deletion/audit surface for chunk:* keys. @@ -84,15 +84,27 @@ func (c *Catalog) TxHashIndexKeys(w geometry.TxHashIndexID) ([]geometry.TxHashIn return c.txhashIndexKeysByPrefix(geometry.TxHashIndexPrefixFor(w)) } +// HotChunkKeys returns every hot-DB chunk id (value-blind), sorted ascending. +// The highest is the live chunk — the ingestion/lifecycle partition boundary. +func (c *Catalog) HotChunkKeys() ([]chunk.ID, error) { + return c.hotChunkKeysWith(nil) +} + +// ReadyHotChunkKeys returns only the chunks whose hot-DB key is "ready", sorted +// ascending. The watermark counts only these — a "transient" key never advances +// the bound, which lets recovery demote any hot key without disturbing it. +func (c *Catalog) ReadyHotChunkKeys() ([]chunk.ID, error) { + return c.hotChunkKeysWith(func(s geometry.HotState) bool { return s == geometry.HotReady }) +} + // AllTxHashIndexKeys is TxHashIndexKeys across all indexes. func (c *Catalog) AllTxHashIndexKeys() ([]geometry.TxHashIndexCoverage, error) { return c.txhashIndexKeysByPrefix(geometry.TxHashIndexPrefix) } -// FrozenTxHashIndex returns the index's UNIQUE "frozen" coverage — the key -// readers resolve as "the index" — or ok=false if the index has none -// yet. It asserts INV-2 (at most one frozen coverage per index at any moment) -// by erroring if it observes two — a detectable bug, not a tie-break to resolve. +// FrozenTxHashIndex returns the index's UNIQUE "frozen" coverage (what readers +// resolve as "the index"), or ok=false if none yet. Asserts INV-2 (at most one +// frozen coverage per index) by erroring on two — a detectable bug, not a tie. func (c *Catalog) FrozenTxHashIndex(w geometry.TxHashIndexID) (geometry.TxHashIndexCoverage, bool, error) { covs, err := c.TxHashIndexKeys(w) if err != nil { @@ -118,27 +130,22 @@ func (c *Catalog) FrozenTxHashIndex(w geometry.TxHashIndexID) (geometry.TxHashIn return frozen, found, nil } -// --------------------------------------------------------------------------- -// Config pins. Written once on first start, immutable thereafter. -// --------------------------------------------------------------------------- +// --- Config pins. Written once on first start, immutable thereafter. --- -// EarliestLedger returns the pinned config:earliest_ledger (chunk-aligned). ok -// is false if the pin has not been written yet (a pristine store). +// EarliestLedger returns the pinned config:earliest_ledger (chunk-aligned). ok is +// false if not yet written (a pristine store). func (c *Catalog) EarliestLedger() (uint32, bool, error) { return c.uint32Pin(geometry.ConfigEarliestLedger) } -// PinEarliestLedger commits the config:earliest_ledger pin in one synced write — -// the first-start commit validateConfig mandates. Its presence is the sentinel -// that a prior first start completed: once written, earliest_ledger is immutable -// and validated-or-abort on every restart. chunks_per_txhash_index is no longer -// pinned — it is the fixed geometry.ChunksPerTxhashIndex constant. +// PinEarliestLedger commits the config:earliest_ledger pin in one synced write. +// Its presence is the sentinel that a prior first start completed; once written +// it is immutable and validated-or-abort on every restart. func (c *Catalog) PinEarliestLedger(earliestLedger uint32) error { return c.store.Put(geometry.ConfigEarliestLedger, strconv.FormatUint(uint64(earliestLedger), 10)) } -// ArtifactRef names one per-chunk artifact and the State observed for it — the -// (chunk, kind, State) unit the sweeps and resolver pass around. +// ArtifactRef is the (chunk, kind, State) unit the sweeps and resolver pass around. type ArtifactRef struct { Chunk chunk.ID Kind geometry.Kind @@ -147,13 +154,10 @@ type ArtifactRef struct { func (r ArtifactRef) Key() string { return geometry.ChunkKey(r.Chunk, r.Kind) } -// --------------------------------------------------------------------------- -// Unexported helpers backing the scans and pin getters above. -// --------------------------------------------------------------------------- +// --- Unexported helpers backing the scans and pin getters above. --- -// get returns the value at key. The bool is false (err nil) on a clean miss, -// distinguishing "absent" from a backing-store error — the value-blind primitive -// the typed reads above build on. +// get returns the value at key; ok is false (err nil) on a clean miss, +// distinguishing "absent" from a backing-store error. func (c *Catalog) get(key string) (string, bool, error) { v, err := c.store.Get(key) if errors.Is(err, stores.ErrNotFound) { @@ -171,6 +175,28 @@ func (c *Catalog) has(key string) (bool, error) { return ok, err } +// hotChunkKeysWith returns the chunks whose hot-DB key matches keep, sorted +// ascending. A nil keep matches every value (value-blind). +func (c *Catalog) hotChunkKeysWith(keep func(geometry.HotState) bool) ([]chunk.ID, error) { + var ids []chunk.ID + for e, err := range c.store.PrefixScan(geometry.HotChunkPrefix) { + if err != nil { + return nil, err + } + id, ok := geometry.ParseHotChunkKey(e.Key) + if !ok { + return nil, fmt.Errorf("streaming: malformed hot key %q", e.Key) + } + if keep == nil || keep(geometry.HotState(e.Value)) { + ids = append(ids, id) + } + } + // PrefixScan yields byte-lex order == numeric under the 8-digit padding, so + // the slice is already ascending; sort defensively against a width change. + slices.Sort(ids) + return ids, nil +} + // txhashIndexKeysByPrefix scans coverage keys under prefix, attaching each scanned // value as State. func (c *Catalog) txhashIndexKeysByPrefix(prefix string) ([]geometry.TxHashIndexCoverage, error) { diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol.go index 31d7e0f86..a3f849a35 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol.go @@ -3,38 +3,32 @@ package catalog import ( "errors" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/metastore" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" ) // The one write protocol — mark-then-write. Every durable artifact (per-chunk // file or index coverage) flows through here: // -// 1. Put the key "freezing" via metastore BEFORE any I/O. -// 2. The caller writes the file. -// 3. The caller fsyncs the FILE + its PARENT dirent (+ the GRANDPARENT dirent -// when the parent dir was just created) — geometry.BarrierNewFile. -// 4. Flip to "frozen": a single Put for per-chunk artifacts, or one atomic -// Batch for the index (see CommitTxHashIndex). +// 1. Put the key "freezing" BEFORE any I/O. +// 2. Caller writes the file. +// 3. Caller fsyncs the file + parent dirent (+ grandparent on a new parent dir) +// — geometry.BarrierNewFile. +// 4. Flip to "frozen" — one Put per-chunk, or one atomic Batch for the index. // // "frozen" is the only transition readers trust. The catalog owns steps 1 and 4 // (meta writes); the caller owns 2 and 3 (I/O). // -// One writer per key. The read-then-act sequences here and in the sweeps -// (catalog_sweep.go) — read a key/coverage, then Put or unlink based on it — are -// deliberately UNGUARDED against a second writer racing the same key, because the -// design's Concurrency model rules that out: the ingest thread and the lifecycle -// thread write the catalog at the same time but NEVER the same key; resolve emits -// exactly one index build per window (so even concurrent backfill never has two -// builds for one index); and only one lifecycle run executes at a time. Crash -// re-runs stay safe not because of any in-method guard but because every step is -// idempotent and resolve re-plans from durable state. See the design's -// "Concurrency model" section. +// One writer per key: the read-then-act sequences here and in the sweeps are +// UNGUARDED against a racing second writer because the design rules it out +// (ingest and lifecycle never write the same key; resolve emits one build per +// window; one lifecycle run at a time). Crash re-runs stay safe because every +// step is idempotent and resolve re-plans from durable state. // MarkChunkFreezing is step 1 for every requested kind. Re-marking a -// "freezing"/"pruning"/absent key is idempotent re-materialization; skipping an -// already-"frozen" kind (per-kind idempotency) is the caller's job. +// freezing/pruning/absent key is idempotent; skipping an already-frozen kind is +// the caller's job. func (c *Catalog) MarkChunkFreezing(chunkID chunk.ID, kinds ...geometry.Kind) error { if len(kinds) == 0 { return errors.New("streaming: MarkChunkFreezing requires at least one kind") @@ -47,9 +41,8 @@ func (c *Catalog) MarkChunkFreezing(chunkID chunk.ID, kinds ...geometry.Kind) er }) } -// FlipChunkFrozen is step 4 for per-chunk artifacts: flips every requested kind -// to "frozen". The caller MUST have completed geometry.BarrierNewFile for every -// file first. +// FlipChunkFrozen is step 4 for per-chunk artifacts: flips every kind to +// "frozen". The caller MUST have completed BarrierNewFile for every file first. func (c *Catalog) FlipChunkFrozen(chunkID chunk.ID, kinds ...geometry.Kind) error { if len(kinds) == 0 { return errors.New("streaming: FlipChunkFrozen requires at least one kind") @@ -62,8 +55,8 @@ func (c *Catalog) FlipChunkFrozen(chunkID chunk.ID, kinds ...geometry.Kind) erro }) } -// MarkTxHashIndexFreezing is step 1 for the index, returning the TxHashIndexCoverage for -// CommitTxHashIndex. lo > hi panics (geometry.TxHashIndexKey enforces it). +// MarkTxHashIndexFreezing is step 1 for the index, returning the coverage for +// CommitTxHashIndex. lo > hi panics (TxHashIndexKey enforces it). func (c *Catalog) MarkTxHashIndexFreezing( w geometry.TxHashIndexID, lo, hi chunk.ID, ) (geometry.TxHashIndexCoverage, error) { @@ -83,31 +76,24 @@ func (c *Catalog) MarkTxHashIndexFreezing( // CommitTxHashIndex is step 4 for the index. In one atomic batch it: // // - promotes cov ("freezing" -> "frozen"); -// - demotes the index's predecessor frozen coverage (if any) to "pruning"; -// - iff this build is terminal (cov.Hi == index's last chunk), demotes the -// chunk:{c}:txhash key of every chunk in cov's [Lo, Hi] range to "pruning". -// -// The batch only DEMOTES keys — file deletion is the sweeps' job. So there is no -// instant with two frozen coverages, no live index unreachable, and no "frozen" -// chunk:c:txhash whose .bin was deleted. +// - demotes the predecessor frozen coverage (if any) to "pruning"; +// - iff terminal (cov.Hi == index's last chunk), demotes every chunk:{c}:txhash +// key in cov's [Lo, Hi] range to "pruning". // -// A re-commit of the already-frozen coverage is an idempotent overwrite — the -// crash-re-run case. There is no guard against an out-of-order or duplicate build -// for the same index: the design's Concurrency model precludes it (resolve emits -// one build per window; one lifecycle run at a time — see the header note). -// -// The caller MUST have fsynced the .idx file and its dir first. The predecessor -// is re-read from durable state, so this is safe to call after a crash. +// The batch only DEMOTES (file deletion is the sweeps' job), so there is never an +// instant with two frozen coverages, an unreachable live index, or a "frozen" +// chunk:c:txhash whose .bin was deleted. A re-commit is an idempotent overwrite +// (crash re-run). The caller MUST have fsynced the .idx and its dir first; the +// predecessor is re-read from durable state, so this is crash-safe. func (c *Catalog) CommitTxHashIndex(cov geometry.TxHashIndexCoverage) error { - // Compose demotions against durable state BEFORE opening the batch, so the - // batch body is a pure sequence of puts. + // Compose demotions against durable state BEFORE the batch, so the batch body + // is a pure sequence of puts. prev, hasPrev, err := c.FrozenTxHashIndex(cov.Index) if err != nil { return err } if hasPrev && prev.Key == cov.Key { - // Re-commit of an already-landed batch: nothing to demote against itself; - // the promote below is an idempotent overwrite. + // Re-commit of an already-landed batch: nothing to demote against itself. hasPrev = false } @@ -132,11 +118,9 @@ func (c *Catalog) CommitTxHashIndex(cov geometry.TxHashIndexCoverage) error { } // txhashIndexChunkKeysPresent returns the chunk:{c}:txhash keys that EXIST in -// the inclusive chunk range [lo, hi]. A terminal commit passes cov's own range, -// so it demotes only the .bin inputs the new .idx actually covers — never a key -// below cov.Lo (whose ledgers the new index cannot answer, and whose .bin must -// survive for its own index's build) and never a chunk whose .bin was never -// produced (the spec's cat.Has guard). +// [lo, hi]. A terminal commit passes cov's own range, so it demotes only the .bin +// inputs the new .idx covers — never a key below cov.Lo (whose .bin must survive +// for its own index) nor a chunk whose .bin was never produced. func (c *Catalog) txhashIndexChunkKeysPresent(lo, hi chunk.ID) ([]string, error) { var keys []string for cid := lo; cid <= hi; cid++ { @@ -151,3 +135,24 @@ func (c *Catalog) txhashIndexChunkKeysPresent(lo, hi chunk.ID) ([]string, error) } return keys, nil } + +// --- Hot-DB key bracket: the file protocol's transient/ready bracket applied to +// the chunk's hot directory. --- + +// PutHotTransient marks a hot-DB key "transient" — the open end, written before +// the dir is created or a discard begins removing it. A crash mid-operation is +// detectable from this value alone. +func (c *Catalog) PutHotTransient(chunkID chunk.ID) error { + return c.store.Put(geometry.HotChunkKey(chunkID), string(geometry.HotTransient)) +} + +// FlipHotReady marks a hot-DB key "ready" (dir exists and usable). The caller +// MUST have fsynced the dir (and its parent on creation) first. +func (c *Catalog) FlipHotReady(chunkID chunk.ID) error { + return c.store.Put(geometry.HotChunkKey(chunkID), string(geometry.HotReady)) +} + +// DeleteHotKey removes a hot-DB key — the close end, after rmdir. Idempotent. +func (c *Catalog) DeleteHotKey(chunkID chunk.ID) error { + return c.store.Delete(geometry.HotChunkKey(chunkID)) +} diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_test.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_test.go index 3f3ccaeab..c9b986eb9 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/require" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) // PinEarliestLedger writes the sole config pin; EarliestLedger reads it back. @@ -27,6 +27,27 @@ func TestConfigPins(t *testing.T) { require.Equal(t, uint32(2), el) } +// --------------------------------------------------------------------------- +// Scans: HotChunkKeys (value-blind) vs ReadyHotChunkKeys (ready-only). +// --------------------------------------------------------------------------- + +func TestHotChunkKeysValueBlindVsReadyOnly(t *testing.T) { + cat, _ := testCatalog(t) + + require.NoError(t, cat.PutHotTransient(3)) + require.NoError(t, cat.FlipHotReady(5)) + require.NoError(t, cat.PutHotTransient(9)) + require.NoError(t, cat.FlipHotReady(12)) + + all, err := cat.HotChunkKeys() + require.NoError(t, err) + require.Equal(t, []chunk.ID{3, 5, 9, 12}, all, "value-blind: every hot key") + + ready, err := cat.ReadyHotChunkKeys() + require.NoError(t, err) + require.Equal(t, []chunk.ID{5, 12}, ready, "ready-only excludes transient") +} + func TestChunkArtifactKeys(t *testing.T) { cat, _ := testCatalog(t) diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index 8b0dbc059..c2d0d1004 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -153,8 +153,9 @@ func startConfig( Workers: deref(cfg.Backfill.Workers), MaxRetries: deref(cfg.Backfill.MaxRetries), Process: backfill.ProcessConfig{ - Backend: backend, - Sink: sink, + Backend: backend, + Sink: sink, + HotProbe: NewRocksHotProbe(cat.Layout().HotChunkPath, logger), }, } return StartConfig{ diff --git a/cmd/stellar-rpc/internal/fullhistory/geometry/keys.go b/cmd/stellar-rpc/internal/fullhistory/geometry/keys.go index ba672301a..fb684d788 100644 --- a/cmd/stellar-rpc/internal/fullhistory/geometry/keys.go +++ b/cmd/stellar-rpc/internal/fullhistory/geometry/keys.go @@ -15,18 +15,28 @@ import ( type State string const ( - // StateFreezing — the immutable file is being written. Set BEFORE any I/O - // (mark-then-write), so a crash mid-write is detectable from the key alone - // and every on-disk file is reachable from a key. + // StateFreezing — file being written. Set BEFORE any I/O (mark-then-write), + // so a crash mid-write is detectable and every file is reachable from a key. StateFreezing State = "freezing" - // StateFrozen — file and dirent are fsynced and durable. Trusted blindly by - // readers, the resolver, and buildTxhashIndex's precondition. + // StateFrozen — file + dirent fsynced and durable. Trusted blindly by readers. StateFrozen State = "frozen" - // StatePruning — file queued for removal, may or may not still be on disk. - // A sweep finishes the unlink, then deletes the key. + // StatePruning — queued for removal. A sweep unlinks, then deletes the key. StatePruning State = "pruning" ) +// HotState is a hot-DB key's value. One key per chunk brackets the chunk's hot +// RocksDB directory; the column families inside carry no individual key. +type HotState string + +const ( + // HotTransient — a dir operation is in flight (create/delete) or recovery + // demoted the key. Recovery is identical either way: open wipes+recreates, + // discard re-runs the scan. + HotTransient HotState = "transient" + // HotReady — the dir exists and is usable. + HotReady HotState = "ready" +) + // Kind is a per-chunk artifact kind. Each maps to one meta-store key suffix // and one set of on-disk files. type Kind string @@ -58,17 +68,15 @@ type TxHashIndexID uint32 // chunk ids, matching the {idx:08d} segment in keys and paths. func (i TxHashIndexID) String() string { return fmt.Sprintf("%08d", uint32(i)) } -// --------------------------------------------------------------------------- -// Key prefixes and constructors — the single source of truth for the -// key<->path bijection (paths.go holds the inverse). -// --------------------------------------------------------------------------- +// --- Key prefixes and constructors — single source of truth for the key↔path +// bijection (paths.go holds the inverse). --- const ( ChunkPrefix = "chunk:" + HotChunkPrefix = "hot:chunk:" TxHashIndexPrefix = "txhash_index:" - // ConfigEarliestLedger is the sole config pin key. (chunks_per_txhash_index is - // the fixed ChunksPerTxhashIndex constant, not a pin.) + // ConfigEarliestLedger is the sole config pin key. ConfigEarliestLedger = "config:earliest_ledger" ) @@ -77,9 +85,15 @@ func ChunkKey(c chunk.ID, kind Kind) string { return ChunkPrefix + c.String() + ":" + string(kind) } -// TxHashIndexKey returns the index coverage key txhash_index:{idx:08d}:{lo:08d}:{hi:08d}. -// The coverage [lo, hi] lives in the key NAME; the value is pure lifecycle -// state. lo > hi is a programmer error, surfaced loudly via panic. +// HotChunkKey returns the hot-DB key hot:chunk:{chunk:08d}. One key per chunk +// brackets the hot RocksDB dir; the value is a HotState. +func HotChunkKey(c chunk.ID) string { + return HotChunkPrefix + c.String() +} + +// TxHashIndexKey returns txhash_index:{idx:08d}:{lo:08d}:{hi:08d}. The coverage +// [lo, hi] lives in the key NAME; the value is pure lifecycle state. lo > hi +// panics (programmer error). func TxHashIndexKey(idx TxHashIndexID, lo, hi chunk.ID) string { if lo > hi { panic(fmt.Sprintf("streaming: TxHashIndexKey lo %s > hi %s", lo, hi)) @@ -87,19 +101,16 @@ func TxHashIndexKey(idx TxHashIndexID, lo, hi chunk.ID) string { return TxHashIndexPrefix + idx.String() + ":" + lo.String() + ":" + hi.String() } -// TxHashIndexPrefixFor returns the scan prefix txhash_index:{idx:08d}: that enumerates -// all coverage keys of one index. +// TxHashIndexPrefixFor returns the scan prefix txhash_index:{idx:08d}: that +// enumerates all coverage keys of one index. func TxHashIndexPrefixFor(idx TxHashIndexID) string { return TxHashIndexPrefix + idx.String() + ":" } -// --------------------------------------------------------------------------- -// Key parsing — each parser is the reverse bijection of exactly one -// constructor above. -// --------------------------------------------------------------------------- +// --- Key parsing — each parser is the reverse bijection of one constructor. --- -// TxHashIndexCoverage is one parsed index coverage key: the index, the covered -// chunk range [Lo, Hi], the full key string, and its lifecycle State. +// TxHashIndexCoverage is one parsed index coverage key: the index, range [Lo, +// Hi], the full key string, and its lifecycle State. type TxHashIndexCoverage struct { Index TxHashIndexID Lo, Hi chunk.ID @@ -129,6 +140,20 @@ func ParseChunkKey(key string) (chunk.ID, Kind, bool) { return chunk.ID(n), kind, true } +// ParseHotChunkKey decodes hot:chunk:{chunk:08d}. ok is false for any key that +// is not a well-formed hot-chunk key. +func ParseHotChunkKey(key string) (chunk.ID, bool) { + rest, found := strings.CutPrefix(key, HotChunkPrefix) + if !found { + return 0, false + } + n, err := ParsePadded(rest) + if err != nil { + return 0, false + } + return chunk.ID(n), true +} + // ParseTxHashIndexKey decodes txhash_index:{idx:08d}:{lo:08d}:{hi:08d}. State is not part // of the key; callers fill TxHashIndexCoverage.State from the scanned value. func ParseTxHashIndexKey(key string) (TxHashIndexCoverage, bool) { diff --git a/cmd/stellar-rpc/internal/fullhistory/geometry/keys_test.go b/cmd/stellar-rpc/internal/fullhistory/geometry/keys_test.go index 17685323a..971c82030 100644 --- a/cmd/stellar-rpc/internal/fullhistory/geometry/keys_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/geometry/keys_test.go @@ -84,3 +84,12 @@ func TestParseRejectsMalformed(t *testing.T) { func TestIndexKeyPanicsOnLoGreaterThanHi(t *testing.T) { require.Panics(t, func() { TxHashIndexKey(5, 5349, 5100) }) } + +func TestHotKeyBijection(t *testing.T) { + for _, id := range []chunk.ID{0, 7, 5350} { + key := HotChunkKey(id) + got, ok := ParseHotChunkKey(key) + require.True(t, ok) + require.Equal(t, id, got) + } +} diff --git a/cmd/stellar-rpc/internal/fullhistory/geometry/paths.go b/cmd/stellar-rpc/internal/fullhistory/geometry/paths.go index 58eb6752b..60c07cef7 100644 --- a/cmd/stellar-rpc/internal/fullhistory/geometry/paths.go +++ b/cmd/stellar-rpc/internal/fullhistory/geometry/paths.go @@ -10,10 +10,10 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash" ) -// Layout is the SINGLE source of truth for storage paths: a fixed key<->path -// bijection (design-docs/full-history-streaming-workflow.md "Directory layout") -// holding one root PER artifact tree, so a Layout plus a key finds any file -// without listing a directory. NewLayout defaults all roots under one data dir: +// Layout is the SINGLE source of truth for storage paths: a fixed key↔path +// bijection holding one root per artifact tree, so a Layout plus a key finds any +// file without listing a directory. NewLayout defaults all roots under one data +// dir: // // {root}/ // ├── catalog/rocksdb/ @@ -24,8 +24,8 @@ import ( // ├── raw/{bucket:05d}/{chunk:08d}.bin // └── index/{idx:08d}/{lo:08d}-{hi:08d}.idx // -// Each root is independently settable (NewLayoutFromRoots) for the [storage] -// path overrides. Bucket ids never appear in meta-store keys. +// Each root is independently settable (NewLayoutFromRoots) for [storage] +// overrides. Bucket ids never appear in meta-store keys. type Layout struct { catalogRoot string // meta-store RocksDB dir (a leaf, not a tree root) hotRoot string @@ -49,10 +49,8 @@ func NewLayout(root string) Layout { } // NewLayoutFromRoots binds a Layout to explicit per-tree roots — the resolved, -// independently-overridable storage paths the daemon flocks and opens. Taking -// strings (rather than the config Paths struct) keeps geometry free of any -// config dependency; the streaming package's NewLayoutFromPaths adapts a Paths -// to this so lock and data location can never disagree. +// overridable storage paths. Taking strings (not the config Paths struct) keeps +// geometry free of a config dependency; NewLayoutFromPaths adapts a Paths to this. func NewLayoutFromRoots(catalogRoot, hotRoot, ledgersRoot, eventsRoot, txhashRawRoot, txhashIndexRoot string) Layout { return Layout{ catalogRoot: catalogRoot, @@ -76,8 +74,7 @@ func (l Layout) HotChunkPath(c chunk.ID) string { } // LedgerPackPath is a chunk's ledger pack. Layout composes the bucket dir; the -// leaf is owned by ledger.PackName (shared with the cold writer and reader). -// EventsPaths/TxHashBinPath follow the same split. +// leaf is owned by ledger.PackName. EventsPaths/TxHashBinPath split the same way. func (l Layout) LedgerPackPath(c chunk.ID) string { return filepath.Join(l.ledgersRoot, c.BucketID(), ledger.PackName(c)) } @@ -104,9 +101,9 @@ func (l Layout) LedgersRoot() string { return l.ledgersRoot } // EventsRoot is the root EventsPaths composes under. func (l Layout) EventsRoot() string { return l.eventsRoot } -// TxHashRawRoot is its own root because the cold pipeline takes an explicit -// per-kind root (ingest.ColdDirs) rather than the single coldDir/ -// layout RunCold derives. +// TxHashRawRoot is the root under which per-chunk raw txhash runs are bucketed +// (matches TxHashBinPath). Its own root because the cold pipeline takes an +// explicit per-kind root (ingest.ColdDirs), not a coldDir/ derivation. func (l Layout) TxHashRawRoot() string { return l.txhashRawRoot } // TxHashIndexRoot is the root TxHashIndexDir composes under. @@ -124,8 +121,8 @@ func (l Layout) TxHashIndexFilePath(cov TxHashIndexCoverage) string { return filepath.Join(l.TxHashIndexDir(cov.Index), name) } -// ArtifactPaths is the single (chunk, kind)->files map, so the sweep and the -// freeze writer agree on what a kind owns on disk. +// ArtifactPaths is the single (chunk, kind)->files map, so the sweep and freeze +// writer agree on what a kind owns on disk. func (l Layout) ArtifactPaths(c chunk.ID, kind Kind) []string { switch kind { case KindLedgers: @@ -139,16 +136,12 @@ func (l Layout) ArtifactPaths(c chunk.ID, kind Kind) []string { } } -// --------------------------------------------------------------------------- -// fsync barriers — the os-level durability primitives the one-write protocol and -// the sweeps depend on. A creation is durable only once both the file's data AND -// the directory entry naming it are fsynced; a freshly created directory needs -// its own parent fsynced too. See the One write protocol section: "the key never -// outlives the file's creation". -// --------------------------------------------------------------------------- +// --- fsync barriers for the one-write protocol and sweeps. A creation is +// durable only once the file's data AND the dirent naming it are fsynced; a +// freshly created dir needs its own parent fsynced too. --- -// syncAndClose fsyncs an open file/dir handle then closes it, preferring the -// sync error over the close error so a durability failure is never masked. +// syncAndClose fsyncs an open handle then closes it, preferring the sync error so +// a durability failure is never masked. func syncAndClose(f *os.File) error { syncErr := f.Sync() closeErr := f.Close() @@ -168,10 +161,9 @@ func fsyncFile(path string) error { return syncAndClose(f) } -// FsyncDir fsyncs a directory entry, making creations and unlinks within it -// durable. A missing directory is not an error: a sweep may run where the file -// (and its on-demand bucket/index dir) was never created, so there is no dirent -// to make durable. +// FsyncDir fsyncs a directory entry, making creations/unlinks within it durable. +// A missing dir is not an error: a sweep may run where the file (and its +// on-demand dir) was never created. func FsyncDir(dir string) error { f, err := os.Open(dir) if os.IsNotExist(err) { @@ -211,10 +203,9 @@ func FsyncParentDirs(paths []string) error { // BarrierNewFile applies the two-level barrier to a freshly written file: fsync // the file, its parent dir, then the grandparent dirent. The grandparent fsync -// persists the parent's own directory entry, which matters when the write just -// created the parent (e.g. a new bucket every 1000th chunk). On an unchanged -// grandparent it has no dirty metadata to flush and is nearly free, so the -// barrier runs it unconditionally rather than tracking whether the parent is new. +// persists the parent's own dirent — load-bearing when the write just created the +// parent (a new bucket every 1000th chunk); on an unchanged grandparent it is +// nearly free, so it runs unconditionally rather than tracking parent-newness. func BarrierNewFile(path string) error { if err := fsyncFile(path); err != nil { return err @@ -226,9 +217,8 @@ func BarrierNewFile(path string) error { return FsyncDir(filepath.Dir(parent)) } -// DeepestExistingDir returns the deepest ancestor of path (path itself when it -// already exists) present on disk, walking up until a stat succeeds. It bounds -// FsyncNewDirs to only the directories a subsequent MkdirAll actually creates. +// DeepestExistingDir returns the deepest on-disk ancestor of path (path itself if +// it exists), bounding FsyncNewDirs to only the dirs a subsequent MkdirAll creates. func DeepestExistingDir(path string) string { for { if _, err := os.Stat(path); err == nil { @@ -242,16 +232,14 @@ func DeepestExistingDir(path string) string { } } -// FsyncNewDirs makes a directory chain freshly produced by MkdirAll durable. -// MkdirAll fsyncs neither the new directories nor the direntries naming them, so -// on a fresh deployment a crash can lose a whole storage subtree while the synced -// catalog still advertises a "frozen" artifact under it — BarrierNewFile's -// grandparent fsync reaches a storage root's CONTENTS, never the root's own link -// in its parent. Given existingAncestor (the deepest dir that already existed, -// from DeepestExistingDir before the MkdirAll), this fsyncs createdLeaf and every -// ancestor up to and including existingAncestor, persisting each new dirent. When -// nothing was created (existingAncestor == createdLeaf) it costs one harmless dir -// fsync. Run once per root at startup. +// FsyncNewDirs makes a dir chain freshly produced by MkdirAll durable. MkdirAll +// fsyncs neither the new dirs nor their direntries, so on a fresh deployment a +// crash can lose a whole storage subtree while the synced catalog advertises a +// "frozen" artifact under it (BarrierNewFile's grandparent fsync reaches a root's +// CONTENTS, never the root's own link). Given existingAncestor (from +// DeepestExistingDir before the MkdirAll), this fsyncs createdLeaf up to and +// including it. When nothing was created it costs one harmless fsync. Run once +// per root at startup. func FsyncNewDirs(existingAncestor, createdLeaf string) error { for d := createdLeaf; ; d = filepath.Dir(d) { if err := FsyncDir(d); err != nil { diff --git a/cmd/stellar-rpc/internal/fullhistory/helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/helpers_test.go index d7dc16241..44759ea07 100644 --- a/cmd/stellar-rpc/internal/fullhistory/helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/helpers_test.go @@ -66,6 +66,15 @@ func testCatalog(t *testing.T) (*catalog.Catalog, string) { return cat, root } +// smallTxHashIndexCatalog builds a test catalog whose indexes are cpi chunks +// wide, so a "terminal" (full-index) build needs only a few chunks. Returns the +// catalog and the artifact root. +func smallTxHashIndexCatalog(t *testing.T, cpi uint32) (*catalog.Catalog, string) { + t.Helper() + cat, _, root := newTestCatalog(t, cpi) + return cat, root +} + // freezeKinds flips the given per-chunk kinds to "frozen" via the one-write protocol. func freezeKinds(t *testing.T, cat *catalog.Catalog, chunkID chunk.ID, kinds ...geometry.Kind) { t.Helper() @@ -105,9 +114,14 @@ func (r *recordingMetrics) BackfillPass(time.Duration) { r.backfillPasses++ } -func (*recordingMetrics) Freeze(time.Duration) {} -func (*recordingMetrics) Rebuild(time.Duration) {} -func (*recordingMetrics) Prune(int, time.Duration) {} +func (*recordingMetrics) LedgerCommitted(uint32) {} +func (*recordingMetrics) ChunkBoundary(uint32) {} +func (*recordingMetrics) Freeze(time.Duration) {} +func (*recordingMetrics) Rebuild(time.Duration) {} +func (*recordingMetrics) Prune(int, time.Duration) {} +func (*recordingMetrics) LiveHotChunks(int) {} +func (*recordingMetrics) ColdTierBytes(int64) {} +func (*recordingMetrics) Discard(int, time.Duration) {} var _ observability.Metrics = (*recordingMetrics)(nil) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotsource.go b/cmd/stellar-rpc/internal/fullhistory/hotsource.go new file mode 100644 index 000000000..b5f048c0b --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/hotsource.go @@ -0,0 +1,134 @@ +package fullhistory + +import ( + "context" + "errors" + "fmt" + "iter" + "os" + + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" + supportlog "github.com/stellar/go-stellar-sdk/support/log" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" +) + +// rocksHotProbe is the production backfill.HotProbe: it opens the chunk's shared +// multi-CF hot DB and answers backfillSource's completeness question (decision +// (a): the single maxCommittedSeq). +type rocksHotProbe struct { + hotRoot func(chunkID chunk.ID) string + logger *supportlog.Entry +} + +// NewRocksHotProbe returns the production backfill.HotProbe (hotChunkPath maps a +// chunk to its hot-DB dir — the daemon passes Layout.HotChunkPath). +// +// Caller contract: OpenHotChunk must NOT be passed the LIVE chunk — ingestion +// holds its hot DB open read-write and a second open fails on RocksDB's LOCK. The +// freeze only ever targets chunks ingestion has already released. +func NewRocksHotProbe(hotChunkPath func(chunk.ID) string, logger *supportlog.Entry) backfill.HotProbe { + return &rocksHotProbe{hotRoot: hotChunkPath, logger: logger} +} + +func (p *rocksHotProbe) OpenHotChunk(chunkID chunk.ID) (backfill.HotChunk, bool, error) { + dir := p.hotRoot(chunkID) + if _, err := os.Stat(dir); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil // dir absent — caller treats as loss under "ready" + } + return nil, false, fmt.Errorf("stat hot dir %s: %w", dir, err) + } + + // Open the chunk's shared multi-CF DB READ-ONLY: the freeze reads its ledgers + // to re-derive the cold artifacts and must never mutate it (the design's + // openRocksDBReadOnly). The probe only ever opens a chunk ingestion already + // released, so its data is fully in SST — no concurrent writer, no WAL replay. + db, err := hotchunk.OpenReadOnly(dir, chunkID, p.logger) + if err != nil { + return nil, false, fmt.Errorf("open hot chunk DB: %w", err) + } + return &rocksHotChunk{chunkID: chunkID, db: db}, true, nil +} + +// rocksHotChunk is one chunk's opened hot tier — the single shared DB. +type rocksHotChunk struct { + chunkID chunk.ID + db *hotchunk.DB +} + +// MaxCommittedSeq returns the single authoritative watermark (decision (a)): the +// highest ledger seq the shared DB has durably committed. ok=false on an empty DB. +func (h *rocksHotChunk) MaxCommittedSeq() (uint32, bool, error) { + seq, ok, err := h.db.MaxCommittedSeq() + if err != nil { + return 0, false, fmt.Errorf("hot DB max committed seq: %w", err) + } + return seq, ok, nil +} + +// Source streams the chunk's LCMs from the ledgers CF as a LedgerStream the cold +// writer (WriteColdChunk) drains, so a just-closed chunk freezes straight from its +// hot DB without a refetch. +func (h *rocksHotChunk) Source() ledgerbackend.LedgerStream { + return &hotLedgerStream{store: h.db.Ledgers()} +} + +// Close releases the shared hot DB. +func (h *rocksHotChunk) Close() error { + if h.db == nil { + return nil + } + return h.db.Close() +} + +// hotLedgerStream is a ledgerbackend.LedgerStream over a ledger.HotStore, so the +// source-blind cold pipeline freezes a just-closed chunk from its hot DB. +type hotLedgerStream struct { + store *ledger.HotStore +} + +var _ ledgerbackend.LedgerStream = (*hotLedgerStream)(nil) + +// RawLedgers yields the range's wire bytes from the hot store. IterateLedgers +// yields BORROWED buffers (valid only to the next step); the drain loop consumes +// each fully before the next yield, so the borrow is safe. ctx cancellation is +// observed between ledgers (the LedgerStream contract drain relies on). +func (st *hotLedgerStream) RawLedgers( + ctx context.Context, r ledgerbackend.Range, _ ...ledgerbackend.StreamOption, +) iter.Seq2[[]byte, error] { + return func(yield func([]byte, error) bool) { + if st.store == nil { + yield(nil, errors.New("fullhistory: hotLedgerStream has no store")) + return + } + to := r.To() + if !r.Bounded() { + last, ok, err := st.store.LastSeq() + if err != nil { + yield(nil, err) + return + } + if !ok { + return + } + to = last + } + for e, ierr := range st.store.IterateLedgers(r.From(), to) { + if cerr := ctx.Err(); cerr != nil { + yield(nil, cerr) + return + } + if ierr != nil { + yield(nil, ierr) + return + } + if !yield(e.Bytes, nil) { + return + } + } + } +} diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest.go b/cmd/stellar-rpc/internal/fullhistory/ingest.go new file mode 100644 index 000000000..22463fe3e --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/ingest.go @@ -0,0 +1,236 @@ +package fullhistory + +import ( + "context" + "fmt" + "os" + "path/filepath" + + supportlog "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/go-stellar-sdk/xdr" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/ingest" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/observability" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" +) + +// The hot-DB ingestion loop (decision (a)). One goroutine polls ledgers by seq +// (core.GetLedger) into the per-chunk shared multi-CF hot DB, committing each as +// one atomic synced WriteBatch across all CFs. It keeps NO progress variable — +// the last synced batch IS the watermark, re-derived at startup. Its only +// coupling to the lifecycle is the channel: at each boundary it sends the +// just-completed chunk id (the two goroutines share no memory). Clean-shutdown vs +// crash is decided at the daemon top level (a ctx-cancelled return is clean). + +// LedgerGetter is the indexed-poll source the ingestion loop drives: it returns +// one ledger's view, blocking until that ledger is available (the design's +// core.GetLedger(ctx, seq)). Production wraps captive core; tests pass a fake. +type LedgerGetter interface { + GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseMetaView, error) +} + +// allHotTypes is the hot tier's ingest selection: the hot DB is the sole copy of +// a chunk's recently ingested ledgers until the cold artifacts freeze, so it +// always ingests all three types in the one atomic batch. +// +//nolint:gochecknoglobals // immutable selection, the production ingest config +var allHotTypes = hotchunk.Ingest{Ledgers: true, Txhash: true, Events: true} + +// openHotTierForChunk opens/recovers/creates the chunk's shared hot DB, keyed on +// the durable hot:chunk state: +// - "ready": open it. A MISSING dir is hot-volume loss (the hot DB is the sole +// copy of recently-ingested ledgers) — refuse with ErrHotVolumeLost, never auto-heal. +// - "transient" or absent: wipe any leftover dir and create fresh +// (transient -> fsync dir+parent -> ready), so a crash mid-create can't +// fabricate the "ready but dir missing" fatal above. +func openHotTierForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlog.Entry) (*hotchunk.DB, error) { + dir := cat.Layout().HotChunkPath(chunkID) + + state, err := cat.HotState(chunkID) + if err != nil { + return nil, fmt.Errorf("streaming: read hot state chunk %s: %w", chunkID, err) + } + + if state == geometry.HotReady { + if _, statErr := os.Stat(dir); statErr != nil { + if os.IsNotExist(statErr) { + // The key promises a DB the filesystem lacks — hot storage was + // lost under a surviving meta store. Surfaced as the sentinel so + // the daemon's top-level loop owns the fatal-and-surface decision. + return nil, fmt.Errorf( + "%w: chunk %s is %q but its hot dir %s is missing", + backfill.ErrHotVolumeLost, chunkID, geometry.HotReady, dir) + } + return nil, fmt.Errorf( + "%w: chunk %s: stat hot dir %s: %w", + backfill.ErrHotVolumeLost, chunkID, dir, statErr) + } + db, openErr := hotchunk.Open(dir, chunkID, logger) + if openErr != nil { + // The dir existed at the stat above; an open failure now is loss. + return nil, fmt.Errorf("%w: chunk %s: open hot DB: %w", backfill.ErrHotVolumeLost, chunkID, openErr) + } + return db, nil + } + + // "transient" or absent: wipe any leftover dir, then create fresh under the bracket. + if rmErr := os.RemoveAll(dir); rmErr != nil { + return nil, fmt.Errorf("streaming: wipe leftover hot dir %s: %w", dir, rmErr) + } + if putErr := cat.PutHotTransient(chunkID); putErr != nil { + return nil, fmt.Errorf("streaming: mark hot transient chunk %s: %w", chunkID, putErr) + } + + db, openErr := hotchunk.Open(dir, chunkID, logger) + if openErr != nil { + return nil, fmt.Errorf("streaming: create hot DB chunk %s: %w", chunkID, openErr) + } + + // The dir + dirent must be durable BEFORE the key flips to "ready", else a + // crash between the flip and the dir's durability fabricates the "ready but + // dir missing" fatal above for a DB that was actually fine. + if syncErr := geometry.FsyncDir(dir); syncErr != nil { + _ = db.Close() + return nil, fmt.Errorf("streaming: fsync hot dir %s: %w", dir, syncErr) + } + if syncErr := geometry.FsyncDir(parentDir(dir)); syncErr != nil { + _ = db.Close() + return nil, fmt.Errorf("streaming: fsync hot parent dir %s: %w", parentDir(dir), syncErr) + } + if flipErr := cat.FlipHotReady(chunkID); flipErr != nil { + _ = db.Close() + return nil, fmt.Errorf("streaming: flip hot ready chunk %s: %w", chunkID, flipErr) + } + return db, nil +} + +// runIngestionLoop polls core for LCMs by seq into hotDB (one atomic synced +// WriteBatch each), and at each chunk boundary hands the frontier forward by +// closing the just-filled DB and opening the next. It never returns nil; the +// daemon classifies a ctx-cancelled return as clean shutdown, any other as +// RESTARTABLE (startup re-derives the watermark, losing nothing). +// +// HANDOFF FENCE: the DB is CLOSED before the next chunk's hot:chunk key is +// created — that key is what makes THIS chunk complete to the lifecycle, which +// could then discard a dir a still-live writer holds. notify() fires only after +// the next DB is open. The HotService (nil-sink-safe) is rebuilt each boundary. +func runIngestionLoop( + ctx context.Context, + core LedgerGetter, + hotDB *hotchunk.DB, + cat *catalog.Catalog, + lifecycleCh chan<- chunk.ID, + ingestTypes hotchunk.Ingest, + logger *supportlog.Entry, + metrics observability.Metrics, + sink ingest.MetricSink, +) (err error) { + metrics = observability.MetricsOrNop(metrics) + + // notify hands the just-completed chunk id to the lifecycle. A FULL buffer + // (LifecycleQueueDepth) means freeze has fallen that many boundaries behind — + // fail loud (a wedged lifecycle ingesting on cannot recover). + notify := func(complete chunk.ID) { + select { + case lifecycleCh <- complete: + default: + logger.Fatalf("streaming: lifecycle fell %d boundaries behind ingestion; investigate", + lifecycle.LifecycleQueueDepth) + } + } + + // The loop is hotDB's single writer and reopens it at every boundary. On any + // exit, close the live handle so the rocksdb instance does not leak (the + // boundary handoff already closed every prior chunk's DB); no writer races + // this close (the loop has stopped on every exit path). + defer func() { + if hotDB != nil { + if cerr := hotDB.Close(); cerr != nil && err == nil { + err = fmt.Errorf("streaming: close live hot DB: %w", cerr) + } + } + }() + + // Resume point: one past the live chunk's durable watermark (re-derived, not + // stored — a re-delivered committed ledger is an idempotent retry). + resume, err := nextIngestLedger(hotDB) + if err != nil { + return fmt.Errorf("streaming: derive resume ledger: %w", err) + } + + // hotService binds the metrics sink to THIS hotDB instance; the boundary + // handoff rebuilds it for the reopened chunk DB below. + hotService := ingest.NewHotService(hotDB, ingestTypes, sink) + + // Indexed poll from the resume ledger. GetLedger blocks until seq is + // available; its error ends the loop for the daemon top level to classify. + for seq := resume; ; seq++ { + lcm, gerr := core.GetLedger(ctx, seq) + if gerr != nil { + return fmt.Errorf("streaming: get ledger %d: %w", seq, gerr) + } + + // One atomic synced WriteBatch across all enabled CFs (via + // hotDB.IngestLedger), reporting per-type LedgerCounts to the sink. + if ierr := hotService.Ingest(ctx, seq, lcm); ierr != nil { + return fmt.Errorf("streaming: ingest ledger %d: %w", seq, ierr) + } + + // Per-ledger liveness gauge — the moving health signal a wedged ingester + // trips between boundaries (the tick-granular LastCommitted can't). + metrics.LedgerCommitted(seq) + + // Chunk boundary: this seq is the chunk's last ledger. + if seq == chunk.IDFromLedger(seq).LastLedger() { + closed := chunk.IDFromLedger(seq) + next := closed + 1 + // Handoff fence: close the write handle BEFORE the next chunk's key is + // created (that key is what makes THIS chunk complete to a tick, which + // may then freeze and discard its hot DB — no writer may hold it then). + if cerr := hotDB.Close(); cerr != nil { + hotDB = nil // closed (failed) — do not double-close in defer + return fmt.Errorf("streaming: close hot DB at boundary chunk %s: %w", closed, cerr) + } + hotDB = nil // released; reopen below republishes it for the defer + + nextDB, oerr := openHotTierForChunk(cat, next, logger) + if oerr != nil { + return fmt.Errorf("streaming: open hot DB for chunk %s at boundary: %w", next, oerr) + } + hotDB = nextDB + hotService = ingest.NewHotService(hotDB, ingestTypes, sink) + // next's key (created inside openHotTierForChunk) moved the partition; + // only now notify the lifecycle of the completed chunk. + notify(closed) + + // Boundary observability (the woken tick reports the freeze/discard/prune). + metrics.ChunkBoundary(uint32(closed)) + logger.WithField("closed_chunk", closed.String()). + WithField("next_chunk", next.String()). + WithField("last_ledger", seq). + Info("streaming: ingestion chunk boundary — handed off to lifecycle") + } + } +} + +// nextIngestLedger is the resume point for a just-opened live hot DB: one past +// its authoritative watermark, or the bound chunk's first ledger on an empty DB. +func nextIngestLedger(db *hotchunk.DB) (uint32, error) { + maxSeq, ok, err := db.MaxCommittedSeq() + if err != nil { + return 0, err + } + if !ok { + return db.ChunkID().FirstLedger(), nil + } + return maxSeq + 1, nil +} + +// parentDir is the dirent the hot-tier create barrier fsyncs so the chunk dir's +// creation is itself durable. +func parentDir(dir string) string { return filepath.Dir(dir) } diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go b/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go index 5667214d9..126f1ac61 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go @@ -1,67 +1,44 @@ -// Package ingest drives full-history ingestion: it pulls raw ledgers -// (wire-format xdr.LedgerCloseMeta bytes) from an injected ledger stream -// and writes the three data types — ledgers, txhashes, contract events — -// into the full-history stores, one chunk at a time, via the zero-copy -// view extractors in the go-stellar-sdk ingest package and the RPC-side -// events.LCMViewToPayloads emitter. +// Package ingest drives full-history ingestion: it pulls raw ledgers from an +// injected stream and writes the three data types (ledgers, txhashes, contract +// events) into the full-history stores, one chunk at a time, via the SDK's +// zero-copy view extractors and events.LCMViewToPayloads. // -// Two tiers share the per-ledger extraction but differ in everything -// else: +// Two tiers share the per-ledger extraction: // -// - Hot (RunHot): one chunk into the long-lived, caller-owned hot -// stores, from an injected ledgerbackend.LedgerStream. The stores -// are INJECTED and never opened or closed here, and neither is the -// stream; each ledger is durable before the next is pulled. -// Per-ledger fan-out across the enabled ingesters is concurrent -// (HotService). -// - Cold (WriteColdChunk): one chunk into per-chunk cold artifacts -// (ledger .pack, txhash .bin, events pack+index). It is -// SOURCE-BLIND — the caller resolves the chunk's ledger source and -// passes the raw ledger iterator, so the materializer never learns -// whether the bytes came from a local .pack or the bulk backend. -// Each cold ingester OPENS its own per-chunk writer; Finalize -// publishes the artifact and Close drops partials on the failure -// path (ColdService orchestrates). +// - Hot (HotService): one ledger into the per-chunk shared multi-CF hot DB +// (hotchunk.DB), committed as ONE atomic synced WriteBatch across all enabled +// CFs — a ledger is fully present or fully absent (decision (a)), no fan-out. +// - Cold (WriteColdChunk): one chunk into per-chunk cold artifacts (ledger +// .pack, txhash .bin, events pack+index). SOURCE-BLIND — the caller resolves +// the ledger source and passes the raw iterator, so the materializer never +// learns whether the bytes came from a local .pack or the bulk backend. Each +// cold ingester opens its own writer; Finalize publishes, Close drops +// partials on failure (ColdService orchestrates). // -// Artifact model (cold) — the contract every layer here relies on: +// Artifact model (cold) — the contract every layer relies on: // -// - Cold artifacts are NOT authoritative on their own. The -// orchestrator's completion record — written only after every -// enabled ingester's Finalize returned and its data is durable -// (writers fsync before reporting success) — is the single source -// of truth for whether a chunk exists. -// - Nothing may consume cold artifacts by scanning directories. A -// consumer (the serving tier, the deferred index build) takes +// - Cold artifacts are NOT authoritative alone. The orchestrator's completion +// record — written only after every Finalize returned and its data is durable +// — is the single source of truth for whether a chunk exists. +// - Nothing consumes cold artifacts by scanning directories; consumers take // explicit paths composed from the completion record. -// - A chunk attempt owns its chunk's paths exclusively and -// overwrites freely. Disk under the cold roots is scratch until the -// completion record says otherwise: stale or partial files from a -// failed or crashed attempt are inert, and the retry's overwrite -// is the cleanup. No writer needs tmp+rename atomicity, no -// constructor needs to pre-clean, and no failure path needs to -// roll committed siblings back. +// - A chunk attempt owns its paths exclusively and overwrites freely. Disk is +// scratch until the completion record says otherwise: partial/stale files +// from a failed attempt are inert and the retry's overwrite is the cleanup — +// so no writer needs tmp+rename, no pre-clean, no rollback. // -// Failure semantics (cold) follow from the model: a chunk either fully -// finalizes — and only then may the orchestrator record completion — or -// the attempt is abandoned and re-run from scratch; there is no -// mid-chunk resume. Once any Ingest fails, the chunk is released via -// Close (Finalize must not run; see the ColdIngester contract). Once -// any Finalize fails, ColdService stops at the first error; whatever -// the earlier ingesters already wrote stays on disk as inert scratch. +// Failure semantics (cold) follow: a chunk fully finalizes (then the orchestrator +// records completion) or is abandoned and re-run from scratch — no mid-chunk +// resume. A failed Ingest releases via Close (Finalize must not run); a failed +// Finalize stops ColdService at the first error. // -// Data types are processed in canonical ledgers→txhash→events order; -// the constructor table in buildColdIngesters is the order's single -// definition site. The on-disk formats and per-chunk filenames are -// owned by the store packages (ledger.PackName, txhash.ColdBinName + -// its .bin codec, eventstore's cold-format helpers); this package only -// composes the {bucketID:05d}/ bucket directories around them. +// Types are processed in canonical ledgers→txhash→events order (buildColdIngesters +// is the single definition site). On-disk formats/filenames are owned by the store +// packages; this package only composes the {bucketID:05d}/ bucket dirs. // -// Inputs are borrowed: every Ingest receives a view over the source -// stream's buffer, valid only until the next ledger is pulled, and -// each ingester copies what it retains (see HotIngester). The raw -// ledger iterator's contract includes yielding an error on ctx -// cancellation — the drain loop relies on it for cancellation rather -// than polling ctx itself. Metrics flow through MetricSink (Prometheus in prod, -// recorders in tests); the cold tier's invariant is exactly one -// ColdChunkTotal per chunk attempt, including pre-service failures. +// Inputs are borrowed: every Ingest gets a view valid only until the next pull, +// and each ingester copies what it retains. The raw iterator yields an error on +// ctx cancellation, so the drain loop needn't poll ctx. Metrics flow through +// MetricSink; the cold invariant is exactly one ColdChunkTotal per chunk attempt, +// including pre-service failures. package ingest diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go index 417cc2d37..a38b747b6 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go @@ -7,75 +7,27 @@ import ( "iter" "time" - "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" supportlog "github.com/stellar/go-stellar-sdk/support/log" "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash" ) -// HotStores holds the long-lived, caller-owned hot stores injected into RunHot. -// The caller (the daemon) opens and closes these; RunHot only borrows them to -// build the per-type hot ingesters. A field left nil for an enabled data type is -// a configuration error caught by RunHot. Every hot store is chunk-bound (each -// instance accumulates exactly one chunk before being frozen into cold -// artifacts), so each injected store must already be bound to the chunk being -// ingested — RunHot rejects a mismatch up front. -type HotStores struct { - Ledgers *ledger.HotStore - Txhash *txhash.HotStore - Events *eventstore.HotStore -} - -// buildHotIngesters constructs one HotIngester per data type enabled in cfg, in -// canonical ledgers→txhash→events order, from the injected stores. It errors if -// an enabled type's store is nil. -func buildHotIngesters(stores HotStores, sink MetricSink, cfg Config) ([]HotIngester, error) { - var ings []HotIngester - if cfg.Ledgers { - if stores.Ledgers == nil { - return nil, errors.New("ingest: Ledgers enabled but HotStores.Ledgers is nil") - } - ings = append(ings, NewLedgerHotIngester(stores.Ledgers, sink)) - } - if cfg.Txhash { - if stores.Txhash == nil { - return nil, errors.New("ingest: Txhash enabled but HotStores.Txhash is nil") - } - ings = append(ings, NewTxhashHotIngester(stores.Txhash, sink)) - } - if cfg.Events { - if stores.Events == nil { - return nil, errors.New("ingest: Events enabled but HotStores.Events is nil") - } - ings = append(ings, NewEventsHotIngester(stores.Events, sink)) - } - return ings, nil -} - -// errColdBuildAborted is the synthetic error recorded against an -// already-built cold ingester's metric when a LATER constructor fails and the -// build is rolled back. Without it, closing a fully-built ingester would emit -// a clean (nil-err, 0-items) ColdIngest — a phantom "success" for a chunk that -// never actually ingested anything. +// errColdBuildAborted is recorded against an already-built cold ingester when a +// LATER constructor fails and the build rolls back — without it, closing a +// fully-built ingester emits a clean ColdIngest, a phantom "success" for a chunk +// that ingested nothing. var errColdBuildAborted = errors.New("ingest: cold ingester build aborted (sibling constructor failed)") -// coldAborter is implemented by the concrete cold ingesters so the -// constructor-rollback path can mark their per-chunk metric as aborted before -// Close emits it, turning what would be a phantom success into a recorded -// abort. Optional: an ingester that does not implement it just gets its normal -// Close emission. +// coldAborter lets the rollback path mark an ingester's metric aborted before +// Close emits it. Optional — a non-implementer just gets its normal emission. type coldAborter interface { abortMetric(err error) } -// closeColdAll closes every cold ingester built so far, joining each Close error -// into err. Used when a LATER constructor fails mid-build: the already-built -// ingesters never ingested anything, so each one's metric is first marked -// aborted (so the deferred Close emit is not a phantom success). +// closeColdAll closes every ingester built so far (joining errors), first marking +// each aborted so the deferred Close emit is not a phantom success. Used when a +// LATER constructor fails mid-build. func closeColdAll(ings []ColdIngester, err error) error { for _, ing := range ings { if a, ok := ing.(coldAborter); ok { @@ -88,71 +40,10 @@ func closeColdAll(ings []ColdIngester, err error) error { return err } -// RunHot feeds each ledger of chunkID (as a view) from the injected stream to a -// HotService over the enabled hot ingesters, built from the INJECTED, -// caller-owned stores in hotStores. Ingest errors abort fast; HotService.Ingest -// waits for all ingesters before the loop pulls again so the borrowed view is -// never read past its lifetime. The hot stores are NOT closed here, and neither -// is the stream — the caller owns both lifecycles. -func RunHot( - ctx context.Context, - logger *supportlog.Entry, - stream ledgerbackend.LedgerStream, - chunkID chunk.ID, - hotStores HotStores, - sink MetricSink, - cfg Config, -) error { - if verr := cfg.validate(); verr != nil { - return verr - } - // Every hot store is chunk-bound — each instance accumulates exactly one - // chunk's data before being frozen into the chunk's cold artifacts — and - // records its chunk at open time. An injected store bound to a different - // chunk than we're ingesting would silently interleave two chunks' data - // (ledgers, txhash) or fail every per-ledger write with an out-of-range - // offset (events, whose LedgerOffsets are chunk-relative), so catch the - // mismatch up front with a clear message. Nil stores are skipped here: - // buildHotIngesters rejects a nil store for an enabled type with a more - // specific error. - checkBinding := func(name string, got chunk.ID) error { - if got != chunkID { - return fmt.Errorf("ingest: RunHot chunk %d but injected %s store is bound to chunk %d", - uint32(chunkID), name, uint32(got)) - } - return nil - } - if cfg.Ledgers && hotStores.Ledgers != nil { - if err := checkBinding("Ledgers", hotStores.Ledgers.ChunkID()); err != nil { - return err - } - } - if cfg.Txhash && hotStores.Txhash != nil { - if err := checkBinding("Txhash", hotStores.Txhash.ChunkID()); err != nil { - return err - } - } - if cfg.Events && hotStores.Events != nil { - if err := checkBinding("Events", hotStores.Events.ChunkID()); err != nil { - return err - } - } - ings, berr := buildHotIngesters(hotStores, sink, cfg) - if berr != nil { - return berr - } - logger.Debugf("RunHot: ingesting chunk %d [%d, %d]", uint32(chunkID), chunkID.FirstLedger(), chunkID.LastLedger()) - service := NewHotService(ings, sink) - raw := stream.RawLedgers(ctx, ledgerbackend.BoundedRange(chunkID.FirstLedger(), chunkID.LastLedger())) - return drain(ctx, raw, chunkID, service) -} - -// drain pulls the chunk's raw ledgers from the iterator and feeds each (as a view) -// to the service, then verifies the full [first,last] range was consumed. For the -// cold path this completeness check runs before Finalize, so a short stream never -// produces a finalized truncated artifact. The caller passes an iterator already -// bounded to the chunk's range; cancellation is the iterator's job (RawLedgers -// yields an error once ctx is canceled), so the loop needs no ctx poll of its own. +// drain feeds each of the chunk's raw ledgers (as a view) to the service, then +// verifies the full [first,last] range was consumed — for cold this runs before +// Finalize, so a short stream never finalizes a truncated artifact. Cancellation +// is the iterator's job (RawLedgers errors on canceled ctx), so no ctx poll here. func drain(ctx context.Context, ledgers iter.Seq2[[]byte, error], chunkID chunk.ID, ing HotIngester) error { first, last := chunkID.FirstLedger(), chunkID.LastLedger() seq := first @@ -160,21 +51,18 @@ func drain(ctx context.Context, ledgers iter.Seq2[[]byte, error], chunkID chunk. if serr != nil { return fmt.Errorf("RawLedgers(%d): %w", seq, serr) } - // Reject a stream that runs PAST the chunk before ingesting anything - // out-of-chunk. Without this, an in-order overrun would only trip the - // post-loop count check after the extra ledgers were durably ingested - // (the ledger and txhash hot stores accept any sequence). All in-repo - // sources bound themselves; this guards custom iterators. + // Reject an overrun before ingesting it: without this, the post-loop + // count check would only trip AFTER the extra ledgers were durably + // written (the ledger/txhash hot stores accept any seq). Guards custom + // iterators; in-repo sources self-bound. if seq > last { return fmt.Errorf("ingest: stream for chunk %d yielded a ledger past %d (chunk overrun)", uint32(chunkID), last) } lcm := xdr.LedgerCloseMetaView(raw) - // Validate the actual ledger sequence before ingesting. The final - // count check below only catches a short/long stream; a source that - // yields a duplicate or out-of-order ledger with the right total - // count would otherwise pass silently (e.g. on the txhash and - // ledger-hot paths, which key on the LCM's own seq). + // Validate the actual seq before ingesting: the count check only catches + // short/long streams, so a duplicate or out-of-order ledger with the + // right total count would otherwise pass silently. actual, aerr := lcm.LedgerSequence() if aerr != nil { return fmt.Errorf("ingest: stream for chunk %d: ledger sequence at expected %d: %w", @@ -184,8 +72,8 @@ func drain(ctx context.Context, ledgers iter.Seq2[[]byte, error], chunkID chunk. return fmt.Errorf("ingest: stream for chunk %d yielded ledger %d, expected %d", uint32(chunkID), actual, seq) } - // seq is now VALIDATED as lcm's sequence — pass it through so the - // ingesters consume it instead of each re-deriving it from the view. + // seq is now VALIDATED as lcm's sequence — pass it through so ingesters + // needn't each re-derive it. if err := ing.Ingest(ctx, seq, lcm); err != nil { return err } @@ -235,20 +123,16 @@ func buildColdIngesters(dirs ColdDirs, chunkID chunk.ID, sink MetricSink, cfg Co return ings, nil } -// WriteColdChunk materializes ONE chunk's cold artifacts into the roots named by -// dirs, in a single pass, from the already-opened raw ledger iterator. It is -// SOURCE-BLIND: the caller (backfill) resolves the chunk's ledger source — the -// local frozen .pack or the bulk backend — and hands its RawLedgers iterator here, -// so the cold materializer never learns where the bytes came from and is faked in -// tests with a literal slice iterator. The ingesters overwrite any crashed -// partial, so this is the freeze protocol's re-materialization. On any failure the -// attempt is abandoned — leftover files are inert scratch (see the package doc's -// artifact model) and a retry's overwrite is the cleanup. +// WriteColdChunk materializes ONE chunk's cold artifacts into dirs in a single +// pass from the raw ledger iterator. SOURCE-BLIND: the caller resolves the ledger +// source (local .pack or bulk backend) and hands its iterator here, so the +// materializer never learns where the bytes came from. Ingesters overwrite any +// crashed partial (the freeze protocol's re-materialization); on failure the +// attempt is abandoned, leftover files inert (package doc's artifact model). // // Source resolution (pack-stat, coverage wait) runs in the caller BEFORE this, so -// a pack-missing or coverage-timeout failure is metered there rather than as a -// ColdChunkTotal attempt here. The only pre-service failures left to meter here -// are a canceled ctx and a cold-ingester constructor failure. +// the only pre-service failures left to meter here are a canceled ctx and a +// constructor failure. func WriteColdChunk( ctx context.Context, logger *supportlog.Entry, @@ -263,10 +147,8 @@ func WriteColdChunk( } sink = orNop(sink) - // Pre-service failures (ctx and the constructor failure below) emit the - // chunk's single ColdChunkTotal here: the ColdService that normally owns that - // aggregate isn't built yet, but the invariant is "exactly one ColdChunkTotal - // per chunk attempt, including failures." + // Pre-service failures emit the chunk's single ColdChunkTotal here (the owning + // ColdService isn't built yet) — invariant: one ColdChunkTotal per attempt. start := time.Now() if cerr := ctx.Err(); cerr != nil { sink.ColdChunkTotal(time.Since(start)) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/events.go b/cmd/stellar-rpc/internal/fullhistory/ingest/events.go index 6bf9268b9..e5df3ba17 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/events.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/events.go @@ -25,53 +25,6 @@ func eventPayloads(seq uint32, lcm xdr.LedgerCloseMetaView) ([]events.Payload, e return payloads, nil } -// ───────────────────────── Hot ingester ───────────────────────── - -// eventsHot derives []events.Payload from the view (events.LCMViewToPayloads) and -// writes them with IngestLedgerEvents. Each call is one atomic RocksDB batch -// (sync=true) plus an in-memory mirror update. The store is INJECTED, already -// bound to a chunk, and owned by the caller. -// -// IngestLedgerEvents is called on every ledger, including ones with zero -// payloads — LedgerOffsets.Append requires a contiguous sequence and would -// reject the next non-empty ledger if an empty one were skipped. -type eventsHot struct { - store *eventstore.HotStore - sink MetricSink -} - -// NewEventsHotIngester returns a HotIngester writing contract events into the -// injected, caller-owned store (already bound to a chunk). -func NewEventsHotIngester(store *eventstore.HotStore, sink MetricSink) HotIngester { - return &eventsHot{store: store, sink: orNop(sink)} -} - -func (e *eventsHot) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { - m := newHotMetrics(e.sink, dataTypeEvents) - var err error - defer func() { m.emit(err) }() - - estart := time.Now() - payloads, eerr := eventPayloads(seq, lcm) - if eerr != nil { - err = eerr - return err - } - e.sink.IngestStage(dataTypeEvents, tierHot, stageExtract, time.Since(estart), len(payloads)) - // IngestLedgerEvents marshals each payload into a scratch buffer that - // RocksDB copies synchronously, so the borrowed ContractEventBytes (aliasing - // the view) is safe to pass. Term indexing happens inside the store call, - // so the write stage here covers term derivation + the RocksDB batch. - wstart := time.Now() - if ierr := e.store.IngestLedgerEvents(seq, payloads); ierr != nil { - err = fmt.Errorf("IngestLedgerEvents(seq=%d, n=%d): %w", seq, len(payloads), ierr) - return err - } - e.sink.IngestStage(dataTypeEvents, tierHot, stageWrite, time.Since(wstart), len(payloads)) - m.items = len(payloads) - return nil -} - // ───────────────────────── Cold ingester ───────────────────────── // eventsCold models the backfill path: per-ledger view → payloads → term-index diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go index e98898302..3e60b0baa 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go @@ -436,68 +436,6 @@ func (s *errAtSeqStream) RawLedgers( // ───────────────────────── per-ingester unit tests ───────────────────────── -// TestLedgerHotIngester_Readback ingests one ledger via the hot ledger ingester -// (injected store) and reads the bytes back. -func TestLedgerHotIngester_Readback(t *testing.T) { - seq := chunk.ID(0).FirstLedger() - raw := marshalLCM(t, seq) - dir := t.TempDir() - logger := testLogger() - - store, err := ledger.OpenHotStore(dir, chunk.ID(0), logger) - require.NoError(t, err) - defer func() { require.NoError(t, store.Close()) }() - - ing := NewLedgerHotIngester(store, nil) - require.NoError(t, ing.Ingest(context.Background(), seq, xdr.LedgerCloseMetaView(raw))) - - got, err := store.GetLedgerRaw(seq) - require.NoError(t, err) - require.Equal(t, raw, got) -} - -// TestTxhashHotIngester_Lookup ingests an event/tx-bearing ledger via the hot -// txhash ingester and looks the hash up. -func TestTxhashHotIngester_Lookup(t *testing.T) { - seq := chunk.ID(0).FirstLedger() - raw, hash, _ := marshalLCMWithEvent(t, seq) - dir := t.TempDir() - logger := testLogger() - - store, err := txhash.NewHotStore(dir, chunk.ID(0), logger) - require.NoError(t, err) - defer func() { require.NoError(t, store.Close()) }() - - ing := NewTxhashHotIngester(store, nil) - require.NoError(t, ing.Ingest(context.Background(), seq, xdr.LedgerCloseMetaView(raw))) - - got, err := store.Get(hash) - require.NoError(t, err) - require.Equal(t, seq, got) -} - -// TestEventsHotIngester_Query ingests an event-bearing ledger via the hot events -// ingester and resolves the term. -func TestEventsHotIngester_Query(t *testing.T) { - chunkID := chunk.ID(0) - seq := chunkID.FirstLedger() - raw, _, term := marshalLCMWithEvent(t, seq) - dir := t.TempDir() - logger := testLogger() - - store, err := eventstore.OpenHotStore(dir, chunkID, logger) - require.NoError(t, err) - defer func() { require.NoError(t, store.Close()) }() - - ing := NewEventsHotIngester(store, nil) - require.NoError(t, ing.Ingest(context.Background(), seq, xdr.LedgerCloseMetaView(raw))) - - bm, err := store.Lookup(context.Background(), term) - require.NoError(t, err) - require.NotNil(t, bm) - require.Equal(t, uint64(1), bm.GetCardinality()) -} - // TestLedgerColdIngester_Readback ingests one ledger via the cold ledger // ingester, finalizes, and reads back through the cold reader. func TestLedgerColdIngester_Readback(t *testing.T) { @@ -584,28 +522,6 @@ func TestEventsColdIngester_Readback(t *testing.T) { // ───────────────────────── V0 (pre-Soroban) events handling ───────────────────────── -// TestEventsHotIngester_V0AsEmpty asserts the hot events ingester treats a V0 -// LCM as a zero-event ledger (no error) rather than failing the range, and that -// the store records the empty ledger (its event count is unchanged). -func TestEventsHotIngester_V0AsEmpty(t *testing.T) { - chunkID := chunk.ID(0) - seq := chunkID.FirstLedger() - dir := t.TempDir() - logger := testLogger() - - store, err := eventstore.OpenHotStore(dir, chunkID, logger) - require.NoError(t, err) - defer func() { require.NoError(t, store.Close()) }() - - ing := NewEventsHotIngester(store, nil) - require.NoError(t, ing.Ingest(context.Background(), seq, xdr.LedgerCloseMetaView(marshalV0LCM(t, seq))), - "V0 ledger must ingest as zero events, not error") - - cnt, err := store.EventCount() - require.NoError(t, err) - require.Equal(t, uint32(0), cnt, "V0 ledger contributes no events") -} - // TestEventsColdIngester_V0KeepsOffsetsContiguous ingests a V0 ledger followed by // an event-bearing V2 ledger and asserts: the V0 ledger does not error, and the // LedgerOffsets stay contiguous (both ledgers present, the event-bearing one's @@ -706,92 +622,6 @@ func TestWriteColdChunk_EventlessChunk_FullyReadable(t *testing.T) { require.Zero(t, sink.coldErrorTypes()[dataTypeEvents], "eventless chunk is not an error") } -// ───────────────────────── HotService tests ───────────────────────── - -// TestHotService_AllTypes_FanOut runs HotService with all three hot ingesters -// over event/tx-bearing ledgers and reads each store back, asserting the -// aggregate HotLedgerTotal and per-ingester signals fired. -func TestHotService_AllTypes_FanOut(t *testing.T) { - chunkID := chunk.ID(0) - first := chunkID.FirstLedger() - logger := testLogger() - dir := t.TempDir() - - ls, err := ledger.OpenHotStore(filepath.Join(dir, "ledgers"), chunkID, logger) - require.NoError(t, err) - defer func() { require.NoError(t, ls.Close()) }() - ts, err := txhash.NewHotStore(filepath.Join(dir, "txhash"), chunkID, logger) - require.NoError(t, err) - defer func() { require.NoError(t, ts.Close()) }() - es, err := eventstore.OpenHotStore(filepath.Join(dir, "events"), chunkID, logger) - require.NoError(t, err) - defer func() { require.NoError(t, es.Close()) }() - - sink := &testSink{} - service := NewHotService([]HotIngester{ - NewLedgerHotIngester(ls, sink), - NewTxhashHotIngester(ts, sink), - NewEventsHotIngester(es, sink), - }, sink) - - rawA, hashA, termA := marshalLCMWithEvent(t, first) - rawB, hashB, _ := marshalLCMWithEvent(t, first+1) - require.NoError(t, service.Ingest(context.Background(), first, xdr.LedgerCloseMetaView(rawA))) - require.NoError(t, service.Ingest(context.Background(), first+1, xdr.LedgerCloseMetaView(rawB))) - - // All three stores retained the data. - gotRawA, err := ls.GetLedgerRaw(first) - require.NoError(t, err) - require.Equal(t, rawA, gotRawA) - gotA, err := ts.Get(hashA) - require.NoError(t, err) - require.Equal(t, first, gotA) - gotB, err := ts.Get(hashB) - require.NoError(t, err) - require.Equal(t, first+1, gotB) - bm, err := es.Lookup(context.Background(), termA) - require.NoError(t, err) - require.Equal(t, uint64(2), bm.GetCardinality()) - - // Aggregate + per-ingester signals. - require.Equal(t, 2, sink.hotLedgerTotals, "one HotLedgerTotal per ledger") - dt := sink.hotDataTypes() - require.Equal(t, 2, dt[dataTypeLedgers]) - require.Equal(t, 2, dt[dataTypeTxhash]) - require.Equal(t, 2, dt[dataTypeEvents]) - - // Per-stage signals: each ledger fired the hot extract/write stages its - // data type defines (ledgers has no extract — it writes the view verbatim). - st := sink.stageCounts() - require.Equal(t, 2, st[dataTypeLedgers+"/"+tierHot+"/"+stageWrite]) - require.Equal(t, 2, st[dataTypeTxhash+"/"+tierHot+"/"+stageExtract]) - require.Equal(t, 2, st[dataTypeTxhash+"/"+tierHot+"/"+stageWrite]) - require.Equal(t, 2, st[dataTypeEvents+"/"+tierHot+"/"+stageExtract]) - require.Equal(t, 2, st[dataTypeEvents+"/"+tierHot+"/"+stageWrite]) -} - -// TestHotService_EnabledSubset runs HotService with only the ledger ingester and -// asserts only that type's signals fire. -func TestHotService_EnabledSubset(t *testing.T) { - seq := chunk.ID(0).FirstLedger() - logger := testLogger() - dir := t.TempDir() - - ls, err := ledger.OpenHotStore(dir, chunk.ID(0), logger) - require.NoError(t, err) - defer func() { require.NoError(t, ls.Close()) }() - - sink := &testSink{} - service := NewHotService([]HotIngester{NewLedgerHotIngester(ls, sink)}, sink) - require.NoError(t, service.Ingest(context.Background(), seq, viewOf(t, seq))) - - require.Equal(t, 1, sink.hotLedgerTotals) - dt := sink.hotDataTypes() - require.Equal(t, 1, dt[dataTypeLedgers]) - require.Zero(t, dt[dataTypeTxhash]) - require.Zero(t, dt[dataTypeEvents]) -} - // ───────────────────────── ColdService tests ───────────────────────── // TestColdService_Success drives ledger+txhash+events cold ingesters through a @@ -986,78 +816,6 @@ func TestPrometheusSink_Smoke(t *testing.T) { require.NotEmpty(t, mfs) } -// ───────────────────────── hot driver tests ───────────────────────── - -// TestRunHot_AllTypes_Readback runs the RunHot driver with injected hot stores -// over event/tx-bearing ledgers and asserts each hot store reads back. The short -// stream ends early so RunHot returns the completeness error after both ledgers -// are fully ingested. -func TestRunHot_AllTypes_Readback(t *testing.T) { - chunkID := chunk.ID(0) - first := chunkID.FirstLedger() - logger := testLogger() - dir := t.TempDir() - - ls, err := ledger.OpenHotStore(filepath.Join(dir, "ledgers"), chunkID, logger) - require.NoError(t, err) - defer func() { require.NoError(t, ls.Close()) }() - ts, err := txhash.NewHotStore(filepath.Join(dir, "txhash"), chunkID, logger) - require.NoError(t, err) - defer func() { require.NoError(t, ts.Close()) }() - es, err := eventstore.OpenHotStore(filepath.Join(dir, "events"), chunkID, logger) - require.NoError(t, err) - defer func() { require.NoError(t, es.Close()) }() - - evSeqA, evSeqB := first, first+1 - rawA, hashA, termA := marshalLCMWithEvent(t, evSeqA) - rawB, hashB, _ := marshalLCMWithEvent(t, evSeqB) - gen := func(tt *testing.T, seq uint32) []byte { - switch seq { - case evSeqA: - return rawA - case evSeqB: - return rawB - default: - return marshalLCM(tt, seq) - } - } - stream := &fakeStream{t: t, count: 2, gen: gen} - - stores := HotStores{Ledgers: ls, Txhash: ts, Events: es} - cfg := Config{Ledgers: true, Txhash: true, Events: true} - - err = RunHot(context.Background(), logger, stream, chunkID, stores, nil, cfg) - require.Error(t, err) - require.Contains(t, err.Error(), "ended at") - - gotRawA, err := ls.GetLedgerRaw(evSeqA) - require.NoError(t, err) - require.Equal(t, rawA, gotRawA) - - gotA, err := ts.Get(hashA) - require.NoError(t, err) - require.Equal(t, evSeqA, gotA) - gotB, err := ts.Get(hashB) - require.NoError(t, err) - require.Equal(t, evSeqB, gotB) - - bm, err := es.Lookup(context.Background(), termA) - require.NoError(t, err) - require.NotNil(t, bm) - require.Equal(t, uint64(2), bm.GetCardinality(), "both sentinel events share the term") -} - -// TestRunHot_MissingStore asserts RunHot rejects an enabled type with a nil -// injected store. -func TestRunHot_MissingStore(t *testing.T) { - chunkID := chunk.ID(0) - logger := testLogger() - err := RunHot(context.Background(), logger, &fakeStream{t: t, count: 1}, chunkID, - HotStores{}, nil, Config{Ledgers: true}) - require.Error(t, err) - require.Contains(t, err.Error(), "HotStores.Ledgers is nil") -} - // ───────────────────────── cold driver tests ───────────────────────── func TestWriteColdChunk_RoundTrip(t *testing.T) { @@ -1275,104 +1033,7 @@ func TestWriteColdChunk_DrainStreamError_NoArtifact(t *testing.T) { // pkg/stores/txhash (cold_bin_test.go); these tests only cover the // ingester-level behavior on top of it. -// ───────────────────────── HotService failure path (P1-c) ───────────────────────── - -// failingHot is a HotIngester whose Ingest always fails. ctxObserved records -// whether the ingester's context was already canceled when it ran (used to -// show errgroup sibling cancellation in the multi-ingester path). -type failingHot struct { - mu sync.Mutex - ran int - ctxObserved error -} - -var errFailingHot = errors.New("failingHot: induced ingest failure") - -func (f *failingHot) Ingest(ctx context.Context, _ uint32, _ xdr.LedgerCloseMetaView) error { - f.mu.Lock() - f.ran++ - f.ctxObserved = ctx.Err() - f.mu.Unlock() - return errFailingHot -} - -// blockingHot blocks until its context is canceled, then reports the cancel -// error. Pairs with failingHot in the multi-ingester test to prove the first -// error cancels the siblings via the errgroup context. -type blockingHot struct { - canceled chan struct{} - once sync.Once -} - -func (b *blockingHot) Ingest(ctx context.Context, _ uint32, _ xdr.LedgerCloseMetaView) error { - <-ctx.Done() - b.once.Do(func() { close(b.canceled) }) - return ctx.Err() -} - -// TestHotService_SingleIngesterFailure asserts the len==1 fast path returns the -// ingester error and still emits exactly one HotLedgerTotal. -func TestHotService_SingleIngesterFailure(t *testing.T) { - sink := &testSink{} - fail := &failingHot{} - service := NewHotService([]HotIngester{fail}, sink) - - err := service.Ingest(context.Background(), chunk.ID(0).FirstLedger(), viewOf(t, chunk.ID(0).FirstLedger())) - require.ErrorIs(t, err, errFailingHot) - require.Equal(t, 1, sink.hotLedgerTotals, "HotLedgerTotal fires exactly once even on failure") -} - -// TestHotService_MultiIngesterFailureCancelsSiblings asserts the errgroup path -// propagates the failing ingester's error, cancels the sibling via the group -// context, and still emits exactly one HotLedgerTotal. -func TestHotService_MultiIngesterFailureCancelsSiblings(t *testing.T) { - sink := &testSink{} - fail := &failingHot{} - block := &blockingHot{canceled: make(chan struct{})} - service := NewHotService([]HotIngester{fail, block}, sink) - - err := service.Ingest(context.Background(), chunk.ID(0).FirstLedger(), viewOf(t, chunk.ID(0).FirstLedger())) - require.ErrorIs(t, err, errFailingHot) - - // The blocking sibling only returns once its context is canceled, so a - // non-blocking Ingest return already proves cancellation propagated. - select { - case <-block.canceled: - case <-time.After(2 * time.Second): - t.Fatal("sibling ingester was not canceled by the failing ingester") - } - require.Equal(t, 1, sink.hotLedgerTotals, "HotLedgerTotal fires exactly once even on failure") -} - -// TestHotIngester_Failure_RecordsErrorMetric drives a REAL hot ingester -// (eventsHot, built via NewEventsHotIngester) with a malformed view so its own -// Ingest fails through the production hotMetrics emit path — unlike the -// failingHot/blockingHot stubs, which bypass hotMetrics entirely. Per #765 a -// failed hot Ingest must record exactly one HotIngest carrying a non-nil error -// for that data type. Mirrors the cold-side TestColdIngester_Failure_RecordsErrorMetric. -func TestHotIngester_Failure_RecordsErrorMetric(t *testing.T) { - chunkID := chunk.ID(0) - logger := testLogger() - dir := t.TempDir() - sink := &testSink{} - - store, err := eventstore.OpenHotStore(dir, chunkID, logger) - require.NoError(t, err) - defer func() { require.NoError(t, store.Close()) }() - - ing := NewEventsHotIngester(store, sink) - - // A truncated/garbage view makes the event extraction fail inside the real - // Ingest, so the deferred hotMetrics.emit reports the wrapped error. - bad := xdr.LedgerCloseMetaView([]byte{0x00, 0x01, 0x02}) - require.Error(t, ing.Ingest(context.Background(), chunkID.FirstLedger(), bad)) - - sink.mu.Lock() - defer sink.mu.Unlock() - require.Len(t, sink.hotIngests, 1, "exactly one HotIngest recorded") - require.Equal(t, dataTypeEvents, sink.hotIngests[0].dataType) - require.Error(t, sink.hotIngests[0].err, "the recorded HotIngest carries the ingest error") -} +// ───────────────────────── hot ingester failure path (P1-c) ───────────────────────── // ───────────────────────── cold txhash .bin content (P1-d) ───────────────────────── @@ -1442,47 +1103,6 @@ func TestWriteColdChunk_CanceledContext(t *testing.T) { require.Equal(t, 1, sink.coldChunkTotals, "a canceled chunk attempt still emits one ColdChunkTotal") } -// ───────────────────────── RunHot chunkID cross-check (P2-e) ───────────────────────── - -// TestRunHot_ChunkIDMismatch asserts RunHot rejects ANY injected hot store -// bound to a different chunk than the one being ingested, with a clear -// up-front error (rather than silently interleaving chunks on the ledger and -// txhash paths, or a later per-ledger out-of-range on the events path). All -// three hot stores are chunk-bound. -func TestRunHot_ChunkIDMismatch(t *testing.T) { - ingestChunk := chunk.ID(1) - storeChunk := chunk.ID(0) - logger := testLogger() - - run := func(t *testing.T, stores HotStores, cfg Config) { - t.Helper() - err := RunHot(context.Background(), logger, &fakeStream{t: t, count: 1}, ingestChunk, - stores, nil, cfg) - require.Error(t, err) - require.Contains(t, err.Error(), "bound to chunk 0") - require.Contains(t, err.Error(), "RunHot chunk 1") - } - - t.Run("ledgers", func(t *testing.T) { - ls, err := ledger.OpenHotStore(t.TempDir(), storeChunk, logger) - require.NoError(t, err) - defer func() { require.NoError(t, ls.Close()) }() - run(t, HotStores{Ledgers: ls}, Config{Ledgers: true}) - }) - t.Run("txhash", func(t *testing.T) { - ts, err := txhash.NewHotStore(t.TempDir(), storeChunk, logger) - require.NoError(t, err) - defer func() { require.NoError(t, ts.Close()) }() - run(t, HotStores{Txhash: ts}, Config{Txhash: true}) - }) - t.Run("events", func(t *testing.T) { - es, err := eventstore.OpenHotStore(t.TempDir(), storeChunk, logger) - require.NoError(t, err) - defer func() { require.NoError(t, es.Close()) }() - run(t, HotStores{Events: es}, Config{Events: true}) - }) -} - // ───────────────────────── Config validate / guard negatives (P2-g) ───────────────────────── // TestWriteColdChunk_ConfigGuards covers the validate guard on the cold materializer: @@ -1498,14 +1118,6 @@ func TestWriteColdChunk_ConfigGuards(t *testing.T) { require.Contains(t, err.Error(), "enables no data types") } -// TestRunHot_EmptyConfig asserts the hot driver also rejects an empty Config. -func TestRunHot_EmptyConfig(t *testing.T) { - err := RunHot(context.Background(), testLogger(), &fakeStream{t: t, count: 1}, - chunk.ID(0), HotStores{}, nil, Config{}) - require.Error(t, err) - require.Contains(t, err.Error(), "enables no data types") -} - // ───────────────────────── constructor-rollback metrics ───────────────────────── // countCleanColdIngests counts recorded ColdIngest signals with a nil error. diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go index d59453293..801db2b88 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go @@ -8,45 +8,31 @@ import ( // HotIngester ingests one data type for one ledger into a long-lived hot store. // -// Ownership: the hot store is INJECTED into the ingester's constructor and owned -// by the caller (the daemon). The ingester does NOT open the store and does NOT -// close it — Close is intentionally absent from this interface. +// Ownership: the store is INJECTED and caller-owned (the daemon); the ingester +// does NOT open or close it — Close is intentionally absent. // -// Input: seq is the DRIVER-VALIDATED ledger sequence of lcm — the drain loop -// has already read it off the view and checked it against the chunk's expected -// position (duplicate / out-of-order / overrun), so ingesters consume it -// directly instead of each re-deriving and re-error-handling it. lcm is a -// zero-copy xdr.LedgerCloseMetaView (a []byte alias over the source stream's -// BORROWED buffer), valid only for the current iteration step; an ingester -// must copy any bytes it retains. The hot fan-out (HotService) waits for all -// ingesters to finish a ledger before the source pulls the next one, so -// synchronous consumption inside Ingest is safe. -// -// Concurrency: distinct HotIngester instances are run concurrently for the same -// ledger (HotService fans out via errgroup); each instance touches only its own -// store plus the read-only view. +// Input: seq is the DRIVER-VALIDATED sequence of lcm (drain already checked it +// against the chunk's expected position), so ingesters consume it directly. lcm +// is a zero-copy view over the source's BORROWED buffer, valid only this step — +// an ingester must copy what it retains. The view is consumed synchronously +// within Ingest, so it is never read past its lifetime. type HotIngester interface { Ingest(ctx context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error } // ColdIngester ingests one data type for one chunk into a per-chunk cold writer. // -// Ownership: the ingester OPENS its own per-chunk writer in its constructor and -// owns its lifecycle. Finalize commits the chunk's artifact (explicit, -// error-checked, never deferred). Close is always deferred and idempotent; on -// the failure path (Finalize never ran) it drops any partial file. +// Ownership: the ingester OPENS its own writer and owns its lifecycle. Finalize +// commits the artifact (explicit, error-checked); Close is deferred + idempotent +// and drops any partial on the failure path. // -// Contract: Finalize must NOT be called after a failed Ingest — once any -// Ingest errors, the chunk is abandoned via Close and retried from scratch. -// Implementations may have committed partial per-ledger state before the -// error (e.g. the events ingester's mirror/pack run ahead of its offsets -// commit point), so a post-failure Finalize could publish an inconsistent -// artifact; implementations are encouraged to latch the failure and refuse -// (eventsCold does). +// Contract: Finalize must NOT be called after a failed Ingest — the chunk is +// abandoned via Close and retried from scratch. Partial per-ledger state may +// already be committed, so a post-failure Finalize could publish an inconsistent +// artifact; implementations should latch the failure and refuse (eventsCold does). // -// Input: same driver-validated-seq and borrowed-view contract as HotIngester. -// ColdService drives the per-ledger Ingest calls sequentially, so each view is -// fully consumed before the next. +// Input: same driver-validated-seq + borrowed-view contract as HotIngester; +// ColdService drives Ingest sequentially. type ColdIngester interface { Ingest(ctx context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error Finalize(ctx context.Context) error diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go index f9bab63af..75192ea89 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go @@ -13,42 +13,6 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" ) -// ───────────────────────── Hot ingester ───────────────────────── - -// ledgerHot writes raw ledger bytes verbatim into a long-lived ledger.HotStore. -// AddLedgers fsyncs once per call, so each ledger is durable before Ingest -// returns. The store is INJECTED and owned by the caller — ledgerHot never -// opens or closes it. -type ledgerHot struct { - store *ledger.HotStore - sink MetricSink -} - -// NewLedgerHotIngester returns a HotIngester writing raw ledger bytes into the -// injected, caller-owned store. -func NewLedgerHotIngester(store *ledger.HotStore, sink MetricSink) HotIngester { - return &ledgerHot{store: store, sink: orNop(sink)} -} - -func (h *ledgerHot) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { - m := newHotMetrics(h.sink, dataTypeLedgers) - var err error - defer func() { m.emit(err) }() - - // ledger.HotStore.AddLedgers copies the bytes into its RocksDB batch - // synchronously, so aliasing the borrowed view buffer here is safe. - wstart := time.Now() - if aerr := h.store.AddLedgers(ledger.Entry{Seq: seq, Bytes: []byte(lcm)}); aerr != nil { - err = fmt.Errorf("AddLedgers(seq=%d): %w", seq, aerr) - return err - } - h.sink.IngestStage(dataTypeLedgers, tierHot, stageWrite, time.Since(wstart), 1) - // Set AFTER the store call so a failed write reports items=0, matching - // the MetricSink "items written" contract and the other hot ingesters. - m.items = 1 - return nil -} - // ───────────────────────── Cold ingester ───────────────────────── // ledgerCold writes raw ledger bytes into a per-chunk ledger.ColdWriter (one diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go index 22ab631dc..714458e93 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go @@ -88,38 +88,6 @@ func orNop(sink MetricSink) MetricSink { return sink } -// hotMetrics emits a single HotIngest signal for one hot ingester's per-ledger -// Ingest. The ingester sets items as it learns the count, then a single deferred -// emit reports the wall-clock since start, the final item count, and the WRAPPED -// error captured from the named return — so every Ingest has exactly one emit -// site regardless of which return path it takes. -// -// Usage: -// -// func (h *fooHot) Ingest(...) (err error) { -// m := newHotMetrics(h.sink, dataTypeFoo) -// defer func() { m.emit(err) }() -// ... -// m.items = len(things) -// return nil -// } -type hotMetrics struct { - sink MetricSink - dataType string - start time.Time - items int -} - -func newHotMetrics(sink MetricSink, dataType string) hotMetrics { - return hotMetrics{sink: orNop(sink), dataType: dataType, start: time.Now()} -} - -// emit reports the single HotIngest signal: the wall-clock since construction, -// the accumulated item count, and the (wrapped) error from the named return. -func (m *hotMetrics) emit(err error) { - m.sink.HotIngest(m.dataType, time.Since(m.start), m.items, err) -} - // coldMetrics is the per-chunk metric accumulator shared by all three cold // ingesters. Each ingester accumulates Ingest wall-clock (accum), item count // (items), and the FIRST error it saw (firstErr) across the chunk, then emits a diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go index 1d5430f06..00205af05 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go @@ -6,9 +6,9 @@ import ( "fmt" "time" - "golang.org/x/sync/errgroup" - "github.com/stellar/go-stellar-sdk/xdr" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" ) // errOrFirst returns prev if it is non-nil, else cur. Used to retain the FIRST @@ -21,60 +21,66 @@ func errOrFirst(prev, cur error) error { return cur } -// HotService fans one ledger out to a set of HotIngesters concurrently, waiting -// for all to finish before returning (so the borrowed view is safe to release), -// and emits the aggregate per-ledger wall-clock via the sink. +// HotService commits one ledger to the shared per-chunk hot DB as ONE atomic +// synced WriteBatch across all enabled CFs (decision (a)) and emits per-ledger +// wall-clock + per-type volume signals. No fan-out — the three types are CFs of +// one RocksDB committing in one WriteBatch (hotchunk.DB.IngestLedger). type HotService struct { - ingesters []HotIngester - sink MetricSink + db *hotchunk.DB + cfg hotchunk.Ingest + sink MetricSink } -// NewHotService builds a HotService over the enabled hot ingesters. A nil sink -// defaults to NopSink. -func NewHotService(ingesters []HotIngester, sink MetricSink) *HotService { - return &HotService{ingesters: ingesters, sink: orNop(sink)} +// NewHotService builds a HotService that writes the data types enabled in cfg +// into the shared per-chunk DB. A nil sink defaults to NopSink. +func NewHotService(db *hotchunk.DB, cfg hotchunk.Ingest, sink MetricSink) *HotService { + return &HotService{db: db, cfg: cfg, sink: orNop(sink)} } -// Ingest runs every hot ingester on lcm concurrently and waits for all of them. -// seq is the driver-validated sequence of lcm, passed through unchanged. The -// first ingester error is returned; the production HotIngester.Ingest -// implementations do not check ctx.Err(), so the siblings run to completion -// regardless (g.Wait still returns the first error). The single-ingester config -// skips the errgroup entirely. HotLedgerTotal is emitted with the fan-out -// wall-clock regardless of success. -func (s *HotService) Ingest(ctx context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { +// Ingest commits lcm to the shared hot DB in one atomic synced WriteBatch +// (decision (a)). HotLedgerTotal is emitted regardless of success; on success, +// one HotIngest per enabled type reports its item count. A nil DB (no hot tier) +// is a no-op other than the aggregate timing. +func (s *HotService) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { start := time.Now() - switch len(s.ingesters) { - case 0: - // No hot ingesters enabled for this tier: nothing to do. + if s.db == nil { s.sink.HotLedgerTotal(time.Since(start)) return nil - case 1: - // Single ingester: call directly, skipping the errgroup overhead. - err := s.ingesters[0].Ingest(ctx, seq, lcm) - s.sink.HotLedgerTotal(time.Since(start)) - return err - default: - // Two or more: concurrent fan-out, waiting for all. - g, gctx := errgroup.WithContext(ctx) - for _, ing := range s.ingesters { - g.Go(func() error { return ing.Ingest(gctx, seq, lcm) }) - } - err := g.Wait() - s.sink.HotLedgerTotal(time.Since(start)) - return err } + counts, err := s.db.IngestLedger(seq, lcm, s.cfg) + s.emit(counts, time.Since(start), err) + s.sink.HotLedgerTotal(time.Since(start)) + return err +} + +// emit reports one HotIngest per enabled type. On error, counts are 0 with the +// error attached (a failed atomic commit wrote nothing durably). +func (s *HotService) emit(counts hotchunk.LedgerCounts, d time.Duration, err error) { + if s.cfg.Ledgers { + s.sink.HotIngest(dataTypeLedgers, d, itemsOnSuccess(counts.Ledgers, err), err) + } + if s.cfg.Txhash { + s.sink.HotIngest(dataTypeTxhash, d, itemsOnSuccess(counts.Txhash, err), err) + } + if s.cfg.Events { + s.sink.HotIngest(dataTypeEvents, d, itemsOnSuccess(counts.Events, err), err) + } +} + +// itemsOnSuccess returns n on success and 0 on error — a failed atomic batch +// commits nothing, so no items were written. +func itemsOnSuccess(n int, err error) int { + if err != nil { + return 0 + } + return n } // ColdService drives a set of ColdIngesters for one chunk: sequential per-ledger -// Ingest, then Finalize on each. It times from the first Ingest (or, if none ran, -// from the Finalize/Close call) and emits the aggregate ColdChunkTotal exactly -// once for the chunk — in Finalize on the success path, otherwise in Close on the -// failure path (an Ingest error or short stream short-circuits before Finalize). -// The totalEmitted flag prevents a double-emit: Finalize sets it so the caller's -// deferred Close is a no-op for the aggregate. (A ctx or constructor failure -// happens before the service is built — WriteColdChunk emits that chunk's single -// ColdChunkTotal directly.) +// Ingest, then Finalize on each. It emits the aggregate ColdChunkTotal exactly +// once — in Finalize on success, else in Close on failure; totalEmitted prevents +// the double-emit. (Pre-service ctx/constructor failures are metered directly by +// WriteColdChunk.) type ColdService struct { ingesters []ColdIngester sink MetricSink @@ -82,18 +88,14 @@ type ColdService struct { totalEmitted bool } -// NewColdService builds a ColdService over the enabled cold ingesters. A nil -// sink defaults to NopSink. The per-chunk aggregate timer starts here; the only -// case where no Ingest follows is an already-errored short/empty stream, where -// the timing sample is meaningless anyway. +// NewColdService builds a ColdService over the enabled cold ingesters (nil sink +// → NopSink). The per-chunk aggregate timer starts here. func NewColdService(ingesters []ColdIngester, sink MetricSink) *ColdService { return &ColdService{ingesters: ingesters, sink: orNop(sink), start: time.Now()} } // Ingest runs every cold ingester on lcm sequentially (each owns mutable -// per-chunk state, so no concurrency within the service). seq is the -// driver-validated sequence of lcm, passed through unchanged. The first error -// aborts the ledger. +// per-chunk state). The first error aborts the ledger. func (s *ColdService) Ingest(ctx context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { for _, ing := range s.ingesters { if err := ing.Ingest(ctx, seq, lcm); err != nil { @@ -103,14 +105,11 @@ func (s *ColdService) Ingest(ctx context.Context, seq uint32, lcm xdr.LedgerClos return nil } -// Finalize commits each cold ingester's chunk artifact (explicit, error-checked, -// never deferred). The first Finalize error STOPS the loop: the remaining -// (unfinalized) ingesters are released by the caller's deferred Close, and the -// failed chunk attempt is reported to the orchestrator, which never records -// completion for it. Artifacts the earlier ingesters already wrote are left in -// place — without the orchestrator's completion record they are inert scratch -// (see the package doc's artifact model), and the retry's overwrite is the -// cleanup. The per-chunk ColdChunkTotal is emitted here on the success path. +// Finalize commits each cold ingester's chunk artifact (explicit, error-checked). +// The first error STOPS the loop: unfinalized ingesters are released by the +// caller's deferred Close, and the failed attempt is never recorded complete by +// the orchestrator — earlier-written artifacts stay as inert scratch (package +// doc's artifact model). Emits ColdChunkTotal here on the success path. func (s *ColdService) Finalize(ctx context.Context) error { var ferr error for _, ing := range s.ingesters { @@ -123,12 +122,11 @@ func (s *ColdService) Finalize(ctx context.Context) error { return ferr } -// Close closes every cold ingester, joining each Close error, and emits the -// aggregate ColdChunkTotal if Finalize never reached it (the failure path). Each -// ingester's own Close in turn emits that ingester's per-chunk ColdIngest if its -// Finalize never ran, so a failed chunk still produces one per-ingester signal -// and one aggregate. Idempotent: on the failure path a writer's Close drops its -// partial file; after a successful Finalize all emissions are no-ops. +// Close closes every cold ingester (joining errors) and emits ColdChunkTotal if +// Finalize never reached it (failure path). Each ingester's Close in turn emits +// its own ColdIngest if its Finalize never ran, so a failed chunk still produces +// one per-ingester signal + one aggregate. Idempotent: on failure a writer's +// Close drops its partial; after a successful Finalize all emissions are no-ops. func (s *ColdService) Close() error { var err error for _, ing := range s.ingesters { diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go b/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go index b80f77de5..dfd667452 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go @@ -16,51 +16,6 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash" ) -// ───────────────────────── Hot ingester ───────────────────────── - -// txhashHot extracts the ledger's transaction hashes via the SDK -// (sdkingest.ExtractTxHashes — apply order, hashes copied off the view) and -// writes (txhash, seq) tuples in one AddEntries call (one fsync per ledger). -// The store is INJECTED and owned by the caller. -type txhashHot struct { - store *txhash.HotStore - sink MetricSink -} - -// NewTxhashHotIngester returns a HotIngester writing (txhash, seq) tuples into -// the injected, caller-owned store. -func NewTxhashHotIngester(store *txhash.HotStore, sink MetricSink) HotIngester { - return &txhashHot{store: store, sink: orNop(sink)} -} - -func (t *txhashHot) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { - m := newHotMetrics(t.sink, dataTypeTxhash) - var err error - defer func() { m.emit(err) }() - - estart := time.Now() - hashes, eerr := sdkingest.ExtractTxHashes(lcm) - if eerr != nil { - err = fmt.Errorf("ExtractTxHashes seq %d: %w", seq, eerr) - return err - } - t.sink.IngestStage(dataTypeTxhash, tierHot, stageExtract, time.Since(estart), len(hashes)) - if len(hashes) > 0 { - entries := make([]txhash.Entry, len(hashes)) - for i, h := range hashes { - entries[i] = txhash.Entry{Hash: [32]byte(h), LedgerSeq: seq} - } - wstart := time.Now() - if aerr := t.store.AddEntries(entries); aerr != nil { - err = fmt.Errorf("AddEntries(seq=%d, n=%d): %w", seq, len(entries), aerr) - return err - } - t.sink.IngestStage(dataTypeTxhash, tierHot, stageWrite, time.Since(wstart), len(entries)) - } - m.items = len(hashes) - return nil -} - // ───────────────────────── Cold ingester ───────────────────────── // txhashCold accumulates (txhash[:ColdKeySize], seq) tuples per ledger; at diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest_test.go b/cmd/stellar-rpc/internal/fullhistory/ingest_test.go new file mode 100644 index 000000000..3789b4faf --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/ingest_test.go @@ -0,0 +1,369 @@ +package fullhistory + +import ( + "context" + "errors" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stellar/go-stellar-sdk/xdr" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" +) + +// ledgerEntry builds a ledgers-CF entry carrying a real zero-tx LCM for seq — +// the bytes the cold pipeline can later re-read if the chunk freezes from the +// hot DB. +func ledgerEntry(t *testing.T, seq uint32) ledger.Entry { + t.Helper() + return ledger.Entry{Seq: seq, Bytes: zeroTxLCMBytes(t, seq)} +} + +// --------------------------------------------------------------------------- +// fakeLedgerGetter — an injectable LedgerGetter the ingestion loop polls by +// sequence (the design's indexed core.GetLedger(ctx, seq)). For seqs it has a +// programmed frame it returns those bytes; once the poll runs past the last +// programmed seq it either blocks until ctx is cancelled (a live tip stream that +// only ends on shutdown) or returns endErr (a crashed backend). It records the +// FIRST seq it was asked for (the restart resume point) and the GetLedger call +// count. +// --------------------------------------------------------------------------- + +type fakeLedgerGetter struct { + frames map[uint32][]byte // seq -> raw LCM bytes + maxSeq uint32 // highest programmed seq + blockOnCtx bool // past the last frame, block until ctx.Done + endErr error // past the last frame, return this (when not blocking) + yieldErrAt uint32 // if non-zero, return errAt at this seq instead of bytes + errAt error + + calls atomic.Int32 + firstSeen atomic.Uint32 + sawFirst atomic.Bool +} + +var _ LedgerGetter = (*fakeLedgerGetter)(nil) + +func (g *fakeLedgerGetter) GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseMetaView, error) { + g.calls.Add(1) + if g.sawFirst.CompareAndSwap(false, true) { + g.firstSeen.Store(seq) + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + if g.yieldErrAt != 0 && seq == g.yieldErrAt { + return nil, g.errAt + } + if raw, ok := g.frames[seq]; ok { + return xdr.LedgerCloseMetaView(raw), nil + } + // Past the programmed frames. + if g.blockOnCtx { + <-ctx.Done() + return nil, ctx.Err() + } + if g.endErr != nil { + return nil, g.endErr + } + return nil, errors.New("fakeLedgerGetter: no frame for seq") +} + +// getterForSeqs builds a fakeLedgerGetter with zero-tx LCM frames for [from,to]. +func getterForSeqs(t *testing.T, from, to uint32) *fakeLedgerGetter { + t.Helper() + g := &fakeLedgerGetter{frames: map[uint32][]byte{}, maxSeq: to} + for seq := from; seq <= to; seq++ { + g.frames[seq] = zeroTxLCMBytes(t, seq) + } + return g +} + +// openLiveHotDB opens (and brackets ready) the live hot DB for a chunk via the +// production opener, returning the handle and the catalog it lives under. +func openLiveHotDB(t *testing.T, cat *catalog.Catalog, c chunk.ID) *hotchunk.DB { + t.Helper() + db, err := openHotTierForChunk(cat, c, silentLogger()) + require.NoError(t, err) + return db +} + +// seedWatermark writes a single ledgers-CF entry at seq into the chunk's hot DB +// so the indexed poll resumes at seq+1 — letting a boundary test drive the loop +// over only the last ledger or two of a chunk instead of all 10,000. The +// returned DB is the (re-opened, ready) live handle the loop then owns. Used by +// the boundary tests, whose ingestTypes are Ledgers+Txhash (no events +// contiguity requirement, so a sparse ledgers-CF watermark is valid). +func seedWatermark(t *testing.T, cat *catalog.Catalog, c chunk.ID, seq uint32) *hotchunk.DB { + t.Helper() + db := openLiveHotDB(t, cat, c) + require.NoError(t, db.Ledgers().AddLedgers(ledgerEntry(t, seq))) + require.NoError(t, db.Close()) + reopened, err := openHotTierForChunk(cat, c, silentLogger()) + require.NoError(t, err) + return reopened +} + +// drainLifecycle counts how many chunk ids the buffered lifecycle channel +// delivered after the loop returned (the loop is done, so no send races this). +func drainLifecycle(ch chan chunk.ID) []chunk.ID { + var got []chunk.ID + for { + select { + case c := <-ch: + got = append(got, c) + default: + return got + } + } +} + +// --------------------------------------------------------------------------- +// openHotTierForChunk — the bracket's open end. +// --------------------------------------------------------------------------- + +// TestOpenHotTier_CreatesBracketAndDir: a fresh open writes the dir and flips +// the key "ready"; the returned DB is empty (resume at FirstLedger). +func TestOpenHotTier_CreatesBracketAndDir(t *testing.T) { + cat, _ := testCatalog(t) + c := chunk.ID(3) + + db, err := openHotTierForChunk(cat, c, silentLogger()) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + state, err := cat.HotState(c) + require.NoError(t, err) + assert.Equal(t, geometry.HotReady, state, "open flips the key ready") + + _, statErr := os.Stat(cat.Layout().HotChunkPath(c)) + require.NoError(t, statErr, "the dir exists") + + resume, err := nextIngestLedger(db) + require.NoError(t, err) + assert.Equal(t, c.FirstLedger(), resume, "an empty resume DB resumes at the chunk's first ledger") +} + +// TestOpenHotTier_ReadyButDirMissingIsCase4 is the case-4 fatal: a "ready" key +// whose dir is gone is hot-volume loss, never auto-healed. +func TestOpenHotTier_ReadyButDirMissingIsCase4(t *testing.T) { + cat, _ := testCatalog(t) + c := chunk.ID(5) + require.NoError(t, cat.PutHotTransient(c)) + require.NoError(t, cat.FlipHotReady(c)) // key says ready, but no dir created + + _, err := openHotTierForChunk(cat, c, silentLogger()) + require.Error(t, err) + require.ErrorIs(t, err, backfill.ErrHotVolumeLost) +} + +// TestOpenHotTier_TransientRecreatesFresh: a "transient" key (crashed +// create/discard) is recovered by wiping any leftover and recreating. +func TestOpenHotTier_TransientRecreatesFresh(t *testing.T) { + cat, _ := testCatalog(t) + c := chunk.ID(2) + require.NoError(t, cat.PutHotTransient(c)) // a crash left a transient key + + db, err := openHotTierForChunk(cat, c, silentLogger()) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + state, err := cat.HotState(c) + require.NoError(t, err) + assert.Equal(t, geometry.HotReady, state) +} + +// --------------------------------------------------------------------------- +// runIngestionLoop — atomic landing. +// --------------------------------------------------------------------------- + +// TestRunIngestionLoop_LedgerLandsAcrossAllCFs: polling a short contiguous +// prefix lands each ledger atomically across the ledgers, txhash, and events +// CFs — the single watermark advances to the last committed seq, and every CF +// is readable. The getter then errs (backend crash), which the loop returns. +func TestRunIngestionLoop_LedgerLandsAcrossAllCFs(t *testing.T) { + cat, _ := testCatalog(t) + c := chunk.ID(0) + first := c.FirstLedger() + db := openLiveHotDB(t, cat, c) + + // A short contiguous prefix from the chunk's first ledger (events require + // strict contiguity from FirstLedger), then the poll runs dry and errs. + getter := getterForSeqs(t, first, first+2) + getter.endErr = errors.New("backend crashed") + ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) + + err := runIngestionLoop(context.Background(), getter, db, cat, ch, allHotTypes, silentLogger(), nil, nil) + require.Error(t, err, "poll ran past the prefix and the getter errored") + require.NotErrorIs(t, err, backfill.ErrHotVolumeLost) + + // Reopen the (loop-closed) DB and assert every CF advanced together. + reopened, err := hotchunk.Open(cat.Layout().HotChunkPath(c), c, silentLogger()) + require.NoError(t, err) + t.Cleanup(func() { _ = reopened.Close() }) + + maxSeq, ok, err := reopened.MaxCommittedSeq() + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, first+2, maxSeq, "the single watermark is the last committed seq") + + raw, err := reopened.Ledgers().GetLedgerRaw(first + 2) + require.NoError(t, err) + assert.NotEmpty(t, raw) + assert.Equal(t, uint32(0), reopened.Events().NextEventID(), "zero-tx ledgers carry no events") +} + +// --------------------------------------------------------------------------- +// runIngestionLoop — boundary notifications carry the completed chunk id. +// --------------------------------------------------------------------------- + +// TestRunIngestionLoop_BoundaryNotifiesCompletedChunk: crossing the chunk 0 -> 1 +// boundary sends chunk 0 into the buffered lifecycle channel. The watermark is +// seeded just below the boundary so the poll crosses it in one step. The buffer +// is far above the at-most-one a healthy daemon holds, so it never blocks the +// loop. +func TestRunIngestionLoop_BoundaryNotifiesCompletedChunk(t *testing.T) { + cat, _ := testCatalog(t) + c := chunk.ID(0) + c1 := c + 1 + db := seedWatermark(t, cat, c, c.LastLedger()-1) + + ingestTypes := hotchunk.Ingest{Ledgers: true, Txhash: true} + getter := &fakeLedgerGetter{frames: map[uint32][]byte{ + c.LastLedger(): zeroTxLCMBytes(t, c.LastLedger()), // boundary 0->1 + c1.FirstLedger(): zeroTxLCMBytes(t, c1.FirstLedger()), // a ledger in chunk 1 + }, endErr: errors.New("end")} + ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) + + done := make(chan error, 1) + go func() { + done <- runIngestionLoop(context.Background(), getter, db, cat, ch, ingestTypes, silentLogger(), nil, nil) + }() + + select { + case err := <-done: + require.Error(t, err, "poll ran dry") + case <-time.After(10 * time.Second): + t.Fatal("ingestion loop deadlocked") + } + + sent := drainLifecycle(ch) + assert.Equal(t, []chunk.ID{c}, sent, "the completed chunk id was sent at the boundary") +} + +// --------------------------------------------------------------------------- +// runIngestionLoop — clean shutdown vs crash (classified at the daemon top +// level: ctx-cancelled return is clean, any other error is restartable). +// --------------------------------------------------------------------------- + +// TestRunIngestionLoop_CtxCancelReturnsCtxErr: a ctx cancellation while the poll +// is blocking on the tip makes GetLedger return ctx.Err(); the loop returns that +// (the daemon top level classifies a ctx-cancelled return as a clean shutdown). +func TestRunIngestionLoop_CtxCancelReturnsCtxErr(t *testing.T) { + cat, _ := testCatalog(t) + c := chunk.ID(0) + first := c.FirstLedger() + db := openLiveHotDB(t, cat, c) + + getter := getterForSeqs(t, first, first+1) + getter.blockOnCtx = true // after the frames, behave like a live tip stream + ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) + ctx, cancel := context.WithCancel(context.Background()) + + done := make(chan error, 1) + go func() { + done <- runIngestionLoop(ctx, getter, db, cat, ch, allHotTypes, silentLogger(), nil, nil) + }() + + require.Eventually(t, func() bool { + return getter.calls.Load() >= 3 // ingested 2 frames, blocked on the 3rd + }, 5*time.Second, 5*time.Millisecond) + cancel() + + select { + case err := <-done: + require.Error(t, err) + require.ErrorIs(t, err, context.Canceled, "the loop surfaces the ctx-cancelled GetLedger error") + case <-time.After(10 * time.Second): + t.Fatal("ingestion loop did not stop on ctx cancellation") + } +} + +// TestRunIngestionLoop_GetLedgerErrorReturnsError: a GetLedger error (not a +// shutdown) propagates as a restartable failure. +func TestRunIngestionLoop_GetLedgerErrorReturnsError(t *testing.T) { + cat, _ := testCatalog(t) + c := chunk.ID(0) + first := c.FirstLedger() + db := openLiveHotDB(t, cat, c) + + boom := errors.New("backend exploded") + getter := getterForSeqs(t, first, first) + getter.yieldErrAt = first + 1 + getter.errAt = boom + ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) + + err := runIngestionLoop(context.Background(), getter, db, cat, ch, allHotTypes, silentLogger(), nil, nil) + require.Error(t, err) + require.ErrorIs(t, err, boom) + require.NotErrorIs(t, err, backfill.ErrHotVolumeLost) +} + +// --------------------------------------------------------------------------- +// runIngestionLoop — restart resumes idempotently from the derived watermark. +// --------------------------------------------------------------------------- + +// TestRunIngestionLoop_RestartResumesFromWatermark: after a first run commits a +// prefix and exits, a second run over a FRESH open of the SAME hot dir resumes +// at watermark+1 (asserted via the FIRST seq the getter is asked for) and a +// re-delivered already-committed ledger is the idempotent retry the hot stores +// tolerate — the final watermark is exactly the last delivered seq. +func TestRunIngestionLoop_RestartResumesFromWatermark(t *testing.T) { + cat, _ := testCatalog(t) + c := chunk.ID(0) + first := c.FirstLedger() + + // First run: commit [first, first+2], then the getter errs. + db1 := openLiveHotDB(t, cat, c) + getter1 := getterForSeqs(t, first, first+2) + getter1.endErr = errors.New("end") + ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) + err := runIngestionLoop(context.Background(), getter1, db1, cat, ch, allHotTypes, silentLogger(), nil, nil) + require.Error(t, err) + assert.Equal(t, first, getter1.firstSeen.Load(), "first run resumed at the chunk's first ledger") + + // Restart: re-open the live DB the way startup would. The resume point must + // be watermark+1. + db2, err := openHotTierForChunk(cat, c, silentLogger()) + require.NoError(t, err) + resume, err := nextIngestLedger(db2) + require.NoError(t, err) + assert.Equal(t, first+3, resume, "restart resumes one past the durable watermark") + + // Second run re-delivers the last already-committed ledger (idempotent) plus + // two new ones. + getter2 := getterForSeqs(t, first+2, first+5) + getter2.endErr = errors.New("end") + err = runIngestionLoop(context.Background(), getter2, db2, cat, ch, allHotTypes, silentLogger(), nil, nil) + require.Error(t, err) + assert.Equal(t, first+3, getter2.firstSeen.Load(), "second run resumed at watermark+1") + + reopened, err := hotchunk.Open(cat.Layout().HotChunkPath(c), c, silentLogger()) + require.NoError(t, err) + t.Cleanup(func() { _ = reopened.Close() }) + maxSeq, ok, err := reopened.MaxCommittedSeq() + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, first+5, maxSeq) +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go new file mode 100644 index 000000000..47f507d73 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go @@ -0,0 +1,43 @@ +package lifecycle + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// discardHotTierForChunk retires a chunk's hot DB once its cold artifacts are +// durable (or it fell past retention): transient -> rmdir+fsync parent -> delete +// key. Idempotent — a missing key is a no-op, and a crash mid-discard leaves the +// key "transient" for the next scan to finish. The caller must have closed the +// write handle (the stage runs after executePlan froze the cold artifacts). +func discardHotTierForChunk(cat *catalog.Catalog, chunkID chunk.ID) error { + state, err := cat.HotState(chunkID) + if err != nil { + return fmt.Errorf("streaming: read hot key chunk %s: %w", chunkID, err) + } + if state == "" { + return nil + } + if putErr := cat.PutHotTransient(chunkID); putErr != nil { + return fmt.Errorf("streaming: mark hot transient chunk %s: %w", chunkID, putErr) + } + + dir := cat.Layout().HotChunkPath(chunkID) + if rmErr := os.RemoveAll(dir); rmErr != nil { + return fmt.Errorf("streaming: rmdir hot dir %s: %w", dir, rmErr) + } + // rmdir must be durable BEFORE the key delete: the key outlives the dir, so a + // crash re-runs the discard rather than leaving a key-less dir. + if syncErr := geometry.FsyncDir(filepath.Dir(dir)); syncErr != nil { + return fmt.Errorf("streaming: fsync hot parent dir %s: %w", filepath.Dir(dir), syncErr) + } + if delErr := cat.DeleteHotKey(chunkID); delErr != nil { + return fmt.Errorf("streaming: delete hot key chunk %s: %w", chunkID, delErr) + } + return nil +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard_test.go new file mode 100644 index 000000000..8aa6e4564 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard_test.go @@ -0,0 +1,30 @@ +package lifecycle + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// TestDiscardHotTier_RemovesDirAndKey retires the bracket: the key is deleted +// and the dir is gone. A second discard is a no-op. +func TestDiscardHotTier_RemovesDirAndKey(t *testing.T) { + cat, _ := testCatalog(t) + c := chunk.ID(4) + db := openLiveHotDB(t, cat, c) + require.NoError(t, db.Close()) + + require.NoError(t, discardHotTierForChunk(cat, c)) + + has, err := hotKeyExists(cat, c) + require.NoError(t, err) + assert.False(t, has, "the hot key is deleted") + _, statErr := os.Stat(cat.Layout().HotChunkPath(c)) + assert.True(t, os.IsNotExist(statErr), "the dir is removed") + + require.NoError(t, discardHotTierForChunk(cat, c), "second discard is a no-op") +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go new file mode 100644 index 000000000..9c00a9188 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go @@ -0,0 +1,177 @@ +package lifecycle + +import ( + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// The discard and prune eligibility scans. Each returns zero-arg op closures the +// tick calls in order. Both are PURE READS — eligibility comes from durable keys +// alone, so re-running against the same snapshot yields nothing (quiescence). + +// eligibleDiscardOps returns a discard closure per hot DB the cold artifacts now +// fully serve (or that fell past retention). Per chunk: below the floor → discard; +// complete (last <= through), nothing pending, and the index covers it → discard; +// otherwise (live, or frozen awaiting coverage) → leave alone. +// discardHotTierForChunk is idempotent, so a crash between freeze and discard +// self-heals next tick. +func eligibleDiscardOps(cfg LifecycleConfig, cat *catalog.Catalog, through uint32) ([]func() error, error) { + earliest, _, err := cat.EarliestLedger() + if err != nil { + return nil, err + } + // The "past retention" test shares one definition with the read gate + // (retention.go), so a hot DB retires on exactly the floor the reader stops + // admitting at. A shortened retentionChunks raises the floor at once. + gate := NewRetentionFloor(through, cfg.RetentionChunks, earliest) + + hot, err := cat.HotChunkKeys() + if err != nil { + return nil, err + } + + var ops []func() error + for _, c := range hot { + last := c.LastLedger() + switch { + case gate.Excludes(c): + ops = append(ops, func() error { return discardHotTierForChunk(cat, c) }) + case last <= through: + pending, perr := pendingArtifacts(c, cfg, cat) + if perr != nil { + return nil, perr + } + covers, cerr := indexCovers(c, cat) + if cerr != nil { + return nil, cerr + } + if pending.Empty() && covers { + ops = append(ops, func() error { return discardHotTierForChunk(cat, c) }) + } + // else: frozen awaiting coverage, or still producing — leave alone. + } + // default (last > through): the live chunk or above — ingestion's, not ours. + } + return ops, nil +} + +// pendingArtifacts lists which outputs chunk still needs: ledgers and events must +// be frozen; txhash/.bin is exempt when the window's index already covers the +// chunk (after finalization the chunk:c:txhash key is demoted/swept, so +// regenerating the .bin would orphan it). +func pendingArtifacts(c chunk.ID, cfg LifecycleConfig, cat *catalog.Catalog) (catalog.ArtifactSet, error) { + var need catalog.ArtifactSet + for _, kind := range []geometry.Kind{geometry.KindLedgers, geometry.KindEvents} { + state, err := cat.State(c, kind) + if err != nil { + return need, err + } + if state != geometry.StateFrozen { + need = need.Add(kind) + } + } + txState, err := cat.State(c, geometry.KindTxHash) + if err != nil { + return need, err + } + if txState != geometry.StateFrozen { + covers, cerr := indexCovers(c, cat) + if cerr != nil { + return need, cerr + } + if !covers { + need = need.Add(geometry.KindTxHash) + } + } + return need, nil +} + +// indexCovers reports whether the durable .idx for chunk's window already hashes +// it — the frozen coverage's [Lo, Hi] contains c. +func indexCovers(c chunk.ID, cat *catalog.Catalog) (bool, error) { + fk, ok, err := cat.FrozenTxHashIndex(cat.TxHashIndexLayout().TxHashIndexID(c)) + if err != nil { + return false, err + } + return ok && fk.Lo <= c && c <= fk.Hi, nil +} + +// eligiblePruneOps is the system's only file-deleter, key-driven, covering both +// key families. It returns sweep closures (SweepTxHashIndexKey per index key, one +// batched SweepChunkArtifacts for the chunk family). "Below the floor" is the +// gate predicate shared with the discard scan and read path, so prune deletes +// exactly what the reader has stopped admitting. +func eligiblePruneOps(cfg LifecycleConfig, cat *catalog.Catalog, through uint32) ([]func() error, error) { + earliest, _, err := cat.EarliestLedger() + if err != nil { + return nil, err + } + gate := NewRetentionFloor(through, cfg.RetentionChunks, earliest) + + var ops []func() error + + // Index family: transient debris from any window, plus frozen keys below the floor. + idxKeys, err := cat.AllTxHashIndexKeys() + if err != nil { + return nil, err + } + for _, cov := range idxKeys { + switch { + case cov.State == geometry.StateFreezing || cov.State == geometry.StatePruning: + // Transient debris (a crashed build or unfinished demotion). Safe only + // because no build is in flight when this scan runs (it follows + // executePlan's return, and backfill finishes before the loop starts). + ops = append(ops, func() error { return cat.SweepTxHashIndexKey(cov) }) + case gate.Excludes(cat.TxHashIndexLayout().LastChunk(cov.Index)): + // Frozen index key below the floor; the sweep demotes it first. + ops = append(ops, func() error { return cat.SweepTxHashIndexKey(cov) }) + } + } + + // Chunk family: swept in one batch. + refs, err := cat.ChunkArtifactKeys() + if err != nil { + return nil, err + } + var sweep []catalog.ArtifactRef + for _, ref := range refs { + switch { + case gate.Excludes(ref.Chunk): + // Past retention: any state goes. + sweep = append(sweep, ref) + case ref.State == geometry.StatePruning: + // In-retention .bin demoted by its window's terminal commit batch. + sweep = append(sweep, ref) + case ref.Kind == geometry.KindTxHash: + // A frozen/freezing chunk:c:txhash inside a FINALIZED window: re-derived + // (or left mid-write) by a widening backfill that crashed before its + // terminal rebuild, then abandoned when retention narrowed. The terminal + // .idx provably covers the chunk and is never re-materialized, so it's + // redundant. + redundant, rerr := txhashRedundantInFinalizedWindow(cat, ref.Chunk) + if rerr != nil { + return nil, rerr + } + if redundant { + sweep = append(sweep, ref) + } + } + } + if len(sweep) > 0 { + ops = append(ops, func() error { return cat.SweepChunkArtifacts(sweep) }) + } + return ops, nil +} + +// txhashRedundantInFinalizedWindow reports whether c's window has a TERMINAL +// frozen index coverage (Hi == the window's last chunk) — the branch that makes +// INV-2's no-leftover-txhash-keys clause self-healing, not merely auditable. +func txhashRedundantInFinalizedWindow(cat *catalog.Catalog, c chunk.ID) (bool, error) { + w := cat.TxHashIndexLayout().TxHashIndexID(c) + fk, ok, err := cat.FrozenTxHashIndex(w) + if err != nil { + return false, err + } + return ok && cat.TxHashIndexLayout().IsTerminalCoverage(fk), nil +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go new file mode 100644 index 000000000..1e41fe195 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go @@ -0,0 +1,262 @@ +package lifecycle + +import ( + "bytes" + "context" + "errors" + "fmt" + "iter" + "os" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" + supportlog "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/go-stellar-sdk/xdr" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/metastore" +) + +// This file provides the shared test scaffolding the lifecycle tests need. The +// catalog/fixture helpers are copied verbatim from the root fullhistory package's +// helpers_test.go (which still serves the root tests). The hot-tier helpers +// (allHotTypes / openHotTierForChunk / openLiveHotDB / NewRocksHotProbe) are +// test-local equivalents of the production hot-source primitives that live in the +// root fullhistory package — the lifecycle package cannot import root (root imports +// lifecycle), so the lifecycle tests rebuild them over the same public store APIs. + +// testCPI is the tx-hash index width tests build layouts with; equals the +// production constant so on-disk geometry reads back identically. +const testCPI = geometry.ChunksPerTxhashIndex + +func silentLogger() *supportlog.Entry { + var buf bytes.Buffer + log := supportlog.New() + log.SetLevel(logrus.DebugLevel) + log.SetOutput(&buf) + return log +} + +// newTestCatalog builds a Catalog over a real metastore on temp dirs with +// cpi-wide tx-hash indexes; returns the catalog, open store, and artifact root. +func newTestCatalog(t *testing.T, cpi uint32) (*catalog.Catalog, *metastore.Store, string) { + t.Helper() + metaDir := t.TempDir() + artifactRoot := t.TempDir() + + store, err := metastore.New(filepath.Join(metaDir, "rocksdb"), silentLogger()) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + + idxLayout, err := geometry.NewTxHashIndexLayout(cpi) + require.NoError(t, err) + + return catalog.NewCatalog(store, geometry.NewLayout(artifactRoot), idxLayout), store, artifactRoot +} + +// testCatalog builds a catalog with the default (wide) tx-hash index, returning it +// and the artifact root. +func testCatalog(t *testing.T) (*catalog.Catalog, string) { + t.Helper() + cat, _, root := newTestCatalog(t, testCPI) + return cat, root +} + +// smallTxHashIndexCatalog builds a test catalog whose indexes are cpi chunks +// wide, so a "terminal" (full-index) build needs only a few chunks. Returns the +// catalog and the artifact root. +func smallTxHashIndexCatalog(t *testing.T, cpi uint32) (*catalog.Catalog, string) { + t.Helper() + cat, _, root := newTestCatalog(t, cpi) + return cat, root +} + +// freezeKinds flips the given per-chunk kinds to "frozen" via the one-write protocol. +func freezeKinds(t *testing.T, cat *catalog.Catalog, chunkID chunk.ID, kinds ...geometry.Kind) { + t.Helper() + require.NoError(t, cat.MarkChunkFreezing(chunkID, kinds...)) + require.NoError(t, cat.FlipChunkFrozen(chunkID, kinds...)) +} + +// freezeCoverage marks and commits a frozen index coverage [lo, hi] for index w. +func freezeCoverage(t *testing.T, cat *catalog.Catalog, w geometry.TxHashIndexID, lo, hi chunk.ID) { + t.Helper() + cov, err := cat.MarkTxHashIndexFreezing(w, lo, hi) + require.NoError(t, err) + require.NoError(t, cat.CommitTxHashIndex(cov)) +} + +// zeroTxLCMBytes builds wire bytes of a minimal valid zero-tx V2 LedgerCloseMeta; +// zero-tx keeps a full 10k-ledger chunk pass cheap. +func zeroTxLCMBytes(t *testing.T, seq uint32) []byte { + t.Helper() + lcm := xdr.LedgerCloseMeta{ + V: 2, + V2: &xdr.LedgerCloseMetaV2{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + ScpValue: xdr.StellarValue{CloseTime: xdr.TimePoint(0)}, + LedgerSeq: xdr.Uint32(seq), + }, + }, + TxSet: xdr.GeneralizedTransactionSet{ + V: 1, + V1TxSet: &xdr.TransactionSetV1{Phases: nil}, + }, + TxProcessing: nil, + }, + } + raw, err := lcm.MarshalBinary() + require.NoError(t, err) + return raw +} + +// --------------------------------------------------------------------------- +// Hot-tier test scaffolding: test-local equivalents of the root package's +// production hot-source primitives (ingest.go's openHotTierForChunk/allHotTypes +// and hotsource.go's rocksHotProbe/NewRocksHotProbe). They use only the public +// hotchunk/ledger/catalog/backfill APIs the production code uses, so a lifecycle +// test reads and freezes the SAME on-disk hot DB the real daemon would, without +// importing the root fullhistory package (which would be an import cycle). +// --------------------------------------------------------------------------- + +// allHotTypes is the hot tier's ingest selection (all three CFs), mirroring the +// production ingest config. +var allHotTypes = hotchunk.Ingest{Ledgers: true, Txhash: true, Events: true} + +// openHotTierForChunk creates a "ready" shared hot DB for chunkID under the +// hot:chunk bracket (transient -> create -> ready) and returns an open handle the +// caller owns. The test equivalent of the production opener, trimmed to the +// create branch the lifecycle tests need (no crash-recovery / fsync — those edges +// are covered by the root ingest_test.go opener tests). +func openHotTierForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlog.Entry) (*hotchunk.DB, error) { + dir := cat.Layout().HotChunkPath(chunkID) + if err := os.RemoveAll(dir); err != nil { + return nil, fmt.Errorf("wipe leftover hot dir %s: %w", dir, err) + } + if err := cat.PutHotTransient(chunkID); err != nil { + return nil, fmt.Errorf("mark hot transient chunk %s: %w", chunkID, err) + } + db, err := hotchunk.Open(dir, chunkID, logger) + if err != nil { + return nil, fmt.Errorf("create hot DB chunk %s: %w", chunkID, err) + } + if err := cat.FlipHotReady(chunkID); err != nil { + _ = db.Close() + return nil, fmt.Errorf("flip hot ready chunk %s: %w", chunkID, err) + } + return db, nil +} + +// openLiveHotDB opens (and brackets ready) the live hot DB for a chunk via the +// test opener, returning the handle. +func openLiveHotDB(t *testing.T, cat *catalog.Catalog, c chunk.ID) *hotchunk.DB { + t.Helper() + db, err := openHotTierForChunk(cat, c, silentLogger()) + require.NoError(t, err) + return db +} + +// NewRocksHotProbe returns a test backfill.HotProbe over real per-chunk hot DBs — +// the test equivalent of the production probe. It opens the chunk's shared hot DB +// read-only and answers MaxCommittedSeq / Source / Close over it. +func NewRocksHotProbe(hotChunkPath func(chunk.ID) string, logger *supportlog.Entry) backfill.HotProbe { + return &rocksHotProbe{hotRoot: hotChunkPath, logger: logger} +} + +type rocksHotProbe struct { + hotRoot func(chunkID chunk.ID) string + logger *supportlog.Entry +} + +func (p *rocksHotProbe) OpenHotChunk(chunkID chunk.ID) (backfill.HotChunk, bool, error) { + dir := p.hotRoot(chunkID) + if _, err := os.Stat(dir); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + return nil, false, fmt.Errorf("stat hot dir %s: %w", dir, err) + } + db, err := hotchunk.OpenReadOnly(dir, chunkID, p.logger) + if err != nil { + return nil, false, fmt.Errorf("open hot chunk DB: %w", err) + } + return &rocksHotChunk{chunkID: chunkID, db: db}, true, nil +} + +type rocksHotChunk struct { + chunkID chunk.ID + db *hotchunk.DB +} + +func (h *rocksHotChunk) MaxCommittedSeq() (uint32, bool, error) { + seq, ok, err := h.db.MaxCommittedSeq() + if err != nil { + return 0, false, fmt.Errorf("hot DB max committed seq: %w", err) + } + return seq, ok, nil +} + +func (h *rocksHotChunk) Source() ledgerbackend.LedgerStream { + return &hotLedgerStream{store: h.db.Ledgers()} +} + +func (h *rocksHotChunk) Close() error { + if h.db == nil { + return nil + } + return h.db.Close() +} + +// hotLedgerStream is a ledgerbackend.LedgerStream backed by a ledger.HotStore so +// the cold pipeline can freeze a just-closed chunk straight from its hot DB. +type hotLedgerStream struct { + store *ledger.HotStore +} + +var _ ledgerbackend.LedgerStream = (*hotLedgerStream)(nil) + +func (st *hotLedgerStream) RawLedgers( + ctx context.Context, r ledgerbackend.Range, _ ...ledgerbackend.StreamOption, +) iter.Seq2[[]byte, error] { + return func(yield func([]byte, error) bool) { + if st.store == nil { + yield(nil, errors.New("lifecycle test: hotLedgerStream has no store")) + return + } + to := r.To() + if !r.Bounded() { + last, ok, err := st.store.LastSeq() + if err != nil { + yield(nil, err) + return + } + if !ok { + return + } + to = last + } + for e, ierr := range st.store.IterateLedgers(r.From(), to) { + if cerr := ctx.Err(); cerr != nil { + yield(nil, cerr) + return + } + if ierr != nil { + yield(nil, ierr) + return + } + if !yield(e.Bytes, nil) { + return + } + } + } +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/hot_fakes_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/hot_fakes_test.go new file mode 100644 index 000000000..a8fb1f332 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/hot_fakes_test.go @@ -0,0 +1,101 @@ +package lifecycle + +import ( + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" + "github.com/stretchr/testify/require" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// fakeHotChunk is a test backfill.HotChunk: a hand-set MaxCommittedSeq + an +// injectable LedgerStream source, counting closes when closedTo is non-nil. +type fakeHotChunk struct { + maxSeq uint32 + present bool + maxErr error + source ledgerbackend.LedgerStream + closedTo *atomic.Int32 +} + +func (h *fakeHotChunk) MaxCommittedSeq() (uint32, bool, error) { + return h.maxSeq, h.present, h.maxErr +} +func (h *fakeHotChunk) Source() ledgerbackend.LedgerStream { return h.source } +func (h *fakeHotChunk) Close() error { + if h.closedTo != nil { + h.closedTo.Add(1) + } + return nil +} + +// fakeHotProbe is a test backfill.HotProbe: returns its fake chunk when ok, an +// error when openErr is set, or (nil,false,nil) for "no ready hot DB". Counts +// opens via openedTo when non-nil. +type fakeHotProbe struct { + chunk *fakeHotChunk + ok bool + openErr error + openedTo *atomic.Int32 +} + +func (p *fakeHotProbe) OpenHotChunk(chunk.ID) (backfill.HotChunk, bool, error) { + if p.openedTo != nil { + p.openedTo.Add(1) + } + if p.openErr != nil { + return nil, false, p.openErr + } + if !p.ok { + return nil, false, nil + } + return p.chunk, true, nil +} + +// writeArtifact writes a placeholder artifact file at path (creating parents), +// so a test can assert presence/absence around the catalog protocol. +func writeArtifact(t *testing.T, path string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte("artifact"), 0o644)) +} + +// hotKeyExists reports whether chunk c has a hot:chunk key (any value). The +// catalog's key existence read is unexported; this is the streaming-package test +// shim over the public HotState ("" ⇒ absent). +func hotKeyExists(cat *catalog.Catalog, c chunk.ID) (bool, error) { + s, err := cat.HotState(c) + return s != "", err +} + +func TestRoundTripHotKeys(t *testing.T) { + cat, _ := testCatalog(t) + + state, err := cat.HotState(7) + require.NoError(t, err) + require.Equal(t, geometry.HotState(""), state) + + require.NoError(t, cat.PutHotTransient(7)) + state, err = cat.HotState(7) + require.NoError(t, err) + require.Equal(t, geometry.HotTransient, state) + + require.NoError(t, cat.FlipHotReady(7)) + state, err = cat.HotState(7) + require.NoError(t, err) + require.Equal(t, geometry.HotReady, state) + + require.NoError(t, cat.DeleteHotKey(7)) + state, err = cat.HotState(7) + require.NoError(t, err) + require.Equal(t, geometry.HotState(""), state) + // Idempotent on a missing key. + require.NoError(t, cat.DeleteHotKey(7)) +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go new file mode 100644 index 000000000..52bc80d66 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -0,0 +1,271 @@ +package lifecycle + +import ( + "context" + "log" + "time" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/observability" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// The lifecycle tick runs three stages in order: (1) plan-and-execute (the same +// resolve+executePlan as catch-up, over [floor, lastChunk]); (2) discard scan; +// (3) prune scan. The tick is a pure function of the catalog — the two goroutines +// share no state. +// +// The retention floor has two roles with OPPOSITE safe directions (design +// "Lifecycle"): as a RETENTION boundary erring low is harmless (an extra chunk +// lingers, or a read returns not-found via the missing-file rule); as a +// PRODUCTION boundary erring low is DANGEROUS (it would plan a build below +// existing storage from an unvalidated source). So the plan range never starts +// below storage — start is RAISED to lowestMaterializedChunk; extending the +// bottom is catch-up's job, producibility enforced lazily per chunk. + +// LifecycleConfig bundles the tick/loop dependencies. It composes the scheduler's +// ExecConfig (shared postconditions + worker pool with catch-up) plus the +// retention knob and an injectable fatal sink. +type LifecycleConfig struct { + backfill.ExecConfig + + // RetentionChunks bounds the sliding retention floor's width. 0 disables the + // sliding floor (the fixed earliest-ledger floor alone applies). + RetentionChunks uint32 + + // Fatalf aborts the daemon on a tick op failure. WithLifecycleDefaults fills + // log.Fatalf when unset; tests override it. + Fatalf func(format string, args ...any) +} + +// WithLifecycleDefaults returns a copy with ExecConfig and Fatalf defaults +// applied. Called once at startup before launching the loop. +func (cfg LifecycleConfig) WithLifecycleDefaults() LifecycleConfig { + cfg.ExecConfig = cfg.ExecConfig.WithDefaults() + if cfg.Fatalf == nil { + cfg.Fatalf = log.Fatalf + } + return cfg +} + +// lastCompleteChunkAtID maps geometry.LastCompleteChunkAt to a chunk.ID; +// ok=false when no complete chunk exists (negative result). +func lastCompleteChunkAtID(ledger uint32) (chunk.ID, bool) { + c := geometry.LastCompleteChunkAt(ledger) + if c < 0 { + return 0, false + } + return chunk.ID(c), true //nolint:gosec // c >= 0 +} + +// lowestMaterializedChunk is the lowest chunk holding any chunk:* artifact key +// or hot:chunk key — the bottom of existing storage, and the production-boundary +// anchor (the plan never starts below it). ok=false on an empty catalog. +func lowestMaterializedChunk(cat *catalog.Catalog) (chunk.ID, bool, error) { + lowest := chunk.ID(0) + found := false + note := func(c chunk.ID) { + if !found || c < lowest { + lowest, found = c, true + } + } + + refs, err := cat.ChunkArtifactKeys() + if err != nil { + return 0, false, err + } + for _, ref := range refs { + note(ref.Chunk) + } + + hot, err := cat.HotChunkKeys() + if err != nil { + return 0, false, err + } + for _, c := range hot { + note(c) + } + return lowest, found, nil +} + +// runLifecycleTick runs one tick over the three stages for just-completed chunk +// lastChunk. through = lastChunk.LastLedger() is the single snapshot every stage +// shares, so a boundary committing mid-tick can't make stages contradict (it's +// next tick's work). Plan range is [floor, lastChunk] (start raised to storage); +// discard/prune key off through. +// +// CLEAN-SHUTDOWN (binding): on an op error with ctx cancelled, return WITHOUT +// Fatalf — cancellation is a shutdown, not a failure. Only a genuine failure +// (ctx still live) aborts via Fatalf. +func runLifecycleTick(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog, lastChunk chunk.ID) { + metrics := observability.MetricsOrNop(cfg.Metrics) + logger := cfg.Logger + + // The one snapshot every stage shares. + through := lastChunk.LastLedger() + + earliest, _, err := cat.EarliestLedger() + if err != nil { + if ctx.Err() != nil { + return + } + cfg.Fatalf("streaming: lifecycle tick: read earliest ledger: %v", err) + return + } + floor := EffectiveRetentionFloor(through, cfg.RetentionChunks, earliest) + + // Progress gauges: derived last-committed ledger and effective retention floor. + metrics.LastCommitted(through, floor) + if logger != nil { + logger.WithField("through", through). + WithField("floor", floor). + Debug("streaming: lifecycle tick — derived snapshot") + } + + // Plan start = chunkID(floor), RAISED to lowestMaterializedChunk when higher + // — the production-boundary rule (never plan below existing storage). + start := ChunkIDOfLedger(floor) + low, hasLow, err := lowestMaterializedChunk(cat) + if err != nil { + if ctx.Err() != nil { + return + } + cfg.Fatalf("streaming: lifecycle tick: lowest materialized chunk: %v", err) + return + } + if hasLow && int64(low) > start { + start = int64(low) + } + + // Stage 1 — plan-and-execute (freeze + index fold). + // + // rangeEnd is lastChunk CLAMPED to the highest durably-complete chunk: the + // production stage must never target the live or not-yet-complete chunk (whose + // hot DB ingestion holds open). In the running daemon lastChunk IS that chunk, + // so the clamp is a no-op; it only bites on seed/young-network/recovery edges. + // No complete chunk ⇒ empty range, production skipped, scans below still run. + freezeStart := time.Now() + durableThrough, derr := LastCommittedLedger(cat, nil) // chunk-granularity, no hot DB read + if derr != nil { + if ctx.Err() != nil { + return + } + cfg.Fatalf("streaming: lifecycle tick: derive durable through: %v", derr) + return + } + highestComplete, haveComplete := lastCompleteChunkAtID(durableThrough) + rangeEnd := lastChunk + if haveComplete && highestComplete < rangeEnd { + rangeEnd = highestComplete + } + if haveComplete && start >= 0 && start <= int64(rangeEnd) { + // Plan-and-execute over [start, rangeEnd] via the same entry point catch-up + // uses (resolve → executePlan → Freeze metric, recorded internally). + if eerr := backfill.RunBackfill(ctx, cfg.ExecConfig, chunk.ID(start), rangeEnd); eerr != nil { //nolint:gosec // start >= 0 + // CLEAN-SHUTDOWN: a cancelled ctx makes RunBackfill return ctx.Err() — + // a shutdown, not an op failure. Return before any Fatalf. + if ctx.Err() != nil { + return + } + cfg.Fatalf("streaming: lifecycle tick: run backfill [%d,%s]: %v", start, rangeEnd, eerr) + return + } + } else { + // No complete chunk in range: skip production but report an empty freeze so + // the empty-tick rate stays visible. Scans below still run. + metrics.Freeze(time.Since(freezeStart)) + } + + // Stage 2 — discard scan. + discardStart := time.Now() + discardOps, err := eligibleDiscardOps(cfg, cat, through) + if err != nil { + if ctx.Err() != nil { + return + } + cfg.Fatalf("streaming: lifecycle tick: eligible discard ops: %v", err) + return + } + for _, op := range discardOps { + if oerr := op(); oerr != nil { + if ctx.Err() != nil { + return + } + cfg.Fatalf("streaming: lifecycle tick: discard op: %v", oerr) + return + } + } + metrics.Discard(len(discardOps), time.Since(discardStart)) + if logger != nil && len(discardOps) > 0 { + logger.WithField("discarded", len(discardOps)).Info("streaming: lifecycle discard stage complete") + } + + // Live hot-chunk gauge after the discard stage. + if hot, herr := cat.HotChunkKeys(); herr == nil { + metrics.LiveHotChunks(len(hot)) + } + + // Stage 3 — prune scan. + pruneStart := time.Now() + pruneOps, err := eligiblePruneOps(cfg, cat, through) + if err != nil { + if ctx.Err() != nil { + return + } + cfg.Fatalf("streaming: lifecycle tick: eligible prune ops: %v", err) + return + } + for _, op := range pruneOps { + if oerr := op(); oerr != nil { + if ctx.Err() != nil { + return + } + cfg.Fatalf("streaming: lifecycle tick: prune op: %v", oerr) + return + } + } + metrics.Prune(len(pruneOps), time.Since(pruneStart)) + if logger != nil && len(pruneOps) > 0 { + logger.WithField("pruned", len(pruneOps)).Info("streaming: lifecycle prune stage complete") + } + + // Cold-tier footprint gauge after the prune stage. + if bytes, berr := observability.MeasureColdTierBytes(cat.Layout()); berr == nil { + metrics.ColdTierBytes(bytes) + } +} + +// LifecycleQueueDepth is the notification buffer depth — far above the at-most-one +// boundary a healthy daemon holds in flight. A FULL buffer means freeze has fallen +// this many boundaries behind ingestion, a fatal condition notify() reports. +const LifecycleQueueDepth = 8 + +// lifecycleLoop is the event-driven lifecycle goroutine. Each notification carries +// the just-completed chunk id; the loop drains the buffer to the most-recent id +// (one tick over [floor, lastChunk] subsumes the rest) and runs one tick. It +// selects on both ctx.Done() and the channel, so it never blocks or fatals on +// shutdown. +func lifecycleLoop(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog, ch <-chan chunk.ID) { + for { + select { + case <-ctx.Done(): + return + case lastChunk := <-ch: + // Drain to the most-recent queued chunk: one tick over [floor, lastChunk] + // subsumes every earlier boundary still sitting in the buffer. + drain: + for { + select { + case lastChunk = <-ch: + case <-ctx.Done(): + return + default: + break drain + } + } + runLifecycleTick(ctx, cfg, cat, lastChunk) + } + } +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_arith_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_arith_test.go new file mode 100644 index 000000000..2c71ff8fb --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_arith_test.go @@ -0,0 +1,118 @@ +package lifecycle + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// --------------------------------------------------------------------------- +// Arithmetic: geometry.LastCompleteChunkAt, EffectiveRetentionFloor. +// --------------------------------------------------------------------------- + +func TestLastCompleteChunkAt(t *testing.T) { + tests := []struct { + name string + ledger uint32 + want int64 + }{ + {"below first chunk's last ledger => sentinel -1", chunk.ID(0).LastLedger() - 1, -1}, + {"genesis sentinel (FirstLedgerSeq-1) => -1", chunk.FirstLedgerSeq - 1, -1}, + {"ledger 0 does not underflow => -1", 0, -1}, + {"chunk 0's last ledger => 0", chunk.ID(0).LastLedger(), 0}, + {"chunk 0's last ledger + 1 (into chunk 1) => still 0", chunk.ID(0).LastLedger() + 1, 0}, + {"chunk 5's last ledger => 5", chunk.ID(5).LastLedger(), 5}, + {"the doc's example 10_001 => 0", 10_001, 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, geometry.LastCompleteChunkAt(tc.ledger)) + }) + } +} + +func TestEffectiveRetentionFloor(t *testing.T) { + genesis := uint32(chunk.FirstLedgerSeq) + tests := []struct { + name string + upperBound uint32 + retentionChunks uint32 + earliest uint32 + want uint32 + }{ + { + name: "no sliding (retention 0): earliest floor wins", + upperBound: chunk.ID(100).LastLedger(), + retentionChunks: 0, + earliest: chunk.ID(10).FirstLedger(), + want: chunk.ID(10).FirstLedger(), + }, + { + name: "no sliding, no earliest pin: genesis", + upperBound: chunk.ID(100).LastLedger(), + retentionChunks: 0, + earliest: 0, + want: genesis, + }, + { + name: "sliding floor leads when above earliest", + upperBound: chunk.ID(100).LastLedger(), // last complete chunk = 100 + retentionChunks: 10, // floor chunk = 100-10+1 = 91 + earliest: 0, + want: chunk.ID(91).FirstLedger(), + }, + { + name: "earliest floor leads when above the sliding floor", + upperBound: chunk.ID(100).LastLedger(), + retentionChunks: 10, // sliding floor chunk = 91 + earliest: chunk.ID(95).FirstLedger(), // higher + want: chunk.ID(95).FirstLedger(), + }, + { + name: "retention wider than history clamps to chunk 0, never wraps", + upperBound: chunk.ID(3).LastLedger(), + retentionChunks: 1000, // sliding chunk = 3-1000+1 < 0 => clamp to chunk 0 + earliest: 0, + want: chunk.ID(0).FirstLedger(), + }, + { + name: "young store (upperBound below first chunk) clamps to chunk 0", + upperBound: chunk.FirstLedgerSeq + 5, // no complete chunk yet + retentionChunks: 5, + earliest: 0, + want: chunk.ID(0).FirstLedger(), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, EffectiveRetentionFloor(tc.upperBound, tc.retentionChunks, tc.earliest)) + }) + } +} + +// --------------------------------------------------------------------------- +// lowestMaterializedChunk. +// --------------------------------------------------------------------------- + +func TestLowestMaterializedChunk(t *testing.T) { + t.Run("empty catalog => ok=false", func(t *testing.T) { + cat, _ := testCatalog(t) + _, ok, err := lowestMaterializedChunk(cat) + require.NoError(t, err) + require.False(t, ok) + }) + + t.Run("min over chunk artifact keys and hot keys", func(t *testing.T) { + cat, _ := testCatalog(t) + freezeKinds(t, cat, 7, geometry.KindLedgers) // chunk artifact key at 7 + require.NoError(t, cat.PutHotTransient(4)) // hot key at 4 (lower) + freezeKinds(t, cat, 9, geometry.KindEvents) + low, ok, err := lowestMaterializedChunk(cat) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, chunk.ID(4), low) + }) +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go new file mode 100644 index 000000000..5df02fa1c --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go @@ -0,0 +1,194 @@ +package lifecycle + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stellar/go-stellar-sdk/keypair" + "github.com/stellar/go-stellar-sdk/network" + "github.com/stellar/go-stellar-sdk/xdr" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// lifecyclePassphrase is the network passphrase the one-tx fixture hashes +// against (any stable value works; the index only needs deterministic hashes). +const lifecyclePassphrase = network.PublicNetworkPassphrase + +// oneTxLCMRand builds the wire bytes of a V2 LedgerCloseMeta carrying ONE +// transaction for seq, so a chunk ingested with at least one such ledger yields +// a NON-empty txhash .bin — streamhash refuses to build a cold index over zero +// keys (txhash.ErrEmptyBuildSet), so a fully zero-tx chunk cannot exercise the +// real index fold. Mirrors ingest_test's buildLCMReturningHashes, trimmed to one +// tx. +func oneTxLCMRand(t *testing.T, seq uint32) []byte { + t.Helper() + envelope := xdr.TransactionEnvelope{ + Type: xdr.EnvelopeTypeEnvelopeTypeTx, + V1: &xdr.TransactionV1Envelope{ + Tx: xdr.Transaction{ + SourceAccount: xdr.MustMuxedAddress(keypair.MustRandom().Address()), + Ext: xdr.TransactionExt{V: 1, SorobanData: &xdr.SorobanTransactionData{}}, + }, + }, + } + hash, err := network.HashTransactionInEnvelope(envelope, lifecyclePassphrase) + require.NoError(t, err) + + comp := []xdr.TxSetComponent{{ + Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, + TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ + Txs: []xdr.TransactionEnvelope{envelope}, + }, + }} + opResults := []xdr.OperationResult{} + lcm := xdr.LedgerCloseMeta{ + V: 2, + V2: &xdr.LedgerCloseMetaV2{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + ScpValue: xdr.StellarValue{CloseTime: xdr.TimePoint(0)}, + LedgerSeq: xdr.Uint32(seq), + }, + }, + TxSet: xdr.GeneralizedTransactionSet{ + V: 1, + V1TxSet: &xdr.TransactionSetV1{Phases: []xdr.TransactionPhase{{V: 0, V0Components: &comp}}}, + }, + TxProcessing: []xdr.TransactionResultMetaV1{{ + TxApplyProcessing: xdr.TransactionMeta{ + V: 4, + V4: &xdr.TransactionMetaV4{Operations: []xdr.OperationMetaV2{}}, + }, + Result: xdr.TransactionResultPair{ + TransactionHash: hash, + Result: xdr.TransactionResult{ + FeeCharged: 100, + Result: xdr.TransactionResultResult{Code: xdr.TransactionResultCodeTxSuccess, Results: &opResults}, + }, + }, + }}, + }, + } + raw, err := lcm.MarshalBinary() + require.NoError(t, err) + return raw +} + +// ingestFullHotChunk creates a "ready" hot DB for chunk c and ingests every +// ledger in the chunk (all CFs, contiguous from FirstLedger), then closes the +// write handle — the post-boundary state the lifecycle freezes from. The hot +// key is left "ready" and the dir is on disk, as the boundary handoff leaves it. +func ingestFullHotChunk(t *testing.T, cat *catalog.Catalog, c chunk.ID) { + t.Helper() + db := openLiveHotDB(t, cat, c) + for seq := c.FirstLedger(); seq <= c.LastLedger(); seq++ { + // The first ledger carries one tx so the chunk's txhash .bin is non-empty + // (streamhash refuses a zero-key index); the rest stay zero-tx for speed. + var raw []byte + if seq == c.FirstLedger() { + raw = oneTxLCMRand(t, seq) + } else { + raw = zeroTxLCMBytes(t, seq) + } + _, err := db.IngestLedger(seq, xdr.LedgerCloseMetaView(raw), allHotTypes) + require.NoError(t, err) + } + require.NoError(t, db.Close()) // release the write handle (boundary handoff) +} + +// lifecycleTestConfig wires a LifecycleConfig over the real production primitives +// (a real RocksHotProbe over the catalog's hot layout) plus a fatal recorder so a +// tick abort is observable instead of killing the test process. +func lifecycleTestConfig(t *testing.T, cat *catalog.Catalog, retentionChunks uint32) (LifecycleConfig, *fatalRecorder) { + t.Helper() + rec := &fatalRecorder{} + cfg := LifecycleConfig{ + ExecConfig: backfill.ExecConfig{ + Catalog: cat, + Logger: silentLogger(), + Workers: 2, + Process: backfill.ProcessConfig{ + HotProbe: NewRocksHotProbe(cat.Layout().HotChunkPath, silentLogger()), + }, + }, + RetentionChunks: retentionChunks, + Fatalf: rec.fatalf, + } + return cfg, rec +} + +// fatalRecorder captures Fatalf calls so a test can assert a tick did (or did +// NOT) abort the daemon. +type fatalRecorder struct { + count atomic.Int32 + last atomic.Value // string +} + +func (r *fatalRecorder) fatalf(format string, args ...any) { + r.count.Add(1) + r.last.Store(fmt.Sprintf(format, args...)) +} + +func (r *fatalRecorder) fired() bool { return r.count.Load() > 0 } + +// runTickForCatalog runs one lifecycle tick the way ingestion would drive it: +// it derives the highest complete chunk from the catalog (the chunk id ingestion +// hands over at a boundary) and passes it as lastChunk. A negative result (young +// network, no complete chunk) is passed as chunk 0 — the resolve range guard +// then makes the plan empty, matching the design's young-network no-op. +func runTickForCatalog(ctx context.Context, t *testing.T, cfg LifecycleConfig, cat *catalog.Catalog) { + t.Helper() + through, err := deriveCompleteThrough(cat) + require.NoError(t, err) + last, ok := lastCompleteChunkAtID(through) + if !ok { + last = 0 + } + runLifecycleTick(ctx, cfg, cat, last) +} + +// makeReadyHotDirNoData opens and closes a real (empty) hot DB for c so its dir +// exists on disk and its key is "ready" — the state a discard scan inspects +// without needing a full ingest. +func makeReadyHotDirNoData(t *testing.T, cat *catalog.Catalog, c chunk.ID) { + t.Helper() + db, err := openHotTierForChunk(cat, c, silentLogger()) + require.NoError(t, err) + require.NoError(t, db.Close()) +} + +// assertQuiescent re-runs the tick's three derivations against the SAME through +// snapshot and asserts none schedule work — the quiescence postcondition. +func assertQuiescent(t *testing.T, cfg LifecycleConfig, cat *catalog.Catalog, through uint32) { + t.Helper() + earliest, _, err := cat.EarliestLedger() + require.NoError(t, err) + floor := EffectiveRetentionFloor(through, cfg.RetentionChunks, earliest) + start := ChunkIDOfLedger(floor) + low, hasLow, err := lowestMaterializedChunk(cat) + require.NoError(t, err) + if hasLow && int64(low) > start { + start = int64(low) + } + if rangeEnd, ok := lastCompleteChunkAtID(through); ok && start >= 0 { + // At quiescence resolve finds an empty plan, so RunBackfill (resolve + + // executePlan) is a no-op that returns nil — even with no Backend wired, + // since an empty plan never reaches backfillSource. + perr := backfill.RunBackfill(context.Background(), cfg.ExecConfig, chunk.ID(start), rangeEnd) //nolint:gosec // start >= 0 + assert.NoError(t, perr, "re-running backfill schedules no work at quiescence") + } + dops, err := eligibleDiscardOps(cfg, cat, through) + require.NoError(t, err) + assert.Empty(t, dops, "re-scan finds no discard work at quiescence") + pops, err := eligiblePruneOps(cfg, cat, through) + require.NoError(t, err) + assert.Empty(t, pops, "re-scan finds no prune work at quiescence") +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go new file mode 100644 index 000000000..dede88cf5 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go @@ -0,0 +1,122 @@ +package lifecycle + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// --------------------------------------------------------------------------- +// lifecycleLoop: selects on BOTH ctx.Done and the notification channel; drains +// to the most-recent queued chunk id. +// --------------------------------------------------------------------------- + +// TestLifecycleLoop_RunsTickPerNotifyThenStopsOnCtx: a notification (a completed +// chunk id) runs a tick; a ctx cancellation returns the loop. The loop never +// blocks forever and never fatals on shutdown. +func TestLifecycleLoop_RunsTickPerNotifyThenStopsOnCtx(t *testing.T) { + cat, _ := smallTxHashIndexCatalog(t, 1) + cfg, rec := lifecycleTestConfig(t, cat, 0) + + // Make the tick observable WITHOUT a slow full ingest: chunk 0 is already + // fully frozen and folded into its (terminal, cpi=1) window, with a leftover + // "ready" hot DB on disk. The plan stage is a no-op; the discard scan retires + // chunk 0's hot DB. A live chunk 1 keeps chunk 0 below the partition. + freezeKinds(t, cat, 0, geometry.KindLedgers, geometry.KindEvents, geometry.KindTxHash) + freezeCoverage(t, cat, cat.TxHashIndexLayout().TxHashIndexID(0), 0, 0) // terminal coverage of chunk 0 + makeReadyHotDirNoData(t, cat, 0) + live := openLiveHotDB(t, cat, 1) + t.Cleanup(func() { _ = live.Close() }) + + ch := make(chan chunk.ID, LifecycleQueueDepth) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + lifecycleLoop(ctx, cfg, cat, ch) + close(done) + }() + + ch <- chunk.ID(0) // ingestion hands over the just-completed chunk 0 + require.Eventually(t, func() bool { + has, err := hotKeyExists(cat, 0) + return err == nil && !has + }, 10*time.Second, 20*time.Millisecond, "the notification ran a tick that discarded chunk 0") + require.False(t, rec.fired()) + + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the loop did not return on ctx cancellation") + } +} + +// TestLifecycleLoop_DrainsToMostRecent: several chunk ids queued behind one +// notification are coalesced into ONE tick over the most-recent. With chunks 0 +// and 1 both frozen+covered and a live chunk 2, sending 0 then 1 runs a single +// tick up to chunk 1 that discards both. +func TestLifecycleLoop_DrainsToMostRecent(t *testing.T) { + cat, _ := smallTxHashIndexCatalog(t, 1) + cfg, rec := lifecycleTestConfig(t, cat, 0) + + for c := chunk.ID(0); c <= 1; c++ { + freezeKinds(t, cat, c, geometry.KindLedgers, geometry.KindEvents, geometry.KindTxHash) + freezeCoverage(t, cat, cat.TxHashIndexLayout().TxHashIndexID(c), c, c) + makeReadyHotDirNoData(t, cat, c) + } + live := openLiveHotDB(t, cat, 2) + t.Cleanup(func() { _ = live.Close() }) + + ch := make(chan chunk.ID, LifecycleQueueDepth) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { + lifecycleLoop(ctx, cfg, cat, ch) + close(done) + }() + + ch <- chunk.ID(0) + ch <- chunk.ID(1) // drained-to: one tick over [floor, 1] discards both + require.Eventually(t, func() bool { + h0, e0 := hotKeyExists(cat, 0) + h1, e1 := hotKeyExists(cat, 1) + return e0 == nil && e1 == nil && !h0 && !h1 + }, 10*time.Second, 20*time.Millisecond, "one drained tick discarded both completed chunks") + require.False(t, rec.fired()) + + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the loop did not return on ctx cancellation") + } +} + +// TestLifecycleLoop_ReturnsImmediatelyOnAlreadyCancelledCtx: an already-cancelled +// ctx makes the loop return without running any tick (never blocks on the +// channel forever). +func TestLifecycleLoop_ReturnsImmediatelyOnAlreadyCancelledCtx(t *testing.T) { + cat, _ := smallTxHashIndexCatalog(t, 1) + cfg, _ := lifecycleTestConfig(t, cat, 0) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + ch := make(chan chunk.ID) // unbuffered, never sent to + done := make(chan struct{}) + go func() { + lifecycleLoop(ctx, cfg, cat, ch) + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the loop blocked instead of observing the cancelled ctx") + } +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go new file mode 100644 index 000000000..eb1a11ffa --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go @@ -0,0 +1,229 @@ +package lifecycle + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// --------------------------------------------------------------------------- +// End-to-end tick harness: real catalog + real hotchunk DBs. +// --------------------------------------------------------------------------- + +// TestRunLifecycleTick_BoundaryFreezesFoldsDiscards is the "one boundary, end to +// end" walk: chunk 0 just closed (its full hot DB is on disk, ready), chunk 1 is +// the new live chunk. One tick must: +// - freeze chunk 0's cold artifacts FROM its hot DB (via processChunk's hot +// branch), +// - fold chunk 0 into its window's index (terminal coverage, cpi=1), +// - discard chunk 0's hot DB (cold artifacts now fully serve it), +// - leave the live chunk 1 untouched. +// +// Then re-running the tick is a no-op (quiescence). +func TestRunLifecycleTick_BoundaryFreezesFoldsDiscards(t *testing.T) { + t.Parallel() // full-chunk ingest; isolated TempDir/catalog — overlap with the other heavy tests to fit the gate's go-test timeout + cat, _ := smallTxHashIndexCatalog(t, 1) // window w == chunk w; a one-chunk window finalizes immediately + cfg, rec := lifecycleTestConfig(t, cat, 0) + + // Chunk 0: just-closed, full hot DB on disk. Chunk 1: the new live chunk. + ingestFullHotChunk(t, cat, 0) + live := openLiveHotDB(t, cat, 1) // the live chunk's hot DB (held open by "ingestion") + t.Cleanup(func() { _ = live.Close() }) + + runTickForCatalog(context.Background(), t, cfg, cat) + require.False(t, rec.fired(), "a healthy tick never aborts: %v", rec.last.Load()) + + // Chunk 0's cold artifacts are all frozen. + for _, kind := range []geometry.Kind{geometry.KindLedgers, geometry.KindEvents} { + state, err := cat.State(0, kind) + require.NoError(t, err) + assert.Equal(t, geometry.StateFrozen, state, "chunk 0 %s frozen", kind) + } + // The window's index is terminal and covers chunk 0. + covered, err := indexCovers(0, cat) + require.NoError(t, err) + assert.True(t, covered, "the window index folded chunk 0 in") + fk, ok, err := cat.FrozenTxHashIndex(cat.TxHashIndexLayout().TxHashIndexID(0)) + require.NoError(t, err) + require.True(t, ok) + assert.True(t, cat.TxHashIndexLayout().IsTerminalCoverage(fk), "a one-chunk window is terminal") + + // Chunk 0's hot DB is discarded (cold artifacts fully serve it). + has, err := hotKeyExists(cat, 0) + require.NoError(t, err) + assert.False(t, has, "chunk 0's hot key is gone") + + // The live chunk 1 is untouched: its hot key still "ready", no cold artifacts. + hotState, err := cat.HotState(1) + require.NoError(t, err) + assert.Equal(t, geometry.HotReady, hotState, "the live chunk's hot key is untouched") + lfs1, err := cat.State(1, geometry.KindLedgers) + require.NoError(t, err) + assert.Equal(t, geometry.State(""), lfs1, "the live chunk is not frozen") + + // Quiescence: re-running the tick produces no work. + through, err := deriveCompleteThrough(cat) + require.NoError(t, err) + assertQuiescent(t, cfg, cat, through) +} + +// TestRunLifecycleTick_DiscardGatedOnIndexCoverage: a complete chunk whose cold +// ledgers+events are frozen but whose window index does NOT yet cover it keeps its +// hot DB (it still serves tx lookups). Only once a terminal coverage exists does +// the discard fire. cpi=2 so a single chunk does NOT finalize the window. +func TestRunLifecycleTick_DiscardGatedOnIndexCoverage(t *testing.T) { + cat, _ := smallTxHashIndexCatalog(t, 2) // window 0 = chunks [0,1] + cfg, _ := lifecycleTestConfig(t, cat, 0) + + // Pre-freeze chunk 0's ledgers+events+txhash directly (no hot dependence), and + // leave it with a "ready" hot DB on disk. The window is NOT finalized (cpi=2, + // only chunk 0 present), so no terminal coverage exists. + freezeKinds(t, cat, 0, geometry.KindLedgers, geometry.KindEvents, geometry.KindTxHash) + makeReadyHotDirNoData(t, cat, 0) + // A live chunk 1 above it so chunk 0 is below the partition boundary. + require.NoError(t, cat.PutHotTransient(1)) + + through := chunk.ID(0).LastLedger() // chunk 0 complete via cold + // txhash is frozen, ledgers/events frozen, but the window has no FROZEN coverage + // yet => indexCovers(0) is false => NOT discarded (still needed for lookups via + // its .bin/hot DB until the index folds it in). + ops, err := eligibleDiscardOps(cfg, cat, through) + require.NoError(t, err) + require.Empty(t, ops, "no index coverage yet: the hot DB stays") + + // Now finalize the window's index so it covers chunk 0 (terminal needs chunk + // 1's .bin too; build a non-terminal-but-covering frozen coverage [0,0]). + freezeCoverage(t, cat, 0, 0, 0) + covered, err := indexCovers(0, cat) + require.NoError(t, err) + require.True(t, covered) + + ops, err = eligibleDiscardOps(cfg, cat, through) + require.NoError(t, err) + require.Len(t, ops, 1, "covered + nothing pending => discard eligible") + require.NoError(t, ops[0]()) + + has, err := hotKeyExists(cat, 0) + require.NoError(t, err) + assert.False(t, has, "the now-covered chunk's hot DB is discarded") +} + +// TestRunLifecycleTick_PastFloorPrune: a chunk wholly below the effective +// retention floor has its artifact files and hot DB swept, regardless of state. +func TestRunLifecycleTick_PastFloorPrune(t *testing.T) { + cat, _ := smallTxHashIndexCatalog(t, 1) + cfg, rec := lifecycleTestConfig(t, cat, 2) // retain ~2 chunks + + // CompleteThrough will be chunk 5's last ledger (positional: live chunk 6). + // floor = geometry.LastCompleteChunkAt(through)-retention+1 = 5-2+1 = chunk 4's first + // ledger. So chunks 0..3 are wholly past the floor and must be swept. + for c := chunk.ID(0); c <= 5; c++ { + freezeKinds(t, cat, c, geometry.KindLedgers, geometry.KindEvents, geometry.KindTxHash) + writeArtifact(t, cat.Layout().LedgerPackPath(c)) + freezeCoverage(t, cat, cat.TxHashIndexLayout().TxHashIndexID(c), c, c) // each one-chunk window terminal + } + // A past-floor hot DB too (chunk 1). + makeReadyHotDirNoData(t, cat, 1) + live := openLiveHotDB(t, cat, 6) // live chunk + t.Cleanup(func() { _ = live.Close() }) + + through, err := deriveCompleteThrough(cat) + require.NoError(t, err) + require.Equal(t, chunk.ID(5).LastLedger(), through) + floor := EffectiveRetentionFloor(through, cfg.RetentionChunks, 0) + require.Equal(t, chunk.ID(4).FirstLedger(), floor, "floor anchors 2 chunks back") + + runTickForCatalog(context.Background(), t, cfg, cat) + require.False(t, rec.fired(), "prune tick never aborts: %v", rec.last.Load()) + + // Chunks 0..3 (wholly below the floor) are gone: keys and files. + for c := chunk.ID(0); c <= 3; c++ { + ledgers, serr := cat.State(c, geometry.KindLedgers) + require.NoError(t, serr) + assert.Equal(t, geometry.State(""), ledgers, "chunk %s ledgers key swept", c) + assert.NoFileExists(t, cat.Layout().LedgerPackPath(c), "chunk %s pack swept", c) + has, herr := hotKeyExists(cat, c) + require.NoError(t, herr) + assert.False(t, has, "chunk %s hot key swept", c) + } + // Chunk 4 (the floor chunk) and 5 are within retention and survive. + for c := chunk.ID(4); c <= 5; c++ { + ledgers, serr := cat.State(c, geometry.KindLedgers) + require.NoError(t, serr) + assert.Equal(t, geometry.StateFrozen, ledgers, "chunk %s in retention survives", c) + } + + assertQuiescent(t, cfg, cat, through) +} + +// TestRunLifecycleTick_PrunesTransientIndexDebris: a "freezing" index key (a +// crashed build attempt) is swept regardless of window, even within retention. +func TestRunLifecycleTick_PrunesTransientIndexDebris(t *testing.T) { + cat, _ := smallTxHashIndexCatalog(t, 2) + cfg, rec := lifecycleTestConfig(t, cat, 0) + + // A crashed build left a "freezing" coverage key (no commit). + _, err := cat.MarkTxHashIndexFreezing(0, 0, 0) + require.NoError(t, err) + + through, err := deriveCompleteThrough(cat) + require.NoError(t, err) + ops, err := eligiblePruneOps(cfg, cat, through) + require.NoError(t, err) + require.Len(t, ops, 1, "the freezing debris is swept") + require.NoError(t, ops[0]()) + require.False(t, rec.fired()) + + covs, err := cat.AllTxHashIndexKeys() + require.NoError(t, err) + require.Empty(t, covs, "the freezing index key is gone") +} + +// --------------------------------------------------------------------------- +// CLEAN SHUTDOWN: a ctx cancelled mid-tick returns WITHOUT fatal. +// --------------------------------------------------------------------------- + +// genuineFailureTickSetup wires a catalog whose chunk-0 build is GENUINELY +// unproducible: chunk 0 sits below a READY live chunk 1 (so it counts as complete +// and the plan range [0,0] is non-empty), has no frozen artifacts, and its hot key +// is "transient" (not a ready read source). With no bulk Backend configured (the +// lifecycleTestConfig default), backfillSource has no source for chunk 0 and +// RunBackfill fails with a non-cancellation error. MaxRetries defaults to 0, so it +// fails fast. Returns the config and the fatal recorder. +func genuineFailureTickSetup(t *testing.T) (LifecycleConfig, *fatalRecorder, *catalog.Catalog) { + t.Helper() + cat, _ := smallTxHashIndexCatalog(t, 1) + cfg, rec := lifecycleTestConfig(t, cat, 0) // HotProbe wired, no Backend + readyHot(t, cat, 1) // ready live chunk => through = chunk 0 last ledger + require.NoError(t, cat.PutHotTransient(0)) // chunk 0 below live, no frozen artifacts, not a ready source + return cfg, rec, cat +} + +// TestRunLifecycleTick_CleanShutdownNoFatal: when RunBackfill returns an error AND +// ctx was cancelled, the tick must NOT call Fatalf — cancellation is a shutdown, +// never an op failure. The chunk-0 build is genuinely unproducible (no source), but +// the cancelled ctx takes precedence per the clean-shutdown policy. +func TestRunLifecycleTick_CleanShutdownNoFatal(t *testing.T) { + cfg, rec, cat := genuineFailureTickSetup(t) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // shutdown requested before the tick runs + + runLifecycleTick(ctx, cfg, cat, 0) // lastChunk 0: plan range [0,0], build fails under a cancelled ctx + require.False(t, rec.fired(), "a cancelled ctx is a clean shutdown, NOT an op failure — no Fatalf") +} + +// TestRunLifecycleTick_GenuineFailureAborts: when a plan op fails for a real +// reason (NOT ctx cancellation), the tick aborts via Fatalf per the error policy. +func TestRunLifecycleTick_GenuineFailureAborts(t *testing.T) { + cfg, rec, cat := genuineFailureTickSetup(t) + + runLifecycleTick(context.Background(), cfg, cat, 0) // lastChunk 0: plan range [0,0], the failing build + require.True(t, rec.fired(), "a genuine op failure aborts the daemon") +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go new file mode 100644 index 000000000..d0553b7c9 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go @@ -0,0 +1,203 @@ +package lifecycle + +import ( + "fmt" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// Progress is derived, never stored. "Highest complete chunk" arithmetic runs in +// int64 (-1 = "nothing complete") to avoid uint32 wraparound on the pre-genesis +// sentinel; CompleteThrough is the chokepoint. + +// preGenesisLedger is the watermark when nothing is complete (FirstLedgerSeq-1). +const preGenesisLedger uint32 = chunk.FirstLedgerSeq - 1 + +// CompleteThrough maps a signed chunk index to its "complete through" last ledger: +// c < 0 ⇒ preGenesisLedger; c >= 0 ⇒ chunk.ID(c).LastLedger(). +func CompleteThrough(c int64) uint32 { + if c < 0 { + return preGenesisLedger + } + return chunk.ID(c).LastLedger() //nolint:gosec // c >= 0 and bounded by real chunk ids +} + +// LastCommittedLedger is the single highest-durably-committed-ledger derivation. +// It maxes three terms, each in the signed domain so a fresh/young store never +// underflows to MaxUint32: +// +// - COLD — highest chunk with all artifacts durable (highestDurableChunk; -1 on +// a fresh start). Leads at startup before any hot key exists. +// - HOT — only when hot > cold, only over "ready" keys. probe == nil gives the +// positional term CompleteThrough(hot-1); probe != nil refines with one +// MaxCommittedSeq read (safe: derivation runs before ingestion locks the DB). +// - FLOOR — EarliestLedger()-1 as int64(earliest)-1, so an absent/zero pin +// yields the pre-genesis sentinel rather than underflowing. +func LastCommittedLedger(cat *catalog.Catalog, probe backfill.HotProbe) (uint32, error) { + cold, err := highestDurableChunk(cat) + if err != nil { + return 0, err + } + through := CompleteThrough(cold) + + hot, err := highestReadyChunkSigned(cat) + if err != nil { + return 0, err + } + if hot > cold { + if probe == nil { + // Positional term: everything below the live (highest ready) chunk. + through = max(through, CompleteThrough(hot-1)) + } else { + // One refinement read of the highest ready hot DB; loss detected lazily + // on this open (no eager scan over every ready key). + refined, rerr := refineWithHotDB(cat, probe, hot) + if rerr != nil { + return 0, rerr + } + through = max(through, refined) + } + } + + earliest, ok, err := cat.EarliestLedger() + if err != nil { + return 0, err + } + if ok { + // int64 before the -1 so a zero/genesis pin does not underflow. + floor := max(int64(earliest)-1, 0) + through = max(through, uint32(floor)) //nolint:gosec // floor in [0, MaxUint32), fits uint32 + } + + return through, nil +} + +// refineWithHotDB opens the highest ready hot chunk read-only and returns its +// MaxCommittedSeq, or CompleteThrough(live-1) on an empty DB. A "ready" key whose +// dir/DB is gone surfaces as backfill.ErrHotVolumeLost (lazy loss detection). +func refineWithHotDB(cat *catalog.Catalog, probe backfill.HotProbe, live int64) (uint32, error) { + id := chunk.ID(live) //nolint:gosec // live > cold >= -1, so live >= 0 + hot, ok, openErr := probe.OpenHotChunk(id) + if openErr != nil { + return 0, fmt.Errorf("%w: chunk %s is %q but its hot DB won't open (run surgical recovery): %w", + backfill.ErrHotVolumeLost, id, geometry.HotReady, openErr) + } + if !ok { + return 0, fmt.Errorf("%w: chunk %s is %q but its hot dir is missing (run surgical recovery)", + backfill.ErrHotVolumeLost, id, geometry.HotReady) + } + defer func() { _ = hot.Close() }() + + maxSeq, present, seqErr := hot.MaxCommittedSeq() + if seqErr != nil { + return 0, fmt.Errorf("%w: chunk %s: max committed seq: %w", backfill.ErrHotVolumeLost, id, seqErr) + } + if present { + return maxSeq, nil + } + // Empty live DB: positional fallback (everything below it). + return CompleteThrough(live - 1), nil +} + +// highestReadyChunkSigned returns the highest "ready" hot chunk id as int64, or -1 +// when none. The signed return lets CompleteThrough compute the positional term +// without a uint32 underflow when the live chunk is chunk 0. +func highestReadyChunkSigned(cat *catalog.Catalog) (int64, error) { + ready, err := cat.ReadyHotChunkKeys() + if err != nil { + return 0, err + } + if len(ready) == 0 { + return -1, nil + } + // Sorted ascending; the last is the highest. + return int64(ready[len(ready)-1]), nil +} + +// highestDurableChunk returns the highest chunk id with all artifacts durable +// (ledgers AND events frozen AND (txhash frozen OR covered by a frozen index)), +// or -1 on a fresh start. A partially-frozen tip chunk is excluded; backfill +// repairs it. +func highestDurableChunk(cat *catalog.Catalog) (int64, error) { + refs, err := cat.ChunkArtifactKeys() + if err != nil { + return 0, err + } + + // Frozen per-kind state per chunk. + type kinds struct{ ledgers, events, txhash bool } + frozen := map[chunk.ID]*kinds{} + for _, ref := range refs { + if ref.State != geometry.StateFrozen { + continue + } + k := frozen[ref.Chunk] + if k == nil { + k = &kinds{} + frozen[ref.Chunk] = k + } + switch ref.Kind { + case geometry.KindLedgers: + k.ledgers = true + case geometry.KindEvents: + k.events = true + case geometry.KindTxHash: + k.txhash = true + } + } + + // A frozen index coverage satisfies txhash even after the .bin was demoted. + covered, err := frozenCoverageContains(cat) + if err != nil { + return 0, err + } + + highest := int64(-1) + for c, k := range frozen { + if !k.ledgers || !k.events { + continue + } + if !k.txhash && !covered(c) { + continue + } + if id := int64(c); id > highest { + highest = id + } + } + return highest, nil +} + +// frozenCoverageContains returns a predicate reporting whether a chunk falls in +// some frozen index coverage [Lo, Hi]; coverages are read once up front. +func frozenCoverageContains(cat *catalog.Catalog) (func(chunk.ID) bool, error) { + covs, err := cat.AllTxHashIndexKeys() + if err != nil { + return nil, err + } + var frozen []geometry.TxHashIndexCoverage + for _, cov := range covs { + if cov.State == geometry.StateFrozen { + frozen = append(frozen, cov) + } + } + return func(c chunk.ID) bool { + for _, cov := range frozen { + if cov.Lo <= c && c <= cov.Hi { + return true + } + } + return false + }, nil +} + +// ChunkIDOfLedger maps a ledger to its chunk, signed so a sub-genesis ledger +// yields -1 instead of panicking. +func ChunkIDOfLedger(ledger uint32) int64 { + if ledger < chunk.FirstLedgerSeq { + return -1 + } + return int64(chunk.IDFromLedger(ledger)) +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go new file mode 100644 index 000000000..3056967bd --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go @@ -0,0 +1,104 @@ +package lifecycle + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" +) + +// TestDeriveWatermark_RealHotDB_RefinementIsNotStale exercises the watermark +// refinement against a REAL per-chunk hotchunk DB read through the production +// rocksHotProbe — the path the fakeHotProbe table tests stub out. It proves the +// single-DB MaxCommittedSeq refinement reads the actual committed ledger frontier +// (the ledgers CF's last key) and is not a stale/constant value: the bound rises +// to exactly the highest seq committed to the live chunk's real DB. +func TestDeriveWatermark_RealHotDB_RefinementIsNotStale(t *testing.T) { + cat, _ := testCatalog(t) + + live := chunk.ID(5) + // Production bracket: creates the hot dir, opens the SINGLE shared multi-CF + // DB, flips the hot key "ready". This is exactly what ingestion does. + db := openLiveHotDB(t, cat, live) + + // Commit two real ledgers into the ledgers CF (the CF MaxCommittedSeq reads). + first := live.FirstLedger() + committedTop := first + 200 + require.NoError(t, db.Ledgers().AddLedgers( + ledger.Entry{Seq: first, Bytes: []byte("ledger-A")}, + ledger.Entry{Seq: committedTop, Bytes: []byte("ledger-B")}, + )) + // Close the live writer before the probe re-opens read-only (RocksDB LOCK). + require.NoError(t, db.Close()) + + // Sanity: positional baseline (live chunk 5 ⇒ everything below 5) is chunk 4's + // last ledger, strictly below the committed top — so the assertion below can + // only pass if the refinement actually read the real DB. + baseline := mustDeriveCompleteThrough(t, cat) + require.Equal(t, chunk.ID(4).LastLedger(), baseline) + require.Greater(t, committedTop, baseline, "fixture must put the real frontier above the baseline") + + probe := NewRocksHotProbe(cat.Layout().HotChunkPath, silentLogger()) + got, err := deriveWatermark(cat, probe) + require.NoError(t, err) + require.Equal(t, committedTop, got, + "watermark must equal the REAL ledgers-CF last key, not the positional baseline") +} + +// TestDeriveWatermark_RealHotDB_OpensHighestReady proves the refinement opens the +// HIGHEST ready chunk (the live chunk), not just any ready chunk. Two ready chunks +// have independent real hot DBs with DIFFERENT committed frontiers; the watermark +// must reflect the higher chunk's DB. The fakeHotProbe table tests CANNOT cover +// this: fakeHotProbe.OpenHotChunk ignores its chunk-id argument and returns one +// canned DB, so a "open ready[0] instead of ready[len-1]" regression is invisible +// to them — only a real per-chunk probe distinguishes the two. +func TestDeriveWatermark_RealHotDB_OpensHighestReady(t *testing.T) { + cat, _ := testCatalog(t) + + lower, higher := chunk.ID(4), chunk.ID(7) + + // Lower ready chunk: a real DB committed near the TOP of chunk 4. If the + // refinement wrongly opened the lower chunk, the bound would land here. + lowDB := openLiveHotDB(t, cat, lower) + lowTop := lower.FirstLedger() + 9000 + require.NoError(t, lowDB.Ledgers().AddLedgers(ledger.Entry{Seq: lowTop, Bytes: []byte("low")})) + require.NoError(t, lowDB.Close()) + + // Higher ready chunk (the live chunk): committed mid-chunk 7. + highDB := openLiveHotDB(t, cat, higher) + highMid := higher.FirstLedger() + 1234 + require.NoError(t, highDB.Ledgers().AddLedgers(ledger.Entry{Seq: highMid, Bytes: []byte("high")})) + require.NoError(t, highDB.Close()) + + // The two frontiers must be unambiguous: chunk 7 mid-seq is far above chunk 4's + // top, so reading the wrong chunk yields a strictly different (lower) answer. + require.Greater(t, highMid, lowTop) + + probe := NewRocksHotProbe(cat.Layout().HotChunkPath, silentLogger()) + got, err := deriveWatermark(cat, probe) + require.NoError(t, err) + require.Equal(t, highMid, got, + "refinement must open the HIGHEST ready chunk (7), reading its committed mid-seq") +} + +// TestDeriveWatermark_RealHotDB_EmptyLiveFallsBack is the count-only-ready case +// against a real DB: a "ready" live chunk whose real hot DB has NO committed +// ledger (MaxCommittedSeq ok=false) must fall back to deriveCompleteThrough, not +// fabricate a frontier. Read through the production probe. +func TestDeriveWatermark_RealHotDB_EmptyLiveFallsBack(t *testing.T) { + cat, _ := testCatalog(t) + makeChunkDurable(t, cat, 0) // cold term => chunk 0 last ledger + + live := chunk.ID(3) + db := openLiveHotDB(t, cat, live) // ready key + real dir, but NOTHING committed + require.NoError(t, db.Close()) + + // Real probe reads the empty ledgers CF: ok=false, no refinement. + probe := NewRocksHotProbe(cat.Layout().HotChunkPath, silentLogger()) + got, err := deriveWatermark(cat, probe) + require.NoError(t, err) + require.Equal(t, chunk.ID(2).LastLedger(), got, + "empty live DB ⇒ positional baseline (max ready 3 - 1 = chunk 2), no fabricated frontier") +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_shim_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_shim_test.go new file mode 100644 index 000000000..cd68fdced --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_shim_test.go @@ -0,0 +1,24 @@ +package lifecycle + +import ( + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" +) + +// Test-only aliases for the consolidated progress derivation. The design folded +// deriveCompleteThrough + deriveWatermark into ONE LastCommittedLedger(cat[, probe]): +// +// - deriveCompleteThrough(cat) == LastCommittedLedger(cat, nil) (chunk +// granularity, pure catalog read — the positional term, no hot DB open). +// - deriveWatermark(cat, probe) == LastCommittedLedger(cat, probe) (one +// refinement read of the highest ready hot DB, loss detected LAZILY on it). +// +// These shims keep the tests' intent legible; production callers use +// LastCommittedLedger directly. +func deriveCompleteThrough(cat *catalog.Catalog) (uint32, error) { + return LastCommittedLedger(cat, nil) +} + +func deriveWatermark(cat *catalog.Catalog, probe backfill.HotProbe) (uint32, error) { + return LastCommittedLedger(cat, probe) +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go new file mode 100644 index 000000000..34f834fb4 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go @@ -0,0 +1,328 @@ +package lifecycle + +import ( + "errors" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// --------------------------------------------------------------------------- +// progress derivation test helpers. +// --------------------------------------------------------------------------- + +// makeChunkDurable freezes ledgers+events+txhash for a chunk — the durable state +// highestDurableChunk counts. +func makeChunkDurable(t *testing.T, cat *catalog.Catalog, c chunk.ID) { + t.Helper() + freezeKinds(t, cat, c, geometry.KindLedgers, geometry.KindEvents, geometry.KindTxHash) +} + +// makeHotDir creates the on-disk hot dir for a chunk so deriveWatermark's +// per-ready-key dir-existence loop sees it present. +func makeHotDir(t *testing.T, cat *catalog.Catalog, c chunk.ID) { + t.Helper() + require.NoError(t, os.MkdirAll(cat.Layout().HotChunkPath(c), 0o755)) +} + +// readyHot marks a chunk's hot key "ready" AND creates its dir, the production +// pairing deriveWatermark expects (a ready key whose dir is missing is loss). +func readyHot(t *testing.T, cat *catalog.Catalog, c chunk.ID) { + t.Helper() + require.NoError(t, cat.PutHotTransient(c)) + require.NoError(t, cat.FlipHotReady(c)) + makeHotDir(t, cat, c) +} + +// --------------------------------------------------------------------------- +// CompleteThrough — sentinel-safe signed->ledger map. +// +// ALIASING TRAP: a guard-less impl wraps -1 to exactly preGenesisLedger anyway +// (MaxUint32+1 overflows to 0), so a -1-only test is blind to a dropped guard. +// The -2/-100 rows are the load-bearing ones (they wrap to large, distinct values +// the guard must squash). +// --------------------------------------------------------------------------- + +func TestCompleteThrough(t *testing.T) { + tests := []struct { + name string + in int64 + want uint32 + }{ + {"pre-genesis sentinel -1 => FirstLedgerSeq-1, not MaxUint32 (aliases the wrap)", -1, preGenesisLedger}, + {"sentinel -2 does NOT alias the wrap (guard-less would yield 4294957297)", -2, preGenesisLedger}, + {"deeply negative still pre-genesis", -100, preGenesisLedger}, + {"chunk 0 last ledger", 0, chunk.ID(0).LastLedger()}, + {"chunk 5 last ledger", 5, chunk.ID(5).LastLedger()}, + } + require.Equal(t, uint32(1), preGenesisLedger, "FirstLedgerSeq-1 == 1 (the doc's chunkLastLedger(-1))") + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, CompleteThrough(tc.in)) + }) + } + + // Assert the aliasing trap directly so the comment above can't rot: -1 wraps to + // preGenesisLedger, -2 does not. Computed from chunk arithmetic, not hardcoded. + guardlessWrap := func(c int64) uint32 { + return chunk.ID(uint32(c)).LastLedger() + } + require.Equal(t, preGenesisLedger, guardlessWrap(-1), + "-1 aliases preGenesisLedger under the wrap — the coincidence this test must not rely on") + require.NotEqual(t, preGenesisLedger, guardlessWrap(-2), + "-2 must NOT alias — proving the guard (not a coincidence) is what makes CompleteThrough(-2) safe") +} + +// --------------------------------------------------------------------------- +// LastCommittedLedger — chunk-granularity bound, pure catalog read. +// --------------------------------------------------------------------------- + +func TestLastCommittedLedger(t *testing.T) { + t.Run("fresh store => pre-genesis sentinel, never MaxUint32", func(t *testing.T) { + // Every term is -1; the signed domain must yield FirstLedgerSeq-1, not wrap. + cat, _ := testCatalog(t) + got, err := deriveCompleteThrough(cat) + require.NoError(t, err) + require.Equal(t, preGenesisLedger, got) + }) + + t.Run("cold term leads: highest fully-durable chunk", func(t *testing.T) { + cat, _ := testCatalog(t) + makeChunkDurable(t, cat, 0) + makeChunkDurable(t, cat, 1) + makeChunkDurable(t, cat, 2) + got, err := deriveCompleteThrough(cat) + require.NoError(t, err) + require.Equal(t, chunk.ID(2).LastLedger(), got) + }) + + t.Run("incompletely-frozen tip degrades the bound (ledgers frozen, events freezing)", func(t *testing.T) { + cat, _ := testCatalog(t) + makeChunkDurable(t, cat, 0) + makeChunkDurable(t, cat, 1) + // Chunk 2 mid-freeze (events only "freezing") must NOT count: bound stays at 1. + freezeKinds(t, cat, 2, geometry.KindLedgers, geometry.KindTxHash) + require.NoError(t, cat.MarkChunkFreezing(2, geometry.KindEvents)) + got, err := deriveCompleteThrough(cat) + require.NoError(t, err) + require.Equal(t, chunk.ID(1).LastLedger(), got) + }) + + t.Run("txhash satisfied by a frozen index coverage (post-finalization demote)", func(t *testing.T) { + cat, _ := testCatalog(t) + // Chunk 7: txhash demoted but a frozen index coverage spans it ⇒ still durable. + freezeKinds(t, cat, 7, geometry.KindLedgers, geometry.KindEvents) + freezeCoverage(t, cat, cat.TxHashIndexLayout().TxHashIndexID(7), 0, 999) // window 0 covers chunk 7 + got, err := deriveCompleteThrough(cat) + require.NoError(t, err) + require.Equal(t, chunk.ID(7).LastLedger(), got) + }) + + t.Run("chunk NOT covered by any frozen index and no frozen txhash does not count", func(t *testing.T) { + cat, _ := testCatalog(t) + makeChunkDurable(t, cat, 0) + // Chunk 1: ledgers+events frozen, no txhash, no covering index. + freezeKinds(t, cat, 1, geometry.KindLedgers, geometry.KindEvents) + got, err := deriveCompleteThrough(cat) + require.NoError(t, err) + require.Equal(t, chunk.ID(0).LastLedger(), got, "chunk 1 not durable; bound stays at chunk 0") + }) + + t.Run("positional term leads in steady state: everything below the live chunk", func(t *testing.T) { + cat, _ := testCatalog(t) + // No cold artifacts yet (steady state: chunks complete before cold exists). + // Ready hot keys 3,4,5 => live chunk is 5 => everything below 5 complete. + readyHot(t, cat, 3) + readyHot(t, cat, 4) + readyHot(t, cat, 5) + got, err := deriveCompleteThrough(cat) + require.NoError(t, err) + require.Equal(t, chunk.ID(4).LastLedger(), got, "max ready (5) - 1 = chunk 4's last ledger") + }) + + t.Run("transient hot key does NOT advance the positional term", func(t *testing.T) { + cat, _ := testCatalog(t) + readyHot(t, cat, 3) + // A transient key above the highest ready one must be excluded. + require.NoError(t, cat.PutHotTransient(9)) + got, err := deriveCompleteThrough(cat) + require.NoError(t, err) + require.Equal(t, chunk.ID(2).LastLedger(), got, "max READY (3) - 1, ignoring transient 9") + }) + + t.Run("live chunk 0 => positional term is pre-genesis, NOT MaxUint32", func(t *testing.T) { + // The exact uint32-underflow trap: max ready = 0, so 0-1 must be the + // pre-genesis sentinel, not ID(4294967295).LastLedger(). + cat, _ := testCatalog(t) + readyHot(t, cat, 0) + got, err := deriveCompleteThrough(cat) + require.NoError(t, err) + require.Equal(t, preGenesisLedger, got) + }) + + t.Run("earliest pin floor leads when above cold/positional terms", func(t *testing.T) { + cat, _ := testCatalog(t) + // Floor pinned mid-chain, no chunks durable, no hot keys. + const floor = 50000 + require.NoError(t, cat.PinEarliestLedger(floor)) + got, err := deriveCompleteThrough(cat) + require.NoError(t, err) + require.Equal(t, uint32(floor-1), got) + }) + + t.Run("earliest pin == genesis (2) does not underflow", func(t *testing.T) { + cat, _ := testCatalog(t) + require.NoError(t, cat.PinEarliestLedger(chunk.FirstLedgerSeq)) + got, err := deriveCompleteThrough(cat) + require.NoError(t, err) + require.Equal(t, preGenesisLedger, got, "earliest 2 - 1 = 1, not MaxUint32") + }) + + t.Run("max of all three terms", func(t *testing.T) { + cat, _ := testCatalog(t) + makeChunkDurable(t, cat, 0) // cold => chunk 0 last ledger + readyHot(t, cat, 4) // positional => chunk 3 last ledger (highest) + require.NoError(t, cat.PinEarliestLedger(2)) + got, err := deriveCompleteThrough(cat) + require.NoError(t, err) + require.Equal(t, chunk.ID(3).LastLedger(), got) + }) +} + +// --------------------------------------------------------------------------- +// deriveWatermark — deriveCompleteThrough + one refinement read + the +// per-ready-key dir-existence fatal loop. +// --------------------------------------------------------------------------- + +func TestDeriveWatermark(t *testing.T) { + t.Run("no ready hot keys => equals deriveCompleteThrough, no open", func(t *testing.T) { + cat, _ := testCatalog(t) + makeChunkDurable(t, cat, 0) + probe := &fakeHotProbe{} // would error if opened with ok=false under "ready", but none ready + got, err := deriveWatermark(cat, probe) + require.NoError(t, err) + require.Equal(t, chunk.ID(0).LastLedger(), got) + }) + + t.Run("sub-chunk precision: refinement reads mid-chunk seq inside the live chunk", func(t *testing.T) { + cat, _ := testCatalog(t) + readyHot(t, cat, 5) // live chunk 5; positional term = chunk 4 last ledger + midLive := chunk.ID(5).FirstLedger() + 123 + probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{maxSeq: midLive, present: true}} + got, err := deriveWatermark(cat, probe) + require.NoError(t, err) + require.Equal(t, midLive, got, "refined to the live chunk's committed seq") + }) + + t.Run("boundary-crash under-count recovered by refinement", func(t *testing.T) { + // Live chunk crashed at a boundary and was demoted to "transient": the + // highest READY key is the just-completed predecessor (chunk 4), whose + // completion no key advertises (positional term = chunk 3). The refinement + // opens chunk 4 and reads its full committed seq = chunk 4's last ledger, + // recovering the frontier the positional term under-counted. + cat, _ := testCatalog(t) + readyHot(t, cat, 4) + require.NoError(t, cat.PutHotTransient(5)) // the crashed live chunk + require.Equal(t, chunk.ID(3).LastLedger(), mustDeriveCompleteThrough(t, cat), + "positional term alone under-counts to chunk 3") + + chunk4Last := chunk.ID(4).LastLedger() + probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{maxSeq: chunk4Last, present: true}} + got, err := deriveWatermark(cat, probe) + require.NoError(t, err) + require.Equal(t, chunk4Last, got, "refinement recovers the chunk-4 frontier") + }) + + t.Run("count-only-ready: an empty refinement DB falls back to deriveCompleteThrough", func(t *testing.T) { + cat, _ := testCatalog(t) + makeChunkDurable(t, cat, 0) + readyHot(t, cat, 3) // positional => chunk 2 last ledger + // DB present but empty (present=false): no refinement, w stays positional. + probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{present: false}} + got, err := deriveWatermark(cat, probe) + require.NoError(t, err) + require.Equal(t, chunk.ID(2).LastLedger(), got) + }) + + t.Run("refinement only RAISES the bound, never lowers it", func(t *testing.T) { + cat, _ := testCatalog(t) + makeChunkDurable(t, cat, 0) + makeChunkDurable(t, cat, 1) + makeChunkDurable(t, cat, 2) // cold term => chunk 2 last ledger + readyHot(t, cat, 3) // positional => chunk 2 last ledger + // Live DB reports a seq below the cold bound (e.g. just opened); max wins. + probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{maxSeq: 5, present: true}} + got, err := deriveWatermark(cat, probe) + require.NoError(t, err) + require.Equal(t, chunk.ID(2).LastLedger(), got) + }) + + t.Run("LAZY loss (item R2-6): only the highest ready chunk is opened; a lower"+ + " ready key's missing dir is NOT eagerly flagged", func(t *testing.T) { + cat, _ := testCatalog(t) + // Two ready keys; the LOWER one's dir is missing. Under the design's lazy + // detection (no eager all-ready-keys scan) only the HIGHEST ready chunk is + // opened, so the lower key's missing dir is not surfaced here — it surfaces + // later, when ingestion/discard reaches that chunk via openHotTierForChunk. + require.NoError(t, cat.PutHotTransient(2)) + require.NoError(t, cat.FlipHotReady(2)) // ready key 2, NO dir (not opened here) + readyHot(t, cat, 5) // highest ready key 5 WITH dir (opened) + probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{maxSeq: 10, present: true}} + got, err := deriveWatermark(cat, probe) + require.NoError(t, err) + require.Equal(t, uint32(10), got, "refined to the highest ready chunk's seq") + }) + + t.Run("fatal: a ready HIGHEST chunk whose dir is missing (lazy loss on open)", func(t *testing.T) { + cat, _ := testCatalog(t) + // The highest ready chunk's dir is missing: the one open the derivation + // performs surfaces the loss as backfill.ErrHotVolumeLost with recovery guidance. + require.NoError(t, cat.PutHotTransient(5)) + require.NoError(t, cat.FlipHotReady(5)) // ready key 5, NO dir + probe := &fakeHotProbe{ok: false} // OpenHotChunk reports dir absent + _, err := deriveWatermark(cat, probe) + require.Error(t, err) + require.ErrorIs(t, err, backfill.ErrHotVolumeLost) + require.Contains(t, err.Error(), "00000005") + }) + + t.Run("fatal: refinement open error on the highest ready chunk", func(t *testing.T) { + cat, _ := testCatalog(t) + readyHot(t, cat, 3) // dir present + probe := &fakeHotProbe{openErr: errors.New("rocksdb LOCK held")} + _, err := deriveWatermark(cat, probe) + require.Error(t, err) + require.ErrorIs(t, err, backfill.ErrHotVolumeLost) + }) + + t.Run("fatal: refinement read error", func(t *testing.T) { + cat, _ := testCatalog(t) + readyHot(t, cat, 3) + probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{maxErr: errors.New("corrupt")}} + _, err := deriveWatermark(cat, probe) + require.Error(t, err) + require.ErrorIs(t, err, backfill.ErrHotVolumeLost) + }) + + t.Run("live chunk 0 ready, empty DB => pre-genesis, no underflow", func(t *testing.T) { + cat, _ := testCatalog(t) + readyHot(t, cat, 0) + probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{present: false}} + got, err := deriveWatermark(cat, probe) + require.NoError(t, err) + require.Equal(t, preGenesisLedger, got) + }) +} + +func mustDeriveCompleteThrough(t *testing.T, cat *catalog.Catalog) uint32 { + t.Helper() + got, err := deriveCompleteThrough(cat) + require.NoError(t, err) + return got +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention.go new file mode 100644 index 000000000..852b44976 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention.go @@ -0,0 +1,42 @@ +package lifecycle + +import ( + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// RetentionFloor is the lowest chunk still within retention; anything below is +// eligible for discard/prune. It is the reader-side retention contract (design +// "Reader retention contract", gettx §8.2 / §8.5): availability is decided by +// retention, not the on-disk file set, so prune/sweep can unlink a chunk the +// instant it passes the floor without coordinating with the index lifecycle. The +// floor may err LOW harmlessly (a wrongly-retained chunk still hits the reader's +// missing-file rule), so it anchors on the live CompleteThrough; widening history +// is catch-up's job, not the floor's. +type RetentionFloor struct { + chunk chunk.ID // lowest in-retention chunk +} + +// NewRetentionFloor pins the floor for one (through, retentionChunks, earliest) +// snapshot. A shortened retentionChunks raises the floor at once. +func NewRetentionFloor(through, retentionChunks, earliest uint32) RetentionFloor { + return RetentionFloor{chunk: chunk.IDFromLedger(EffectiveRetentionFloor(through, retentionChunks, earliest))} +} + +// Excludes reports whether chunk c is below the floor (past retention). The scans +// use it on a chunk directly and, since an index is below the floor exactly when +// its last chunk is, as Excludes(layout.LastChunk(idx)) for a whole index. +func (f RetentionFloor) Excludes(c chunk.ID) bool { return c < f.chunk } + +// EffectiveRetentionFloor is the chunk-aligned lower bound of the retention +// window: the HIGHER of the sliding floor (retentionChunks back from the last +// complete chunk) and the fixed earliest_ledger. slidingChunk is signed so a +// young store / large retentionChunks clamps to chunk 0 instead of underflowing. +func EffectiveRetentionFloor(upperBound, retentionChunks, earliest uint32) uint32 { + sliding := uint32(chunk.FirstLedgerSeq) // GenesisLedger + if retentionChunks > 0 { + slidingChunk := geometry.LastCompleteChunkAt(upperBound) - int64(retentionChunks) + 1 + sliding = geometry.ChunkFirstLedger(max(slidingChunk, 0)) + } + return max(sliding, earliest) +} diff --git a/cmd/stellar-rpc/internal/fullhistory/retention_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go similarity index 56% rename from cmd/stellar-rpc/internal/fullhistory/retention_test.go rename to cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go index e3defe955..fc954644f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/retention_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go @@ -1,4 +1,4 @@ -package fullhistory +package lifecycle import ( "testing" @@ -6,8 +6,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) // --------------------------------------------------------------------------- @@ -18,7 +18,7 @@ import ( // --------------------------------------------------------------------------- // through = chunk 100's last ledger, retain 10 chunks ⇒ floor = chunk 91 -// (retentionFloorChunk: 100-10+1 = 91). Anything below chunk 91 is excluded. +// (EffectiveRetentionFloor: 100-10+1 = 91). Anything below chunk 91 is excluded. func TestRetentionFloor_ExcludesBelow(t *testing.T) { floor := NewRetentionFloor(chunk.ID(100).LastLedger(), 10, 0) @@ -44,8 +44,8 @@ func TestRetentionFloor_ShorteningRaisesFloorImmediately(t *testing.T) { // A whole tx-hash index is below the floor exactly when its last chunk is, so // callers test Excludes(layout.LastChunk(idx)) — no index-specific method needed. func TestRetentionFloor_ExcludesIndexByLastChunk(t *testing.T) { - layout, err := geometry.NewTxHashIndexLayout(4) // indexes: 0=[0,3], 1=[4,7], 2=[8,11] - require.NoError(t, err) + cat, _ := smallTxHashIndexCatalog(t, 4) // indexes: 0=[0,3], 1=[4,7], 2=[8,11] + layout := cat.TxHashIndexLayout() // through = chunk 11's last ledger, retain 4 chunks ⇒ floor = chunk 8 // (11-4+1 = 8). Index 2 ([8,11]) starts at the floor. @@ -94,3 +94,71 @@ func TestRetentionFloor_YoungStoreClampsToGenesis(t *testing.T) { floor := NewRetentionFloor(chunk.ID(3).LastLedger(), 1000, 0) assert.False(t, floor.Excludes(0), "chunk 0 is at the clamped floor, not below it") } + +// --------------------------------------------------------------------------- +// Scenario: a window STRADDLING the floor serves in-range seqs and not-found +// below. A finalized window's frozen .idx covers [lo, hi] including chunks the +// floor has since risen past; the gate masks those below-floor chunks. This is +// the stale-.idx case gettransaction §8.5 tolerates because the reader gate +// makes below-floor reads not-found regardless of what the .idx resolves. +// --------------------------------------------------------------------------- + +func TestReaderRetention_WindowStraddlingFloorServesInRangeNotBelow(t *testing.T) { + cat, _ := smallTxHashIndexCatalog(t, 4) // window 0 = chunks [0,3] + wins := cat.TxHashIndexLayout() + + // Window 0 was finalized at terminal coverage [0,3] when the floor sat at + // genesis. Its frozen .idx hashes chunks 0..3 — a static, stale-lo artifact. + for c := chunk.ID(0); c <= 3; c++ { + freezeKinds(t, cat, c, geometry.KindLedgers, geometry.KindEvents) + } + freezeCoverage(t, cat, 0, 0, 3) + fk, ok, err := cat.FrozenTxHashIndex(0) + require.NoError(t, err) + require.True(t, ok) + require.True(t, wins.IsTerminalCoverage(fk), "window 0 is finalized") + + // The floor later rose to chunk 2 (its first ledger). Window 0 now STRADDLES + // the floor: chunks 0,1 below it, chunks 2,3 in range. The .idx still claims + // lo=0, but the reader gate is the source of truth. + through := chunk.ID(3).LastLedger() + // Pick retentionChunks so the sliding floor lands on chunk 2: + // geometry.LastCompleteChunkAt(through)=3, floor chunk = 3-retention+1 = 2 ⇒ retention=2. + floor := NewRetentionFloor(through, 2, 0) + + // (The seq-level reader masking — a below-floor read is not-found even though + // the stale .idx still hashes chunks 0,1 — returns with the read path, #772; + // RetentionFloor here exposes only the chunk-granularity prune predicate.) + + // The straddling window's frozen .idx is NOT swept: the window is not wholly + // below the floor (its last chunk, 3, is in range), so only its below-floor + // chunk artifacts (chunks 0,1) are pruned. + assert.False(t, floor.Excludes(wins.LastChunk(0)), + "a straddling window is not wholly below the floor — its .idx is kept") + cfg, _ := lifecycleTestConfig(t, cat, 2) + pops, err := eligiblePruneOps(cfg, cat, through) + require.NoError(t, err) + for _, op := range pops { + require.NoError(t, op()) + } + + // The window's frozen .idx coverage survives the prune (index family). + survives, ok, err := cat.FrozenTxHashIndex(0) + require.NoError(t, err) + require.True(t, ok, "the straddling window keeps its frozen coverage") + require.Equal(t, fk.Key, survives.Key) + + // The below-floor chunks 0,1 ARE pruned (chunk family); the in-range chunks + // 2,3 survive — exactly the data the gate admits. + for c := chunk.ID(0); c <= 1; c++ { + ledgers, serr := cat.State(c, geometry.KindLedgers) + require.NoError(t, serr) + assert.Equal(t, geometry.State(""), ledgers, "below-floor chunk %s pruned", c) + } + for c := chunk.ID(2); c <= 3; c++ { + ledgers, serr := cat.State(c, geometry.KindLedgers) + require.NoError(t, serr) + assert.Equal(t, geometry.StateFrozen, ledgers, "in-range chunk %s survives", c) + } + assertQuiescent(t, cfg, cat, through) +} diff --git a/cmd/stellar-rpc/internal/fullhistory/observability/observability.go b/cmd/stellar-rpc/internal/fullhistory/observability/observability.go index d6972fe10..0a111b85e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/observability/observability.go +++ b/cmd/stellar-rpc/internal/fullhistory/observability/observability.go @@ -1,9 +1,14 @@ package observability import ( + "io/fs" + "os" + "path/filepath" "time" "github.com/prometheus/client_golang/prometheus" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" ) // Metrics is the daemon's control-plane sink — the derived-progress gauges plus @@ -11,15 +16,34 @@ import ( // All methods must be safe for concurrent use. type Metrics interface { // LastCommitted sets the derived last-committed ledger and the effective - // retention floor (the two advance together each backfill pass). + // retention floor (the two advance together each backfill pass / lifecycle tick). LastCommitted(lastCommitted, retentionFloor uint32) + // LedgerCommitted is the per-ledger liveness heartbeat the live-ingestion loop + // emits after each committed ledger: it advances the same last-committed gauge + // between lifecycle ticks (leaving the retention floor untouched mid-chunk), so + // a wedged ingester trips the gauge that the tick-granular LastCommitted can't. + LedgerCommitted(seq uint32) + + // ChunkBoundary counts one ingestion chunk-boundary handoff (closedChunk = just-filled chunk id). + ChunkBoundary(closedChunk uint32) + + // LiveHotChunks sets the count of hot-chunk DBs currently on disk (the + // hot:chunk key count). Reported by every lifecycle tick after the discard + // stage so the gauge tracks the live + awaiting-discard set. + LiveHotChunks(count int) + + // ColdTierBytes sets the cold-tier on-disk footprint. + ColdTierBytes(bytes int64) + // BackfillPass records one completed backfill pass's wall-clock. BackfillPass(d time.Duration) // Freeze records one freeze (plan-and-execute) stage's wall-clock. Freeze(d time.Duration) // Rebuild records one index rebuild's wall-clock. Rebuild(d time.Duration) + // Discard counts the hot DBs a tick retired and records the stage wall-clock. + Discard(count int, d time.Duration) // Prune counts swept artifacts and records the sweep's wall-clock. Prune(count int, d time.Duration) } @@ -28,9 +52,14 @@ type Metrics interface { type NopMetrics struct{} func (NopMetrics) LastCommitted(uint32, uint32) {} +func (NopMetrics) LedgerCommitted(uint32) {} +func (NopMetrics) ChunkBoundary(uint32) {} +func (NopMetrics) LiveHotChunks(int) {} +func (NopMetrics) ColdTierBytes(int64) {} func (NopMetrics) BackfillPass(time.Duration) {} func (NopMetrics) Freeze(time.Duration) {} func (NopMetrics) Rebuild(time.Duration) {} +func (NopMetrics) Discard(int, time.Duration) {} func (NopMetrics) Prune(int, time.Duration) {} // MetricsOrNop returns m, or NopMetrics{} when nil, so call sites never nil-check. @@ -56,9 +85,13 @@ type PrometheusMetrics struct { // Gauges — absolute, last-write-wins. lastCommitted prometheus.Gauge retentionFloor prometheus.Gauge + liveHotChunks prometheus.Gauge + coldTierBytes prometheus.Gauge - // Counter — monotonic tally. - pruned prometheus.Counter + // Counters — monotonic tallies. + chunkBoundaries prometheus.Counter + discarded prometheus.Counter + pruned prometheus.Counter // Durations — per-phase wall-clock histogram, keyed by phase label. phaseDuration *prometheus.HistogramVec @@ -69,6 +102,7 @@ const ( phaseBackfillPass = "backfill_pass" phaseFreeze = "freeze" phaseRebuild = "rebuild" + phaseDiscard = "discard" phasePrune = "prune" ) @@ -79,14 +113,20 @@ func NewPrometheusMetrics(registry *prometheus.Registry, namespace string) *Prom Namespace: namespace, Subsystem: subsystem, Name: name, Help: help, }) } + counter := func(name, help string) prometheus.Counter { + return prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, Subsystem: subsystem, Name: name, Help: help, + }) + } m := &PrometheusMetrics{ - lastCommitted: gauge("last_committed_ledger", "highest ledger durably committed"), - retentionFloor: gauge("retention_floor_ledger", "effective retention floor — lowest in-window ledger"), - pruned: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, Subsystem: subsystem, - Name: "pruned_ops_total", Help: "artifacts swept after an index build", - }), + lastCommitted: gauge("last_committed_ledger", "highest ledger durably committed"), + retentionFloor: gauge("retention_floor_ledger", "effective retention floor — lowest in-window ledger"), + liveHotChunks: gauge("live_hot_chunks", "count of hot-chunk DBs currently on disk"), + coldTierBytes: gauge("cold_tier_bytes", "cold-tier on-disk footprint in bytes"), + chunkBoundaries: counter("chunk_boundaries_total", "ingestion chunk-boundary handoffs"), + discarded: counter("discarded_hot_chunks_total", "hot DBs retired by the discard stage"), + pruned: counter("pruned_ops_total", "artifacts swept after an index build"), phaseDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, Name: "phase_duration_seconds", Help: "wall-clock of a daemon phase action", @@ -94,7 +134,11 @@ func NewPrometheusMetrics(registry *prometheus.Registry, namespace string) *Prom }, []string{"phase"}), } - registry.MustRegister(m.lastCommitted, m.retentionFloor, m.pruned, m.phaseDuration) + registry.MustRegister( + m.lastCommitted, m.retentionFloor, m.liveHotChunks, m.coldTierBytes, + m.chunkBoundaries, m.discarded, m.pruned, + m.phaseDuration, + ) return m } @@ -103,6 +147,14 @@ func (m *PrometheusMetrics) LastCommitted(lastCommitted, retentionFloor uint32) m.retentionFloor.Set(float64(retentionFloor)) } +func (m *PrometheusMetrics) LedgerCommitted(seq uint32) { m.lastCommitted.Set(float64(seq)) } + +func (m *PrometheusMetrics) ChunkBoundary(uint32) { m.chunkBoundaries.Inc() } + +func (m *PrometheusMetrics) LiveHotChunks(count int) { m.liveHotChunks.Set(float64(count)) } + +func (m *PrometheusMetrics) ColdTierBytes(bytes int64) { m.coldTierBytes.Set(float64(bytes)) } + func (m *PrometheusMetrics) BackfillPass(d time.Duration) { m.phaseDuration.WithLabelValues(phaseBackfillPass).Observe(d.Seconds()) } @@ -115,6 +167,13 @@ func (m *PrometheusMetrics) Rebuild(d time.Duration) { m.phaseDuration.WithLabelValues(phaseRebuild).Observe(d.Seconds()) } +func (m *PrometheusMetrics) Discard(count int, d time.Duration) { + if count > 0 { + m.discarded.Add(float64(count)) + } + m.phaseDuration.WithLabelValues(phaseDiscard).Observe(d.Seconds()) +} + func (m *PrometheusMetrics) Prune(count int, d time.Duration) { if count > 0 { m.pruned.Add(float64(count)) @@ -124,3 +183,42 @@ func (m *PrometheusMetrics) Prune(count int, d time.Duration) { // compile-time interface check. var _ Metrics = (*PrometheusMetrics)(nil) + +// MeasureColdTierBytes sums the cold tier's on-disk footprint (ledgers/events/txhash-raw/ +// txhash-index trees; hot tier and meta store excluded), walking each root once and +// ignoring missing trees. A per-tree error is non-fatal to the others (caller skips the gauge). +func MeasureColdTierBytes(layout geometry.Layout) (int64, error) { + var total int64 + var firstErr error + for _, root := range []string{ + layout.LedgersRoot(), + layout.EventsRoot(), + layout.TxHashRawRoot(), + layout.TxHashIndexRoot(), + } { + err := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil // an un-materialized tree contributes nothing + } + return err + } + if d.IsDir() { + return nil + } + info, ierr := d.Info() + if ierr != nil { + if os.IsNotExist(ierr) { + return nil // raced with a prune unlink — count it as gone + } + return ierr + } + total += info.Size() + return nil + }) + if err != nil && firstErr == nil { + firstErr = err + } + } + return total, firstErr +} diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go index f172587e6..18aa8268b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go @@ -58,6 +58,11 @@ type Config struct { // "inherit the pinned defaults"; see CFOptions docstring for // the per-knob inherit/override semantics. PerCFOptions map[string]CFOptions + + // ReadOnly opens the store read-only (dir never created, no writes, no + // flush-on-close). Reads see durable SST/MANIFEST only — an un-flushed WAL is + // NOT replayed (a cleanly-closed DB has none). Used by the freeze source. + ReadOnly bool } // Store is the Layer-1 RocksDB handle. Concrete struct: one impl, @@ -425,8 +430,12 @@ func (s *Store) Close() error { return nil } - if err := s.doFlush(); err != nil { - s.cfg.Logger.WithError(err).Warnf("rocksdb: graceful close Flush failed at %s; next Open will replay WAL", s.cfg.Path) + // A read-only store has nothing to flush (and the RocksDB read-only handle + // would reject it); only a writable store flushes its memtable on close. + if !s.cfg.ReadOnly { + if err := s.doFlush(); err != nil { + s.cfg.Logger.WithError(err).Warnf("rocksdb: graceful close Flush failed at %s; next Open will replay WAL", s.cfg.Path) + } } for _, cfh := range s.cfHandles { @@ -494,14 +503,19 @@ func (s *Store) constructAndOpen() error { if err != nil { return fmt.Errorf("rocksdb: canonicalize path %s: %w", s.cfg.Path, err) } - if err := os.MkdirAll(abs, dirPerm); err != nil { - return fmt.Errorf("rocksdb: mkdir %s: %w", abs, err) + // Read-only opens an existing DB; it never creates the directory. + if !s.cfg.ReadOnly { + if err := os.MkdirAll(abs, dirPerm); err != nil { + return fmt.Errorf("rocksdb: mkdir %s: %w", abs, err) + } } cfNames := resolveCFNames(s.cfg) opts := grocksdb.NewDefaultOptions() - opts.SetCreateIfMissing(true) - opts.SetCreateIfMissingColumnFamilies(true) + if !s.cfg.ReadOnly { + opts.SetCreateIfMissing(true) + opts.SetCreateIfMissingColumnFamilies(true) + } cfOpts := make([]*grocksdb.Options, len(cfNames)) for i := range cfOpts { @@ -511,7 +525,18 @@ func (s *Store) constructAndOpen() error { s.applyTuning(opts, cfNames, cfOpts) start := time.Now() - db, cfHandles, err := grocksdb.OpenDbColumnFamilies(opts, abs, cfNames, cfOpts) + var ( + db *grocksdb.DB + cfHandles []*grocksdb.ColumnFamilyHandle + ) + if s.cfg.ReadOnly { + // errorIfWalFileExists=false: a cleanly-closed DB has no WAL; if a crash ever + // left one, read-only skips it (SST-only) and the caller's completeness gate + // falls through rather than failing the open. + db, cfHandles, err = grocksdb.OpenDbForReadOnlyColumnFamilies(opts, abs, cfNames, cfOpts, false) + } else { + db, cfHandles, err = grocksdb.OpenDbColumnFamilies(opts, abs, cfNames, cfOpts) + } elapsed := time.Since(start) if err != nil { opts.Destroy() diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index 0b95fc8ef..60292340b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -50,18 +50,13 @@ func RemoveHotChunkDir(dataDir string, chunkID chunk.ID) error { return os.RemoveAll(HotChunkDir(dataDir, chunkID)) } -// Per-CF tuning for the hot store, passed via rocksdb.Config.PerCFOptions: +// Per-CF tuning passed via rocksdb.Config.PerCFOptions: // -// - DataCF holds XDR-encoded event payloads: compressible (zstd -// typically 2-3× on XDR) and read in batches via -// BatchedMultiGetCF. Larger blocks give zstd more context per -// compression unit and align with batch-fetch shapes. -// - IndexCF stores 20-byte (term_hash || event_id) keys with -// empty values — nothing in the values to compress, and small -// blocks reduce wasted I/O per random Lookup miss (each Lookup -// reads one block to find one key). -// - OffsetsCF stores 8-byte (ledger_seq -> event_count) rows in -// the tens-of-thousands per chunk — same shape as IndexCF. +// - DataCF holds XDR payloads (compressible, batch-read) — larger blocks give +// zstd more context and align with batch-fetch shapes. +// - IndexCF holds 20-byte keys / empty values — small blocks cut wasted I/O per +// random Lookup (one block per key). +// - OffsetsCF holds 8-byte rows — same shape as IndexCF. const ( dataCFBlockSize = 32 * 1024 indexCFBlockSize = 4 * 1024 @@ -79,14 +74,17 @@ func hotStoreCFOptions() map[string]rocksdb.CFOptions { } } -// openHotChunk opens (or creates) chunkID's per-Chunk hot RocksDB DB -// at HotChunkDir(dataDir, chunkID). The three per-Chunk CFs are -// configured at New so they auto-create on a fresh DB and are -// rediscovered on a reopen. -// -// Unexported: OpenHotStore is the only caller and is the public way -// to open a per-Chunk hot DB (since the warmup step is mandatory -// before the store is usable). +// CFNames returns the three CFs this facade owns. Exported so the hotchunk +// shared-DB opener can register them alongside the other CFs (decision (a)). +func CFNames() []string { return []string{DataCF, IndexCF, OffsetsCF} } + +// CFOptions returns this facade's per-CF options. Exported so the hotchunk +// opener merges them into the shared per-chunk DB's PerCFOptions. +func CFOptions() map[string]rocksdb.CFOptions { return hotStoreCFOptions() } + +// openHotChunk opens (or creates) chunkID's per-Chunk hot DB. The three CFs +// auto-create on a fresh DB and rediscover on reopen. Unexported — OpenHotStore +// is the only caller (warmup is mandatory before the store is usable). func openHotChunk(dataDir string, chunkID chunk.ID, logger *supportlog.Entry) (*rocksdb.Store, error) { store, err := rocksdb.New(rocksdb.Config{ Path: HotChunkDir(dataDir, chunkID), @@ -119,49 +117,38 @@ var ErrLedgerOutOfRange = errors.New("events: ledger outside chunk range") // offset chain if not rejected up front. var ErrLedgerOutOfOrder = errors.New("events: ledger out of order") -// HotStore wraps one chunk's hot RocksDB DB plus the in-memory term -// mirror and ledger-offset cache that feed the query path. Reads and -// writes share the same struct; every HotStore owns its chunkStore -// exclusively and Close releases it. +// HotStore wraps one chunk's hot RocksDB DB plus the in-memory term mirror and +// ledger-offset cache that feed the query path. // -// Atomicity model: the per-Chunk DB is the source of truth. -// IngestLedgerEvents commits data + index + offsets to chunkStore in one -// atomic batch and then updates the in-memory mirrors. Warmup on next -// startup reconstructs the mirrors from the chunk's on-disk CFs. +// Atomicity: the per-Chunk DB is the source of truth. IngestLedgerEvents commits +// data + index + offsets in one atomic batch, then updates the in-memory +// mirrors; warmup reconstructs them from the on-disk CFs on next startup. // -// Concurrency model: +// Concurrency: // -// - Writes (IngestLedgerEvents) follow a single-writer contract — -// the orchestrator drives ingest from one goroutine per chunk. -// The in-memory mirror and offsets have their own concurrency -// primitives for the single-writer-vs-multi-reader pattern. -// - Reads (Lookup, FetchEvents, All) take NO HotStore-level lock. -// They fast-path-guard via h.chunkStore.IsClosed() and rely on -// the in-memory primitives' internal locks (for the mirror) and -// RocksDB's own thread-safety (for chunkStore). -// - Metadata accessors split by Close behavior: -// ChunkID, NextEventID, Index — infallible, return their cached -// value forever (usable for post-Close logging). -// EventCount, Offsets — return ErrClosed after Close, matching -// the ColdReader and Reader-interface contract. -// - Close delegates to chunkStore.Close, which is itself idempotent -// via rocksdb.Store's own atomic.Bool + CompareAndSwap. The -// in-memory mirror has no separate close step — it is dropped -// implicitly when HotStore is GC'd. +// - Writes (IngestLedgerEvents) are single-writer (one goroutine per chunk). +// - Reads (Lookup, FetchEvents, All) take NO HotStore-level lock — they guard +// via chunkStore.IsClosed() and rely on the mirror's internal locks and +// RocksDB's thread-safety. +// - Metadata split by Close: ChunkID, NextEventID, Index are infallible +// (cached, usable post-Close); EventCount, Offsets return ErrClosed after +// Close (Reader-interface contract). type HotStore struct { chunkStore *rocksdb.Store chunkID chunk.ID mirror *events.ConcurrentBitmaps offsets *events.ConcurrentLedgerOffsets + // ownsStore is true on the standalone OpenHotStore path; false when wrapping + // the SHARED per-chunk DB via NewWithStore (decision (a)), which hotchunk.DB + // owns and closes once. + ownsStore bool } // Compile-time guard: *HotStore satisfies Reader. var _ Reader = (*HotStore)(nil) -// OpenHotStore opens (or creates) chunkID's hot DB at -// HotChunkDir(dataDir, chunkID), warms up the in-memory mirror and -// offsets from disk, and returns a ready-to-use HotStore. The -// returned store owns its chunkStore; Close releases it. +// OpenHotStore opens (or creates) chunkID's hot DB, warms up the mirror + +// offsets from disk, and returns a ready HotStore that owns its chunkStore. func OpenHotStore( dataDir string, chunkID chunk.ID, @@ -178,42 +165,51 @@ func OpenHotStore( if err != nil { return nil, err } - mirror, offsets, err := warmup(chunkStore, chunkID) + h, err := NewWithStore(chunkStore, chunkID) if err != nil { _ = chunkStore.Close() + return nil, err + } + h.ownsStore = true + return h, nil +} + +// NewWithStore wraps an ALREADY-OPEN rocksdb.Store as an events HotStore on the +// three events CFs (CFNames()), running the mandatory warmup to rebuild the +// in-memory mirror + offsets. The store is NOT owned (Close is a no-op) — the +// constructor hotchunk uses to compose this facade over the shared per-chunk DB +// (decision (a)). The store must have CFNames() registered + CFOptions() applied. +// A warmup failure returns the error WITHOUT closing the caller-owned store. +func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) (*HotStore, error) { + mirror, offsets, err := warmup(store, chunkID) + if err != nil { return nil, fmt.Errorf("events: warmup chunk %s: %w", chunkID, err) } return &HotStore{ - chunkStore: chunkStore, + chunkStore: store, chunkID: chunkID, mirror: mirror, offsets: offsets, }, nil } -// Close releases the underlying chunk store. Idempotent — delegates -// to chunkStore.Close, which is itself idempotent via its own -// atomic.Bool + CompareAndSwap. The in-memory mirror is dropped -// implicitly when HotStore is GC'd. -// -// Concurrency: must not be called concurrently with in-flight read -// methods on the same HotStore (Lookup, FetchEvents, All). Callers -// drain those reads before invoking Close. The single-writer ingest -// contract means there is no concurrent IngestLedgerEvents call to -// race with either; chunkStore's IsClosed check inside -// IngestLedgerEvents fast-fails any post-Close ingest attempt. +// Close releases the chunk store IF this HotStore owns it (standalone +// OpenHotStore); a no-op when wrapping the shared per-chunk DB (NewWithStore), +// which hotchunk.DB closes once. Idempotent; not safe to call alongside in-flight +// reads/writes on this HotStore. func (h *HotStore) Close() error { + if !h.ownsStore { + return nil + } return h.chunkStore.Close() } // ChunkID returns the chunk this store serves. func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } -// EventCount is the total number of events committed to this Chunk -// so far. Equal to the next event-id IngestLedgerEvents would assign. -// Returns (0, ErrClosed) after Close. The Reader interface signature -// is fallible to accommodate ColdReader's lazy metadata load; on the -// hot side the value is always live and the error is only ErrClosed. +// EventCount is the total events committed to this Chunk so far (== the next +// event-id). Returns (0, ErrClosed) after Close. The fallible signature is for +// the Reader interface (ColdReader's lazy load); the hot value is always live. func (h *HotStore) EventCount() (uint32, error) { if h.chunkStore.IsClosed() { return 0, ErrClosed @@ -221,30 +217,16 @@ func (h *HotStore) EventCount() (uint32, error) { return h.offsets.TotalEvents(), nil } -// NextEventID is the next chunk-relative event ID IngestLedgerEvents -// will assign. Returns the same value as EventCount on the hot side -// and is exposed under both names for the ingest-side and reader-side -// mental models. Infallible at the type level (hot-only API, not on -// the Reader interface). +// NextEventID is the next chunk-relative event ID IngestLedgerEvents will +// assign. Same value as EventCount; exposed under both names for the ingest- and +// reader-side mental models. Infallible (hot-only, not on Reader). func (h *HotStore) NextEventID() uint32 { return h.offsets.TotalEvents() } -// Offsets returns a point-in-time view of the ledger-offset cache. -// The coordinator uses this to stitch a multi-ledger query range -// into chunk-relative event-id ranges (see Reader.Offsets). -// -// Implementation: returns a *LedgerOffsets sharing the live -// backing array, capped at the count visible at call time -// (~24-byte allocation per Query). Concurrent IngestLedgerEvents -// may extend the backing past the cap, but the returned view's -// slice stays bounded to what was visible when Offsets returned. -// Callers (Query) take the view once at entry and pass it through -// their helpers. -// -// Read-only: the returned view's underlying slice shares memory -// with the live backing array. Calling Append on the view would -// silently fork it from the live data; the contract is read-only. -// -// Returns (nil, ErrClosed) after Close. +// Offsets returns a point-in-time, read-only view of the ledger-offset cache +// (see Reader.Offsets), capped at the count visible at call time. A concurrent +// ingest may extend the backing past the cap; the view's slice stays bounded. +// The view shares the live backing array — Append on it would silently fork it, +// so the contract is read-only. Returns (nil, ErrClosed) after Close. func (h *HotStore) Offsets() (*events.LedgerOffsets, error) { if h.chunkStore.IsClosed() { return nil, ErrClosed @@ -252,24 +234,15 @@ func (h *HotStore) Offsets() (*events.LedgerOffsets, error) { return h.offsets.View(), nil } -// Index returns the in-memory term mirror. Used by the freezer to -// snapshot every (events.TermKey, bitmap) pair into WriteColdIndex -// without rebuilding from RocksDB. Callers should typically call -// h.Index().Snapshot() to get a uniquely owned Bitmaps for -// serialization. +// Index returns the in-memory term mirror, used by the freezer to snapshot every +// (TermKey, bitmap) pair without rebuilding from RocksDB. Callers typically use +// h.Index().Snapshot() for a uniquely owned Bitmaps. func (h *HotStore) Index() *events.ConcurrentBitmaps { return h.mirror } -// Lookup returns the bitmap of event IDs in this Chunk that match -// the given term. The returned bitmap is an immutable snapshot of -// the live mirror — writers publish new pointers via atomic.Store -// (see ConcurrentBitmaps), so the caller never observes a mutating -// bitmap. Callers MUST NOT mutate it themselves. See Reader.Lookup -// and ConcurrentBitmaps.Get for the full contract. Returns -// (nil, ErrTermNotFound) when the term has no matching events. -// Returns (nil, ErrClosed) after Close. -// -// ctx is checked as a fast guard but the hot path does no blocking -// I/O — the bitmap comes from the in-memory mirror. +// Lookup returns the bitmap of event IDs in this Chunk matching key. The bitmap +// is an immutable snapshot of the live mirror (writers publish via atomic.Store) +// — callers MUST NOT mutate it. Returns (nil, ErrTermNotFound) on no match, +// (nil, ErrClosed) after Close. ctx is a fast guard; the hot path does no I/O. func (h *HotStore) Lookup(ctx context.Context, key events.TermKey) (*roaring.Bitmap, error) { if h.chunkStore.IsClosed() { return nil, ErrClosed @@ -287,15 +260,9 @@ func (h *HotStore) Lookup(ctx context.Context, key events.TermKey) (*roaring.Bit return bm, nil } -// LookupKeys returns bitmaps for each key, aligned positionally with -// the input slice. result[i] is nil if keys[i] has no matching -// events. See Reader.LookupKeys for the semantics — in particular -// the borrowed-bitmap contract (callers must not mutate). -// -// Hot-side implementation is N in-memory mirror lookups — no I/O -// to batch — but exposing this method satisfies the Reader -// interface so callers can program against batched lookups -// uniformly. +// LookupKeys returns bitmaps positionally aligned with keys; result[i] is nil on +// no match. See Reader.LookupKeys (borrowed-bitmap contract: callers must not +// mutate). Hot-side is just N mirror lookups, exposed only to satisfy Reader. func (h *HotStore) LookupKeys(ctx context.Context, keys []events.TermKey) ([]*roaring.Bitmap, error) { if h.chunkStore.IsClosed() { return nil, ErrClosed @@ -317,26 +284,15 @@ func (h *HotStore) LookupKeys(ctx context.Context, keys []events.TermKey) ([]*ro return results, nil } -// FetchEvents decodes the events_data row for each provided eventID -// and returns them positionally aligned with the input slice. See -// Reader.FetchEvents for the sorted-input precondition. +// FetchEvents decodes the events_data row for each eventID, positionally aligned +// with the input. See Reader.FetchEvents for the sorted-input precondition +// (violations return wrapped ErrUnsortedEventIDs). One BatchMultiGet crosses CGO +// once regardless of count and enables async_io (a win on high-latency storage); +// ctx is honored at entry but the CGO call is not cancellable mid-flight. // -// Implementation: validates eventIDs are sorted ascending with no -// duplicates (returns wrapped ErrUnsortedEventIDs otherwise — same -// shape as the cold side), encodes them to BE-uint32 keys, then -// calls rocksdb.Store.BatchMultiGet once with sortedInput=true. -// The batched API crosses CGO a single time regardless of key count -// and enables async_io so the kernel can overlap SST page reads — -// a meaningful win on EBS / high-random-latency storage. ctx is -// honored at the top of the call; the underlying CGO call is not -// cancellable mid-flight. -// -// A missing row is an error: eventIDs only reach this path through -// Lookup, which only returns IDs the mirror knows about — implying -// RocksDB also has them. A miss indicates corruption or a -// writer/reader mismatch, not a normal not-found case. -// -// After Close, returns ErrClosed. +// A missing row is an error, not a normal miss: IDs only reach here via Lookup, +// which only returns IDs the mirror (hence RocksDB) has — a miss means +// corruption. Returns ErrClosed after Close. func (h *HotStore) FetchEvents(ctx context.Context, eventIDs []uint32) ([]events.Payload, error) { if h.chunkStore.IsClosed() { return nil, ErrClosed @@ -373,10 +329,8 @@ func (h *HotStore) FetchEvents(ctx context.Context, eventIDs []uint32) ([]events if v == nil { return nil, fmt.Errorf("events: event %d missing from chunk %s", id, h.chunkID) } - // BatchMultiGet already copies out of rocksdb's pinned pages - // (see rocksdb.Store.BatchMultiGet); v is Go-owned and outlives - // the returned Payload, so Unmarshal's alias is safe without - // an extra clone. + // BatchMultiGet already copies out of rocksdb's pinned pages; v is + // Go-owned, so Unmarshal's alias is safe without an extra clone. if err := results[i].Unmarshal(v); err != nil { return nil, fmt.Errorf("events: decode event %d from chunk %s: %w", id, h.chunkID, err) } @@ -384,27 +338,15 @@ func (h *HotStore) FetchEvents(ctx context.Context, eventIDs []uint32) ([]events return results, nil } -// FetchRange streams count events starting at chunk-relative event -// ID start, in ascending eventID order. See Reader.FetchRange for -// semantics; the hot path drives rocksdb.Store.IterateRange over -// DataCF with start and end keys derived from encodeDataKey. -// -// Yielded Payloads are borrowed: ContractEventBytes aliases the iteration -// buffer and is valid only until the next step — clone to retain. +// FetchRange streams count events from chunk-relative event ID start, ascending +// (see Reader.FetchRange). Yielded Payloads are borrowed: ContractEventBytes +// aliases the iteration buffer, valid only until the next step — clone to retain. +// After Close yields (zero, ErrClosed). ctx is checked at entry and between +// steps; IterateRange takes no ctx, so a slow Next can block past a cancel. // -// After Close, yields (zero Payload, ErrClosed) and stops. -// ctx is checked at entry and between iterator steps — -// rocksdb.Store.IterateRange does not itself accept a ctx, so a -// very slow Next() can block past a cancellation until the next -// yielded entry observes the cancel. -// -// Out-of-range arguments yield an error and stop: -// - count == 0 is a natural no-op (no yields). -// - start+count > NextEventID (overflow-safe via uint64) yields a -// wrapped out-of-bounds error. -// - A short scan (fewer DataCF rows than count) yields a wrapped -// error after the partial stream — the CF should be dense in -// [0, NextEventID), so a hole indicates corruption. +// Out-of-range args yield an error and stop: count==0 is a no-op; start+count > +// NextEventID is out-of-bounds; a short scan (fewer rows than count) signals +// corruption (the CF should be dense in [0, NextEventID)). func (h *HotStore) FetchRange(ctx context.Context, start, count uint32) iter.Seq2[events.Payload, error] { return func(yield func(events.Payload, error) bool) { if h.chunkStore.IsClosed() { @@ -436,10 +378,8 @@ func (h *HotStore) FetchRange(ctx context.Context, start, count uint32) iter.Seq return } var p events.Payload - // entry.Value is a zero-copy ref into the IterateRange - // iterator buffer, valid only for this step; Unmarshal aliases - // it into p.ContractEventBytes, so the yielded Payload is - // borrowed (see the FetchRange doc). A retaining consumer clones. + // entry.Value is a zero-copy ref valid only this step; Unmarshal + // aliases it into p, so the yielded Payload is borrowed (clone to retain). if err := p.Unmarshal(entry.Value); err != nil { yield(events.Payload{}, fmt.Errorf("events: decode event from chunk %s: %w", h.chunkID, err)) @@ -458,16 +398,11 @@ func (h *HotStore) FetchRange(ctx context.Context, start, count uint32) iter.Seq } } -// All streams every event in this Chunk in chunk-relative eventID -// order. Used by the freeze loop to dump a hot Chunk into a -// ColdWriter without buffering. Thin wrapper over FetchRange; its -// yielded Payloads are likewise borrowed (valid only for the step). -// -// NextEventID is read inside the returned closure body, so a -// concurrent ingest between r.All(ctx) returning the Seq2 and the -// consumer's first range step is included in the snapshot. -// -// After Close, yields (zero Payload, ErrClosed) and stops. +// All streams every event in this Chunk in eventID order — used by the freeze +// loop to dump a hot Chunk without buffering. Thin wrapper over FetchRange (same +// borrowed-Payload contract). NextEventID is read inside the closure, so an +// ingest between All returning and the first range step is included. After Close +// yields (zero, ErrClosed). func (h *HotStore) All(ctx context.Context) iter.Seq2[events.Payload, error] { return func(yield func(events.Payload, error) bool) { // FetchRange stops iterating after yielding an error; we @@ -480,141 +415,192 @@ func (h *HotStore) All(ctx context.Context) iter.Seq2[events.Payload, error] { } } -// IngestLedgerEvents commits one ledger's events to the chunk store -// atomically and then updates the in-memory mirrors. -// -// payloads is produced by events.LCMViewToPayloads, which emits each ledger's -// events in ascending getEvents cursor order — write order here IS the -// cursor contract (event IDs are assigned by arrival position). Terms are -// derived internally via events.TermsForBytes on each payload's -// ContractEventBytes. -// -// Sequence validation is performed up front, before any RocksDB -// write or mirror mutation: -// -// - ledgerSeq must lie within [chunkID.FirstLedger(), -// chunkID.LastLedger()] — out-of-range returns ErrLedgerOutOfRange. -// - ledgerSeq == the next expected ledger (StartLedger + LedgerCount) -// is appended normally. -// - ledgerSeq < expected (an already-ingested ledger) is an idempotent -// no-op returning nil, so a restarted ingester can blindly re-deliver -// the in-flight ledger; the re-delivered events are not re-verified. -// - ledgerSeq > expected (a gap) returns ErrLedgerOutOfOrder. -// -// A rejected call (out-of-range or gap) completes its checks before -// marshaling, leaving the chunk store and in-memory mirrors untouched. -// -// Post-batch atomicity: once the RocksDB batch commits, the in-memory -// mirror + offsets updates are infallible by construction. Any -// failure there panics rather than returning an error, because a -// returned error would leave on-disk state ahead of in-memory state -// with no clean recovery short of close + reopen. -// -//nolint:cyclop // sequential pipeline: validate -> marshal -> batch -> mirror updates +// IngestLedgerEvents commits one ledger's events to the chunk store atomically, +// then updates the in-memory mirrors. payloads (from LCMViewToPayloads) arrive in +// getEvents cursor order; write order here IS the cursor contract (event IDs are +// assigned by arrival position). Terms are derived internally via TermsForBytes. +// +// Sequence validation runs up front (before any write or mirror mutation), so a +// rejected call leaves all state untouched: +// - out of [chunkID.FirstLedger(), LastLedger()] → ErrLedgerOutOfRange. +// - == expected (StartLedger + LedgerCount) → appended. +// - < expected (already ingested) → idempotent no-op nil (a restarted ingester +// can blindly re-deliver; the re-delivered events are not re-verified). +// - > expected (a gap) → ErrLedgerOutOfOrder. +// +// Post-batch atomicity: once the batch commits, the mirror + offsets updates are +// infallible — a failure there panics rather than leaving on-disk state ahead of +// in-memory with no clean recovery short of close + reopen. func (h *HotStore) IngestLedgerEvents(ledgerSeq uint32, payloads []events.Payload) error { if h.chunkStore.IsClosed() { return ErrClosed } - // Validate ledger sequence BEFORE any disk write or mirror mutation. - // Failing the offsets.Append check after the RocksDB batch has - // committed would leave events orphaned under a bad ledger key. + // Same prepare → queue → commit → apply pipeline hotchunk drives across the + // shared DB; here the batch holds only the events CFs. + apply, err := h.IngestLedgerToBatchCommit(ledgerSeq, payloads) + if err != nil { + return err + } + if apply != nil { + apply() + } + return nil +} + +// IngestLedgerToBatchCommit is IngestLedgerEvents over a batch this facade owns +// end-to-end (validate → marshal → one synced batch). Returns the post-commit +// apply hook (mirror+offsets) to run after the batch is durable, or (nil, nil) +// for an idempotent duplicate. Split out so IngestLedgerToBatch can share the +// prepare step while committing into a SHARED cross-CF batch instead. +func (h *HotStore) IngestLedgerToBatchCommit(ledgerSeq uint32, payloads []events.Payload) (func(), error) { + prep, err := h.prepareLedger(ledgerSeq, payloads) + if err != nil { + return nil, err + } + if prep == nil { + return nil, nil // idempotent duplicate no-op + } + if cerr := h.chunkStore.Batch(func(b *rocksdb.BatchWriter) error { + return prep.queue(b) + }); cerr != nil { + return nil, fmt.Errorf("events: commit ledger %d to chunk %s: %w", ledgerSeq, h.chunkID, cerr) + } + return prep.apply, nil +} + +// IngestLedgerToBatch validates+marshals one ledger's events and queues their CF +// Puts into the SHARED batch b, returning the post-commit apply hook the caller +// runs AFTER b commits (decision (a)). Returns (nil, nil) for an idempotent +// duplicate. All validation + term derivation happen up front, so a rejected +// ledger leaves b untouched. +func (h *HotStore) IngestLedgerToBatch(b *rocksdb.BatchWriter, ledgerSeq uint32, payloads []events.Payload) (func(), error) { + if h.chunkStore.IsClosed() { + return nil, ErrClosed + } + prep, err := h.prepareLedger(ledgerSeq, payloads) + if err != nil { + return nil, err + } + if prep == nil { + return nil, nil + } + if qerr := prep.queue(b); qerr != nil { + return nil, qerr + } + return prep.apply, nil +} + +// preparedLedger is one validated, marshaled ledger ready to queue into +// a write batch (queue) and, once that batch is durable, apply to the +// in-memory mirror + offsets (apply). +type preparedLedger struct { + ledgerSeq uint32 + startID uint32 + blobs [][]byte // marshaled payload XDR, positional with payloads + termKeys [][]events.TermKey // per-payload term keys + apply func() // post-commit mirror + offsets update (infallible) +} + +// queue writes the prepared ledger's rows into b: one DataCF row per +// event, one IndexCF row per (term, event), and one OffsetsCF row for +// the ledger's per-ledger event count. +func (p *preparedLedger) queue(b *rocksdb.BatchWriter) error { + for i := range p.blobs { + eventID := p.startID + uint32(i) + b.Put(DataCF, encodeDataKey(eventID), p.blobs[i]) + for _, key := range p.termKeys[i] { + b.Put(IndexCF, encodeIndexKey(key, eventID), nil) + } + } + //nolint:gosec // bounds-checked in prepareLedger's overflow guard + eventCount := uint32(len(p.blobs)) + b.Put(OffsetsCF, encodeOffsetKey(p.ledgerSeq), encodeLedgerEventCount(eventCount)) + return nil +} + +// prepareLedger runs the pre-commit pipeline for one ledger (validate → derive +// terms → marshal into fresh per-event buffers), returning a *preparedLedger +// ready to queue + apply, or (nil, nil) for an idempotent duplicate. It does NO +// disk write and NO mirror mutation, so it is safe to call before touching a +// shared batch. +// +//nolint:cyclop // sequential pipeline: validate -> derive terms -> marshal -> build apply hook +func (h *HotStore) prepareLedger(ledgerSeq uint32, payloads []events.Payload) (*preparedLedger, error) { + // Validate BEFORE marshaling: failing after a shared batch holds this + // ledger's rows would orphan them. if ledgerSeq < h.chunkID.FirstLedger() || ledgerSeq > h.chunkID.LastLedger() { - return fmt.Errorf("%w: ledger %d not in chunk %s [%d, %d]", + return nil, fmt.Errorf("%w: ledger %d not in chunk %s [%d, %d]", ErrLedgerOutOfRange, ledgerSeq, h.chunkID, h.chunkID.FirstLedger(), h.chunkID.LastLedger()) } expected := h.offsets.StartLedger() + uint32(h.offsets.LedgerCount()) //nolint:gosec if ledgerSeq < expected { - // Already ingested: idempotent retry no-op. A restarted ingester - // can blindly re-deliver an already-committed ledger; drop it - // rather than erroring or double-appending. The re-delivered - // events are not re-verified, so a re-delivery carrying different - // events for an already-ingested ledger is silently ignored. - return nil + // Already ingested: idempotent no-op (a restarted ingester may + // re-deliver). Re-delivered events are not re-verified. + return nil, nil } if ledgerSeq > expected { - return fmt.Errorf("%w: expected ledger %d, got %d", + return nil, fmt.Errorf("%w: expected ledger %d, got %d", ErrLedgerOutOfOrder, expected, ledgerSeq) } - // Pre-derive term keys per payload so the post-commit mirror - // update doesn't re-hash. Surfacing TermsForBytes errors here - // (pre-batch) cleanly rejects the ledger commit without touching disk — - // a decode failure on stellar-core-validated XDR is a corruption - // signal worth aborting on. + // Pre-derive term keys so the post-commit mirror update needn't re-hash. A + // TermsForBytes error rejects the ledger without touching the batch (a decode + // failure on core-validated XDR is a corruption signal worth aborting on). termKeys := make([][]events.TermKey, len(payloads)) for i := range payloads { keys, err := events.TermsForBytes(payloads[i].ContractEventBytes) if err != nil { - return fmt.Errorf("events: derive terms for payload %d in ledger %d: %w", i, ledgerSeq, err) + return nil, fmt.Errorf("events: derive terms for payload %d in ledger %d: %w", i, ledgerSeq, err) } termKeys[i] = keys } startID := h.offsets.TotalEvents() if uint64(startID)+uint64(len(payloads)) > math.MaxUint32 { - return fmt.Errorf("events: chunk %s would overflow uint32 event-id space at ledger %d", + return nil, fmt.Errorf("events: chunk %s would overflow uint32 event-id space at ledger %d", h.chunkID, ledgerSeq) } - // Atomic batch on the per-Chunk DB. Each payload is marshaled into one - // reused scratch buffer: BatchWriter.Put copies the value into the write - // batch synchronously, so the scratch is free to reuse on the next - // iteration — no per-payload allocation. A marshal error returns from - // the callback, which aborts the batch so nothing commits. - var scratch []byte - err := h.chunkStore.Batch(func(b *rocksdb.BatchWriter) error { - for i := range payloads { - eventID := startID + uint32(i) - blob, err := payloads[i].MarshalInto(scratch[:0]) - if err != nil { - return fmt.Errorf("events: marshal payload %d for ledger %d: %w", i, ledgerSeq, err) - } - scratch = blob - b.Put(DataCF, encodeDataKey(eventID), blob) - for _, key := range termKeys[i] { - b.Put(IndexCF, encodeIndexKey(key, eventID), nil) - } + // Marshal each payload into its OWN fresh buffer (not reused scratch): a + // shared batch may hold many ledgers' rows before commit, so each blob must + // outlive prepare until the Write copies it. BatchWriter.Put copies + // synchronously, so the buffers are free after queue returns. + blobs := make([][]byte, len(payloads)) + for i := range payloads { + blob, err := payloads[i].MarshalInto(nil) + if err != nil { + return nil, fmt.Errorf("events: marshal payload %d for ledger %d: %w", i, ledgerSeq, err) } - // On-disk shape matches the in-memory API: per-ledger event - // count, not cumulative. Warmup replays directly via - // offsets.Append(eventCount) — no delta arithmetic. - //nolint:gosec // bounds-checked above - eventCount := uint32(len(payloads)) - b.Put(OffsetsCF, encodeOffsetKey(ledgerSeq), encodeLedgerEventCount(eventCount)) - return nil - }) - if err != nil { - return fmt.Errorf("events: commit ledger %d to chunk %s: %w", ledgerSeq, h.chunkID, err) - } - - // Phase 3: the batch is durable — apply it to the in-memory cache. - // Infallible given the validation above (ledgerSeq == expected and - // in-chunk, single writer): mirror.AddTo cannot fail and offsets.Append - // appends at the already-validated next slot, so the only - // non-completion is a crash, after which warmup rebuilds the cache from - // disk. - // - // Ordering invariant: mirror BEFORE offsets. A concurrent Query - // that captures offsets via h.offsets.Snapshot() then later calls - // mirror.Get for the same key sees either the previous state - // (offsets count N-1, mirror without ledger-N events) or a - // consistent later one (offsets count ≥N, mirror with ledger-N - // events). Reversing the order would let a reader observe an - // offsets count that includes IDs the mirror hasn't published - // yet — Query would then ask FetchEvents for IDs not yet - // indexed; the bitmap intersection would simply miss them, with - // no error surface. - // - // Batch by key so each ConcurrentBitmaps.AddTo call clones at most - // once per (key, ledger), not once per (key, event). For popular - // terms that receive many events in one ledger this turns N COW - // clones into 1. Initial capacity 64 ≈ a few × unique-terms per - // typical ledger; the map grows correctly past that. + blobs[i] = blob + } + + prep := &preparedLedger{ + ledgerSeq: ledgerSeq, + startID: startID, + blobs: blobs, + termKeys: termKeys, + } + prep.apply = func() { h.applyLedger(prep) } + return prep, nil +} + +// applyLedger updates the mirror + offsets for a ledger whose rows are durable. +// Infallible by construction (prepare validated seq under the single-writer +// contract); the only non-completion is a crash, after which warmup rebuilds. +// +// Ordering invariant: mirror BEFORE offsets. A concurrent Query that snapshots +// offsets then reads the mirror must see either the prior state or a consistent +// later one. Reversing it would let a reader see an offsets count including IDs +// the mirror hasn't published — FetchEvents would then miss them, silently. +func (h *HotStore) applyLedger(p *preparedLedger) { + // Batch by key so each AddTo clones at most once per (key, ledger), not per + // (key, event) — turns N COW clones into 1 for popular terms. Cap 64 ≈ a few + // × unique-terms per ledger; the map grows past that. perKeyIDs := make(map[events.TermKey][]uint32, 64) - for i, keys := range termKeys { - eventID := startID + uint32(i) + for i, keys := range p.termKeys { + eventID := p.startID + uint32(i) for _, key := range keys { perKeyIDs[key] = append(perKeyIDs[key], eventID) } @@ -622,29 +608,16 @@ func (h *HotStore) IngestLedgerEvents(ledgerSeq uint32, payloads []events.Payloa for key, ids := range perKeyIDs { h.mirror.AddTo(key, ids...) } - //nolint:gosec // len bounded by the overflow check above - h.offsets.Append(uint32(len(payloads))) - return nil + //nolint:gosec // len bounded by prepareLedger's overflow guard + h.offsets.Append(uint32(len(p.blobs))) } -// ────────────────────────────────────────────────────────────────── -// Warmup — reconstructs the in-memory mirror + offsets from the -// per-Chunk DB's on-disk CFs. Called only by OpenHotStore. -// ────────────────────────────────────────────────────────────────── +// Warmup — reconstructs the in-memory mirror + offsets from the on-disk CFs. -// warmup rebuilds the in-memory mirrors for chunkID by prefix-scanning -// the chunk's two on-disk caches once each: -// -// - events_index → *events.ConcurrentBitmaps — every -// (events.TermKey, eventID) row replayed into a fresh in-memory -// bitmap mirror. -// - events_offsets → *events.ConcurrentLedgerOffsets — every -// (ledger_seq, per_ledger_count) row replayed into a fresh -// offset cache. -// -// chunkID seeds events.ConcurrentLedgerOffsets.StartLedger for empty -// chunks; on-disk rows carry the full ledger sequence themselves. -// Both mirrors are empty for fresh chunks. +// warmup rebuilds chunkID's in-memory mirrors by scanning the two on-disk caches +// once each: events_index → ConcurrentBitmaps, events_offsets → +// ConcurrentLedgerOffsets. chunkID seeds StartLedger for empty chunks; both +// mirrors are empty for fresh chunks. func warmup( chunkStore *rocksdb.Store, chunkID chunk.ID, ) (*events.ConcurrentBitmaps, *events.ConcurrentLedgerOffsets, error) { @@ -662,25 +635,19 @@ func warmup( return mirror, offsets, nil } -// verifyChunkConsistency cross-checks the three on-disk CFs after warmup, -// turning a torn or tampered chunk into a loud open failure instead of a -// silently inconsistent in-memory cache. The CFs are written in one -// atomic batch, so under normal operation these invariants always hold; -// a violation means a bug or external corruption. +// verifyChunkConsistency cross-checks the three on-disk CFs after warmup, turning +// a torn/tampered chunk into a loud open failure. A cheap open-time tripwire, not +// load-bearing correctness (the atomic batch makes violations impossible for the +// writer; only a bug/corruption trips it): // -// - the index may not reference an event the offsets don't account for: -// indexUpperBound (max indexed event ID + 1, 0 if none) <= total. -// - the data tail matches total: event total-1 present (when total > 0) -// and no data row at any id >= total. Together those pin the max data -// id to exactly total-1 — one Get plus one bounded seek. +// - index may not reference an event offsets don't account for: +// indexUpperBound (max indexed id + 1, 0 if none) <= total. +// - data tail matches total: id total-1 present (when total > 0) and nothing at +// id >= total — together pinning the max data id to total-1. // -// Not detected here: interior data holes (a missing id within 0..total-2, -// masked by a higher present id), under-indexed terms, and wrong -// per-ledger boundaries — each would need a full scan. The atomic batch -// makes all of them impossible for the writer; an interior hole that did -// appear (corruption/tamper) is caught lazily by FetchRange's short-scan -// check on first read. This is a cheap open-time tripwire on denormalized -// state, not load-bearing correctness. +// Not detected (would need a full scan): interior data holes, under-indexed +// terms, wrong per-ledger boundaries. An interior hole that did appear is caught +// lazily by FetchRange's short-scan check. func verifyChunkConsistency(chunkStore *rocksdb.Store, total, indexUpperBound uint32) error { if indexUpperBound > total { return fmt.Errorf("events: corrupt chunk: index references event %d but only %d committed", @@ -696,9 +663,8 @@ func verifyChunkConsistency(chunkStore *rocksdb.Store, total, indexUpperBound ui total, total-1) } } - // Nothing may live at or beyond total. The bounded seek lands on the - // first such row if one exists; reaching the loop body at all (with no - // iteration error) means an orphan is present — at total or far past it. + // Nothing may live at or beyond total. Reaching the loop body (no iteration + // error) means an orphan row is present. for _, err := range chunkStore.IterateRange(DataCF, encodeDataKey(total), nil) { if err != nil { return fmt.Errorf("events: verify data tail: %w", err) @@ -708,22 +674,13 @@ func verifyChunkConsistency(chunkStore *rocksdb.Store, total, indexUpperBound ui return nil } -// warmupIndex scans the events_index CF and replays every -// (events.TermKey, eventID) row into a fresh events.ConcurrentBitmaps. -// Design doc §12 step 3. -// -// Implementation: build into a single-threaded events.Bitmaps via -// per-term batching (rocksdb's byte-sorted iteration delivers all -// rows for term K consecutively, so a small buffer flushes when the -// term changes), then convert to ConcurrentBitmaps at the end. This -// avoids paying the per-row Clone cost the concurrent ConcurrentBitmaps.AddTo -// would do for popular terms — without batching, warmup of a -// 10M-event chunk does ~50M Clones (one per index row) and saturates -// GC for many minutes. -// -// Also returns the exclusive upper bound of indexed event IDs (max + 1, -// or 0 if the index is empty) so warmup can cross-check it against the -// committed event count. +// warmupIndex replays every (TermKey, eventID) row of events_index into a fresh +// ConcurrentBitmaps. Builds into a single-threaded Bitmaps via per-term batching +// (byte-sorted iteration groups a term's rows, flushed on term change), then +// converts at the end — avoiding the per-row Clone the concurrent AddTo would do +// for popular terms (a 10M-event chunk would otherwise do ~50M Clones, saturating +// GC). Also returns the exclusive upper bound of indexed IDs (max + 1, 0 if +// empty) for warmup's cross-check against the committed count. func warmupIndex(chunkStore *rocksdb.Store) (*events.ConcurrentBitmaps, uint32, error) { builder := events.NewBitmaps() var ( @@ -766,18 +723,11 @@ func warmupIndex(chunkStore *rocksdb.Store) (*events.ConcurrentBitmaps, uint32, return events.NewConcurrentBitmapsFromBitmaps(builder), indexUpperBound, nil } -// warmupOffsets scans events_offsets and replays every (ledger_seq, -// event_count) row into a fresh *events.ConcurrentLedgerOffsets. The -// on-disk shape matches the in-memory Append input directly -// (per-ledger counts, not cumulative), so no delta arithmetic is -// needed. -// -// Iteration order is byte-sorted == numeric-sorted under the big-endian -// uint32 key encoding, so rows arrive in ledger order. On-disk rows are -// untrusted, so each is validated as the next in-chunk ledger before the -// positional Append — a gap or stray row is rejected here rather than -// silently mis-attributing counts (ConcurrentLedgerOffsets.Append no -// longer checks the sequence; the trust boundary is here). +// warmupOffsets replays every (ledger_seq, event_count) row of events_offsets +// into a fresh ConcurrentLedgerOffsets. The on-disk shape (per-ledger counts) +// matches Append's input directly. BE-uint32 keys sort in ledger order; on-disk +// rows are untrusted, so each is validated as the next in-chunk ledger before the +// positional Append (the trust boundary moved here — Append no longer checks). func warmupOffsets(chunkStore *rocksdb.Store, chunkID chunk.ID) (*events.ConcurrentLedgerOffsets, error) { offsets := events.NewConcurrentLedgerOffsets(chunkID.FirstLedger()) @@ -795,10 +745,9 @@ func warmupOffsets(chunkStore *rocksdb.Store, chunkID chunk.ID) (*events.Concurr } ledger := binary.BigEndian.Uint32(entry.Key) eventCount := binary.BigEndian.Uint32(entry.Value) - // Each row must be the next sequential ledger and within the - // chunk. The first test catches a gap, an out-of-order row, or a - // wrong start; the second catches an excess row past the chunk - // (which would otherwise append past capacity and panic). + // Each row must be the next sequential in-chunk ledger: the first test + // catches a gap/out-of-order/wrong-start, the second an excess row past + // the chunk (which would append past capacity and panic). if expected := offsets.EndLedger(); ledger != expected || ledger > chunkID.LastLedger() { return nil, fmt.Errorf("events: warmup offsets: chunk %s expected ledger %d, got %d", chunkID, expected, ledger) @@ -814,9 +763,7 @@ func warmupOffsets(chunkStore *rocksdb.Store, chunkID chunk.ID) (*events.Concurr return offsets, nil } -// ────────────────────────────────────────────────────────────────── // Key encoding helpers — RocksDB key layouts for the per-Chunk DB. -// ────────────────────────────────────────────────────────────────── func encodeDataKey(eventID uint32) []byte { var key [dataKeyLen]byte diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go new file mode 100644 index 000000000..2dca2e546 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -0,0 +1,235 @@ +// Package hotchunk implements decision (a): the per-chunk hot tier is ONE +// RocksDB holding the union of every hot data type's CFs (ledger + 3 events + 16 +// nibble-routed txhash), and each ledger commits as ONE atomic synced WriteBatch +// across ALL of them — so a ledger is fully present or fully absent, with a +// SINGLE per-chunk watermark (max committed seq, from the ledgers CF's last key) +// and no per-store frontiers / min-of-three. The three typed facades +// (ledger/txhash/eventstore HotStore) are composed over the shared store via +// NewWithStore; their write paths queue Puts into the one shared batch. +package hotchunk + +import ( + "fmt" + + sdkingest "github.com/stellar/go-stellar-sdk/ingest" + supportlog "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/go-stellar-sdk/xdr" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/events" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash" +) + +// DB is one chunk's hot tier: a single multi-CF rocksdb.Store plus the three +// typed facades composed over it. It owns the store (Close closes it once); the +// facades wrap it without owning it. +// +// Concurrency: ingestion is single-writer; IngestLedger is not safe to call +// concurrently with itself. Reads via the facades follow each facade's own +// contract and are safe alongside the single writer. +type DB struct { + store *rocksdb.Store + chunkID chunk.ID + + ledger *ledger.HotStore + txhash *txhash.HotStore + events *eventstore.HotStore +} + +// columnFamilies is the full CF list for the shared per-chunk DB (ledger + 3 +// events + 16 txhash). Names are already non-colliding across the facades. +func columnFamilies() []string { + cfs := []string{ledger.LedgersCF} + cfs = append(cfs, eventstore.CFNames()...) + cfs = append(cfs, txhash.CFNames()...) + return cfs +} + +// config builds the shared store's rocksdb.Config: events' per-CF options (ZSTD +// on DataCF, tuned block sizes) plus the txhash workload's Tuning. Tuning's +// per-CF fields apply to every CF — a benign over-application (ledger/events CFs +// just gain a bloom + larger write buffer); the per-CF overrides keep events +// distinct. +func config(path string, logger *supportlog.Entry, readOnly bool) rocksdb.Config { + return rocksdb.Config{ + Path: path, + ColumnFamilies: columnFamilies(), + Logger: logger, + Tuning: txhash.Tuning(), + PerCFOptions: eventstore.CFOptions(), + ReadOnly: readOnly, + } +} + +// Open opens (or creates) the chunk's shared multi-CF hot DB read-WRITE +// (ingestion's handle) and composes the three facades over it. On any +// facade-construction failure the shared store is closed before returning. +func Open(path string, chunkID chunk.ID, logger *supportlog.Entry) (*DB, error) { + return open(path, chunkID, logger, false) +} + +// OpenReadOnly opens an EXISTING hot DB read-only — the freeze source's view. The +// freeze only ever opens a chunk ingestion has already cleanly closed, so all +// data is in SST (no WAL to replay); composing the facades only reads. +func OpenReadOnly(path string, chunkID chunk.ID, logger *supportlog.Entry) (*DB, error) { + return open(path, chunkID, logger, true) +} + +func open(path string, chunkID chunk.ID, logger *supportlog.Entry, readOnly bool) (*DB, error) { + if path == "" { + return nil, stores.ErrInvalidConfig + } + if logger == nil { + return nil, stores.ErrInvalidConfig + } + store, err := rocksdb.New(config(path, logger, readOnly)) + if err != nil { + return nil, fmt.Errorf("hotchunk: open chunk %s: %w", chunkID, err) + } + + es, err := eventstore.NewWithStore(store, chunkID) + if err != nil { + _ = store.Close() + return nil, fmt.Errorf("hotchunk: compose events facade for chunk %s: %w", chunkID, err) + } + return &DB{ + store: store, + chunkID: chunkID, + ledger: ledger.NewWithStore(store, chunkID), + txhash: txhash.NewWithStore(store, chunkID), + events: es, + }, nil +} + +// ChunkID returns the chunk this DB is bound to. +func (d *DB) ChunkID() chunk.ID { return d.chunkID } + +// Ledgers returns the ledger read/write facade over the shared store. +func (d *DB) Ledgers() *ledger.HotStore { return d.ledger } + +// Txhash returns the txhash read/write facade over the shared store. +func (d *DB) Txhash() *txhash.HotStore { return d.txhash } + +// Events returns the events read/write facade over the shared store. +func (d *DB) Events() *eventstore.HotStore { return d.events } + +// Close releases the shared store exactly once. Idempotent. Must not be called +// concurrently with in-flight reads/writes. +func (d *DB) Close() error { return d.store.Close() } + +// MaxCommittedSeq returns the single authoritative per-chunk watermark: the +// highest seq durably committed, from the ledgers CF's last key. Under decision +// (a) this one value pins EVERY CF's frontier. ok=false on an empty DB. +func (d *DB) MaxCommittedSeq() (seq uint32, ok bool, err error) { + return d.ledger.LastSeq() +} + +// Ingest toggles which data types the single per-ledger batch writes. Mirrors +// ingest.Config but kept local so hotchunk needn't depend on ingest. +type Ingest struct { + Ledgers bool + Txhash bool + Events bool +} + +// LedgerCounts reports how many items each data type contributed to one +// IngestLedger call, so the caller can emit per-type volume metrics. +type LedgerCounts struct { + Ledgers int + Txhash int + Events int +} + +// IngestLedger commits ONE ledger as a SINGLE atomic synced WriteBatch across all +// enabled CFs (decision (a)): queue each enabled type's rows into one +// BatchWriter, commit once, and only then apply the events in-memory +// mirror/offsets update. +// +// lcm is a borrowed zero-copy view; every extractor copies what it retains, so +// the view need not outlive this call. An idempotent-duplicate events ledger +// contributes nothing (nil apply hook) while the upsert-keyed CFs still write. +func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView, cfg Ingest) (LedgerCounts, error) { + var counts LedgerCounts + if d.store.IsClosed() { + return counts, stores.ErrStoreClosed + } + + // Pre-extract anything that can fail BEFORE opening the batch, so a decode + // error rejects the ledger without a half-built batch. + var txEntries []txhash.Entry + if cfg.Txhash { + hashes, err := sdkingest.ExtractTxHashes(lcm) + if err != nil { + return counts, fmt.Errorf("hotchunk: extract tx hashes seq %d: %w", seq, err) + } + if len(hashes) > 0 { + txEntries = make([]txhash.Entry, len(hashes)) + for i, h := range hashes { + txEntries[i] = txhash.Entry{Hash: [32]byte(h), LedgerSeq: seq} + } + } + counts.Txhash = len(hashes) + } + + var payloads []events.Payload + if cfg.Events { + p, err := eventPayloads(seq, lcm) + if err != nil { + return counts, err + } + payloads = p + counts.Events = len(payloads) + } + if cfg.Ledgers { + counts.Ledgers = 1 + } + + // The events facade validates + marshals up front (so a rejected ledger + // never touches the batch) and returns the post-commit apply hook (nil for + // an idempotent duplicate). + var applyEvents func() + cerr := d.store.Batch(func(b *rocksdb.BatchWriter) error { + if cfg.Ledgers { + if err := d.ledger.AddLedgerToBatch(b, ledger.Entry{Seq: seq, Bytes: []byte(lcm)}); err != nil { + return fmt.Errorf("hotchunk: queue ledger seq %d: %w", seq, err) + } + } + if cfg.Txhash && len(txEntries) > 0 { + if err := d.txhash.AddEntriesToBatch(b, txEntries); err != nil { + return fmt.Errorf("hotchunk: queue tx hashes seq %d: %w", seq, err) + } + } + if cfg.Events { + apply, err := d.events.IngestLedgerToBatch(b, seq, payloads) + if err != nil { + return fmt.Errorf("hotchunk: queue events seq %d: %w", seq, err) + } + applyEvents = apply + } + return nil + }) + if cerr != nil { + return counts, fmt.Errorf("hotchunk: commit ledger %d to chunk %s: %w", seq, d.chunkID, cerr) + } + + // Batch is durable — now and only now apply the events mirror/offsets update. + if applyEvents != nil { + applyEvents() + } + return counts, nil +} + +// eventPayloads derives one ledger's event payloads from the view (a pre-Soroban +// ledger yields zero, no error). Duplicated from ingest.eventPayloads rather than +// imported — ingest will depend on hotchunk, so importing it would cycle. +func eventPayloads(seq uint32, lcm xdr.LedgerCloseMetaView) ([]events.Payload, error) { + payloads, err := events.LCMViewToPayloads(lcm) + if err != nil { + return nil, fmt.Errorf("hotchunk: LCMViewToPayloads seq %d: %w", seq, err) + } + return payloads, nil +} diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go new file mode 100644 index 000000000..842b65325 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go @@ -0,0 +1,468 @@ +package hotchunk + +import ( + "context" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stellar/go-stellar-sdk/keypair" + "github.com/stellar/go-stellar-sdk/network" + supportlog "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/go-stellar-sdk/xdr" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/events" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash" +) + +const testPassphrase = "Public Global Stellar Network ; September 2015" + +func silentLogger() *supportlog.Entry { + log := supportlog.New() + log.SetLevel(logrus.ErrorLevel) + return log +} + +func openTestDB(t *testing.T, chunkID chunk.ID) *DB { + t.Helper() + db, err := Open(t.TempDir(), chunkID, silentLogger()) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + return db +} + +func allTypes() Ingest { return Ingest{Ledgers: true, Txhash: true, Events: true} } + +func TestOpen_ValidatesInputs(t *testing.T) { + _, err := Open("", chunk.ID(0), silentLogger()) + require.ErrorIs(t, err, stores.ErrInvalidConfig) + + _, err = Open(t.TempDir(), chunk.ID(0), nil) + require.ErrorIs(t, err, stores.ErrInvalidConfig) +} + +func TestColumnFamilies_UnionIsNonColliding(t *testing.T) { + cfs := columnFamilies() + // 1 ledger CF + 3 events CFs + 16 txhash CFs = 20. + require.Len(t, cfs, 1+len(eventstore.CFNames())+len(txhash.CFNames())) + seen := map[string]bool{} + for _, cf := range cfs { + require.False(t, seen[cf], "CF name %q collides across facades", cf) + seen[cf] = true + } + require.Contains(t, seen, ledger.LedgersCF) + for _, cf := range eventstore.CFNames() { + require.Contains(t, seen, cf) + } + for _, cf := range txhash.CFNames() { + require.Contains(t, seen, cf) + } +} + +// TestIngestLedger_AllCFsAdvanceTogether is the core decision-(a) happy path: +// one IngestLedger call writes the ledger, its tx hash, and its event into the +// ONE shared DB, and the single watermark reaches exactly the committed seq — +// every CF readable, every CF in lockstep. +func TestIngestLedger_AllCFsAdvanceTogether(t *testing.T) { + chunkID := chunk.ID(0) + first := chunkID.FirstLedger() + db := openTestDB(t, chunkID) + + // Empty DB: no watermark. + _, ok, err := db.MaxCommittedSeq() + require.NoError(t, err) + require.False(t, ok) + + rawA, hashA, termA := lcmWithEvent(t, first) + rawB, hashB, _ := lcmWithEvent(t, first+1) + + counts, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(rawA), allTypes()) + require.NoError(t, err) + assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 1, Events: 1}, counts) + + counts, err = db.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawB), allTypes()) + require.NoError(t, err) + assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 1, Events: 1}, counts) + + // ledgers CF. + gotA, err := db.Ledgers().GetLedgerRaw(first) + require.NoError(t, err) + assert.Equal(t, rawA, gotA) + // txhash CFs. + seqA, err := db.Txhash().Get(hashA) + require.NoError(t, err) + assert.Equal(t, first, seqA) + seqB, err := db.Txhash().Get(hashB) + require.NoError(t, err) + assert.Equal(t, first+1, seqB) + // events CFs. + bm, err := db.Events().Lookup(context.Background(), termA) + require.NoError(t, err) + require.NotNil(t, bm) + assert.Equal(t, uint64(2), bm.GetCardinality(), "both ledgers share the event term") + assert.Equal(t, uint32(2), db.Events().NextEventID()) + + // The single authoritative watermark equals the last committed seq. + maxSeq, ok, err := db.MaxCommittedSeq() + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, first+1, maxSeq) +} + +// TestIngestLedger_RejectedLedgerPersistsNothingAcrossAnyCF is the atomicity +// guarantee for decision (a): a ledger the events facade rejects (here an +// out-of-range seq) must leave EVERY CF untouched — the ledgers and txhash CFs +// included — because the whole ledger is one batch and the events facade's +// validation aborts that batch before commit. The single watermark must not +// advance. +func TestIngestLedger_RejectedLedgerPersistsNothingAcrossAnyCF(t *testing.T) { + chunkID := chunk.ID(0) + db := openTestDB(t, chunkID) + + // A ledger seq ABOVE the chunk's range: the events facade rejects it + // (ErrLedgerOutOfRange) from inside the batch callback, aborting the write. + badSeq := chunkID.LastLedger() + 1 + raw, hash, term := lcmWithEvent(t, badSeq) + + _, err := db.IngestLedger(badSeq, xdr.LedgerCloseMetaView(raw), allTypes()) + require.Error(t, err) + require.ErrorIs(t, err, eventstore.ErrLedgerOutOfRange) + + // NOTHING persisted, across every CF: + // ledgers CF — no row at badSeq. + _, gerr := db.Ledgers().GetLedgerRaw(badSeq) + require.ErrorIs(t, gerr, stores.ErrNotFound) + // txhash CFs — the hash is absent. + _, gerr = db.Txhash().Get(hash) + require.ErrorIs(t, gerr, stores.ErrNotFound) + // events CFs — no term indexed, no event committed. + _, lerr := db.Events().Lookup(context.Background(), term) + require.ErrorIs(t, lerr, eventstore.ErrTermNotFound) + assert.Equal(t, uint32(0), db.Events().NextEventID()) + + // The single watermark is still empty — nothing committed. + _, ok, err := db.MaxCommittedSeq() + require.NoError(t, err) + require.False(t, ok, "a rejected ledger must not advance the watermark") +} + +// TestIngestLedger_MidBatchCommitFailurePersistsNothing simulates a mid-batch +// COMMIT failure (the store closed under the writer) and asserts the partial +// batch persisted nothing across any CF after reopen — the single synced +// WriteBatch is all-or-nothing. +func TestIngestLedger_MidBatchCommitFailurePersistsNothing(t *testing.T) { + chunkID := chunk.ID(0) + first := chunkID.FirstLedger() + dir := t.TempDir() + + db, err := Open(dir, chunkID, silentLogger()) + require.NoError(t, err) + + // Commit one good ledger so there is a known watermark, then close the DB. + rawGood, hashGood, _ := lcmWithEvent(t, first) + _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(rawGood), allTypes()) + require.NoError(t, err) + require.NoError(t, db.Close()) + + // Reopen and confirm the watermark survived (sync=true durability). + db2, err := Open(dir, chunkID, silentLogger()) + require.NoError(t, err) + t.Cleanup(func() { _ = db2.Close() }) + + maxSeq, ok, err := db2.MaxCommittedSeq() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, first, maxSeq, "the committed ledger is durable across reopen") + + // Now close the DB and attempt to ingest the NEXT ledger into the closed + // store: the commit fails, and nothing for that ledger persists anywhere. + require.NoError(t, db2.Close()) + rawNext, hashNext, _ := lcmWithEvent(t, first+1) + _, err = db2.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawNext), allTypes()) + require.Error(t, err) + + // Reopen a third time: the failed ledger left NO trace in any CF, and the + // watermark is still the last good seq. + db3, err := Open(dir, chunkID, silentLogger()) + require.NoError(t, err) + t.Cleanup(func() { _ = db3.Close() }) + + maxSeq, ok, err = db3.MaxCommittedSeq() + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, first, maxSeq, "the failed ledger did not advance the watermark") + + // The events CF advanced for exactly the one good ledger — the failed + // ledger's event was not committed (warmup reconstructed the offsets from + // disk, which hold only the good ledger). + assert.Equal(t, uint32(1), db3.Events().NextEventID(), + "the failed ledger's event must not be committed to the events CFs") + + // The good ledger's data is intact; the failed ledger's is wholly absent + // across the ledgers and txhash CFs. + _, gerr := db3.Ledgers().GetLedgerRaw(first + 1) + require.ErrorIs(t, gerr, stores.ErrNotFound) + _, gerr = db3.Txhash().Get(hashNext) + require.ErrorIs(t, gerr, stores.ErrNotFound) + + gotGood, err := db3.Ledgers().GetLedgerRaw(first) + require.NoError(t, err) + assert.Equal(t, rawGood, gotGood) + _, err = db3.Txhash().Get(hashGood) + require.NoError(t, err) +} + +// TestSharedBatch_DirectRocksAbortAcrossCFs is the lower-level atomicity proof: +// queue Puts into DIFFERENT CFs of the shared store, then return an error from +// the batch callback — RocksDB applies NONE of them. Pins the property the +// IngestLedger path relies on (intra-store cross-CF atomicity of one +// WriteBatch). +func TestSharedBatch_DirectRocksAbortAcrossCFs(t *testing.T) { + db := openTestDB(t, chunk.ID(0)) + + var hash [32]byte + hash[0] = 0xa0 + sentinelErr := assert.AnError + + err := storeOf(db).Batch(func(b *rocksdb.BatchWriter) error { + b.Put(ledger.LedgersCF, rocksdb.EncodeUint32(2), []byte("ledger-row")) + b.Put(txhash.CFNames()[0xa], hash[:], rocksdb.EncodeUint32(2)) + b.Put(eventstore.DataCF, []byte{0, 0, 0, 0}, []byte("event-row")) + return sentinelErr // abort: nothing should commit + }) + require.ErrorIs(t, err, sentinelErr) + + // None of the three CFs received the aborted writes. + _, gerr := db.Ledgers().GetLedgerRaw(2) + require.ErrorIs(t, gerr, stores.ErrNotFound) + _, gerr = db.Txhash().Get(hash) + require.ErrorIs(t, gerr, stores.ErrNotFound) + _, ok, derr := db.MaxCommittedSeq() + require.NoError(t, derr) + require.False(t, ok) +} + +// storeOf exposes the shared store for the direct-batch atomicity test (same +// package, so no production accessor is needed). +func storeOf(db *DB) *rocksdb.Store { return db.store } + +// TestIngestLedger_DisabledTypesUntouched confirms the Ingest toggles select +// which CFs the single batch writes: ledgers-only leaves txhash/events empty. +func TestIngestLedger_DisabledTypesUntouched(t *testing.T) { + chunkID := chunk.ID(0) + first := chunkID.FirstLedger() + db := openTestDB(t, chunkID) + + raw, hash, term := lcmWithEvent(t, first) + counts, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw), Ingest{Ledgers: true}) + require.NoError(t, err) + assert.Equal(t, LedgerCounts{Ledgers: 1}, counts) + + got, err := db.Ledgers().GetLedgerRaw(first) + require.NoError(t, err) + assert.Equal(t, raw, got) + + _, gerr := db.Txhash().Get(hash) + require.ErrorIs(t, gerr, stores.ErrNotFound) + _, lerr := db.Events().Lookup(context.Background(), term) + require.ErrorIs(t, lerr, eventstore.ErrTermNotFound) +} + +// TestReopen_RecoversEventsMirror confirms the events facade's warmup runs over +// the shared store on reopen (the mirror/offsets are reconstructed from the +// events CFs), so a reopened DB assigns event IDs continuing from disk. +func TestReopen_RecoversEventsMirror(t *testing.T) { + chunkID := chunk.ID(0) + first := chunkID.FirstLedger() + dir := t.TempDir() + + db, err := Open(dir, chunkID, silentLogger()) + require.NoError(t, err) + raw, _, _ := lcmWithEvent(t, first) + _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(raw), allTypes()) + require.NoError(t, err) + require.NoError(t, db.Close()) + + db2, err := Open(dir, chunkID, silentLogger()) + require.NoError(t, err) + t.Cleanup(func() { _ = db2.Close() }) + assert.Equal(t, uint32(1), db2.Events().NextEventID(), "warmup recovered the events offsets") +} + +// TestOpenReadOnly_ReadsCommittedAndRejectsWrites pins the freeze source's +// read-only handle: it sees data a writer committed and cleanly closed (so the +// completeness gate is exact), and any write through it fails — a freeze can +// never mutate the hot DB it reads. +func TestOpenReadOnly_ReadsCommittedAndRejectsWrites(t *testing.T) { + chunkID := chunk.ID(0) + first := chunkID.FirstLedger() + dir := t.TempDir() + + // Writer: ingest two ledgers, then close (flushes the WAL into SST). + db, err := Open(dir, chunkID, silentLogger()) + require.NoError(t, err) + for _, seq := range []uint32{first, first + 1} { + _, ierr := db.IngestLedger(seq, xdr.LedgerCloseMetaView(zeroTxLCM(t, seq)), allTypes()) + require.NoError(t, ierr) + } + require.NoError(t, db.Close()) + + // Reader: a read-only open sees the committed watermark; Close must not flush. + ro, err := OpenReadOnly(dir, chunkID, silentLogger()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, ro.Close()) }) + + seq, ok, err := ro.MaxCommittedSeq() + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, first+1, seq, "read-only handle sees the committed data") + + // A write through the read-only handle must fail — the freeze never mutates. + _, err = ro.IngestLedger(first+2, xdr.LedgerCloseMetaView(zeroTxLCM(t, first+2)), allTypes()) + require.Error(t, err, "read-only DB must reject writes") +} + +// TestIngestLedger_ClosedDBFails confirms a closed shared DB rejects ingest. +func TestIngestLedger_ClosedDBFails(t *testing.T) { + chunkID := chunk.ID(0) + db, err := Open(t.TempDir(), chunkID, silentLogger()) + require.NoError(t, err) + require.NoError(t, db.Close()) + + raw := zeroTxLCM(t, chunkID.FirstLedger()) + _, err = db.IngestLedger(chunkID.FirstLedger(), xdr.LedgerCloseMetaView(raw), allTypes()) + require.ErrorIs(t, err, stores.ErrStoreClosed) +} + +// ──────────────────────────── LCM fixtures ──────────────────────────── + +// lcmWithEvent builds a V2 LCM with one transaction carrying one contract event +// (topic="hotchunk_test"). Returns the wire bytes, the tx hash, and the event's +// term key. +func lcmWithEvent(t *testing.T, seq uint32) ([]byte, [32]byte, events.TermKey) { + t.Helper() + ev := buildContractEvent("hotchunk_test") + meta := xdr.TransactionMeta{ + V: 4, + V4: &xdr.TransactionMetaV4{Operations: []xdr.OperationMetaV2{{Events: []xdr.ContractEvent{ev}}}}, + } + lcm, hash := buildLCMWithTx(t, seq, meta) + raw, err := lcm.MarshalBinary() + require.NoError(t, err) + + evBytes, err := ev.MarshalBinary() + require.NoError(t, err) + keys, err := events.TermsForBytes(evBytes) + require.NoError(t, err) + require.NotEmpty(t, keys) + return raw, hash, keys[0] +} + +func zeroTxLCM(t *testing.T, seq uint32) []byte { + t.Helper() + lcm, _ := buildLCM(t, seq, nil) + raw, err := lcm.MarshalBinary() + require.NoError(t, err) + return raw +} + +func buildContractEvent(topic string) xdr.ContractEvent { + var contractID xdr.ContractId + contractID[0] = 0xab + contractID[1] = 0xcd + sym := xdr.ScSymbol(topic) + return xdr.ContractEvent{ + ContractId: &contractID, + Type: xdr.ContractEventTypeContract, + Body: xdr.ContractEventBody{ + V: 0, + V0: &xdr.ContractEventV0{ + Topics: []xdr.ScVal{{Type: xdr.ScValTypeScvSymbol, Sym: &sym}}, + Data: xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &sym}, + }, + }, + } +} + +func successResult() xdr.TransactionResult { + opResults := []xdr.OperationResult{} + return xdr.TransactionResult{ + FeeCharged: 100, + Result: xdr.TransactionResultResult{ + Code: xdr.TransactionResultCodeTxSuccess, + Results: &opResults, + }, + } +} + +func buildLCMWithTx(t *testing.T, seq uint32, meta xdr.TransactionMeta) (xdr.LedgerCloseMeta, [32]byte) { + t.Helper() + lcm, hashes := buildLCM(t, seq, []xdr.TransactionMeta{meta}) + require.Len(t, hashes, 1) + return lcm, hashes[0] +} + +func buildLCM(t *testing.T, seq uint32, txMetas []xdr.TransactionMeta) (xdr.LedgerCloseMeta, [][32]byte) { + t.Helper() + phases := make([]xdr.TransactionPhase, 0, len(txMetas)) + txProcessing := make([]xdr.TransactionResultMetaV1, 0, len(txMetas)) + hashes := make([][32]byte, 0, len(txMetas)) + + for _, meta := range txMetas { + envelope := xdr.TransactionEnvelope{ + Type: xdr.EnvelopeTypeEnvelopeTypeTx, + V1: &xdr.TransactionV1Envelope{ + Tx: xdr.Transaction{ + SourceAccount: xdr.MustMuxedAddress(keypair.MustRandom().Address()), + Ext: xdr.TransactionExt{ + V: 1, + SorobanData: &xdr.SorobanTransactionData{}, + }, + }, + }, + } + hash, err := network.HashTransactionInEnvelope(envelope, testPassphrase) + require.NoError(t, err) + hashes = append(hashes, hash) + + txProcessing = append(txProcessing, xdr.TransactionResultMetaV1{ + TxApplyProcessing: meta, + Result: xdr.TransactionResultPair{ + TransactionHash: hash, + Result: successResult(), + }, + }) + comp := []xdr.TxSetComponent{{ + Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, + TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ + Txs: []xdr.TransactionEnvelope{envelope}, + }, + }} + phases = append(phases, xdr.TransactionPhase{V: 0, V0Components: &comp}) + } + + lcm := xdr.LedgerCloseMeta{ + V: 2, + V2: &xdr.LedgerCloseMetaV2{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + ScpValue: xdr.StellarValue{CloseTime: xdr.TimePoint(0)}, + LedgerSeq: xdr.Uint32(seq), + }, + }, + TxSet: xdr.GeneralizedTransactionSet{ + V: 1, + V1TxSet: &xdr.TransactionSetV1{Phases: phases}, + }, + TxProcessing: txProcessing, + }, + } + return lcm, hashes +} diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go index 2ba7afd4f..790f37322 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go @@ -17,59 +17,47 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/zstd" ) -// Entry — one (sequence, uncompressed ledger bytes) pair. Both -// hot and cold stores compress on write and decompress on read, -// so callers always pass and receive raw ledger bytes here. +// LedgersCF is the column family the hot ledger data lives in. Registered the +// same whether the DB is the shared per-chunk multi-CF DB (decision (a)) or a +// standalone single-purpose DB (OpenHotStore), so the on-disk layout is +// identical either way. +const LedgersCF = "ledgers" + +// Entry — one (sequence, uncompressed ledger bytes) pair. Compression is +// internal to the store, so callers pass and receive raw bytes here. type Entry struct { Seq uint32 Bytes []byte } -// HotStore — RocksDB-backed hot ledger store. Default-CF only; -// keys are 4-byte big-endian sequences; values are zstd-compressed -// ledger bytes. Compression is internal: callers see raw bytes on -// the boundary. -// -// Like every hot store, a HotStore instance is chunk-bound: it -// accumulates exactly one chunk's ledgers before being frozen into -// the chunk's cold artifacts. The binding is recorded at open time -// (ChunkID) so the ingest driver can reject a store bound to a -// different chunk than it is ingesting; the store does not itself -// range-check writes (the driver's drain loop already validates -// every sequence against the chunk). +// HotStore — RocksDB-backed hot ledger store. Keys are 4-byte BE sequences; +// values are zstd-compressed (internal). Chunk-bound: accumulates one chunk's +// ledgers before freezing, with the binding recorded at open time (ChunkID) so +// the ingest driver can reject a mismatched store. The store does not itself +// range-check writes (the driver's drain loop already validates every sequence). // -// Concurrency: all methods, including Close, are safe for concurrent -// use. rocksdb.Store.Close CAS-marks the store closed and then drains -// in-flight ops (each holds an RLock for its duration) before releasing -// resources; a read/write racing Close either completes first or -// observes the closed store and returns stores.ErrStoreClosed. Close is -// idempotent. HotStore adds no unguarded state of its own — the -// compressor pool and decompressor are both concurrent-safe. +// Concurrency: all methods, including Close, are safe for concurrent use. +// rocksdb.Store.Close drains in-flight ops before releasing; a racing read/write +// either completes first or returns stores.ErrStoreClosed. HotStore adds no +// unguarded state — the compressor pool and decompressor are concurrent-safe. type HotStore struct { store *rocksdb.Store chunkID chunk.ID - dec *zstd.Decompressor - // compPool — per-store pool of zstd.Compressors. Each - // concurrent AddLedgers borrows one for the duration of its - // Encode call; the pool's GC finalizer (set inside - // zstd.NewCompressor) frees the C context when the compressor - // is dropped between GC cycles. + // ownsStore is true on the standalone OpenHotStore path (Close closes the + // store); false when wrapping the SHARED per-chunk DB via NewWithStore, + // which hotchunk.DB owns and closes once. + ownsStore bool + dec *zstd.Decompressor + // compPool — per-store pool of zstd.Compressors; each concurrent AddLedgers + // borrows one for its Encode call. compPool sync.Pool } -// OpenHotStore validates inputs and returns an open HotStore bound -// to chunkID (see the HotStore doc on chunk binding). path and -// logger are both required; logger is forwarded to the -// pkg/rocksdb wrapper (rocksdb writes the on-open state line and -// the close-time Flush warning through it). HotStore itself does -// not emit any logs — the cold store, by contrast, takes no -// logger because packfile is silent. Rides on RocksDB defaults — -// no explicit block cache (RocksDB's per-CF default plus OS page -// cache cover range scans), no bloom filter (callers know in -// advance which sequences this store holds, so it is never asked -// for a key it doesn't have), no WAL cap (graceful Close flushes -// the memtable; ungraceful WAL replay at this scale is sub-second). -// Re-tune only with a workload measurement. +// OpenHotStore validates inputs and returns an open HotStore bound to chunkID. +// path and logger are required (logger is forwarded to pkg/rocksdb). Rides on +// RocksDB defaults — no block cache, no bloom filter (callers only ask for +// sequences this store holds), no WAL cap (graceful Close flushes; ungraceful +// replay at this scale is sub-second). Re-tune only with a measurement. func OpenHotStore(path string, chunkID chunk.ID, logger *supportlog.Entry) (*HotStore, error) { if path == "" { return nil, stores.ErrInvalidConfig @@ -78,12 +66,23 @@ func OpenHotStore(path string, chunkID chunk.ID, logger *supportlog.Entry) (*Hot return nil, stores.ErrInvalidConfig } store, err := rocksdb.New(rocksdb.Config{ - Path: path, - Logger: logger, + Path: path, + ColumnFamilies: []string{LedgersCF}, + Logger: logger, }) if err != nil { return nil, err } + h := NewWithStore(store, chunkID) + h.ownsStore = true + return h, nil +} + +// NewWithStore wraps an ALREADY-OPEN rocksdb.Store as a ledger HotStore on +// LedgersCF. The store is NOT owned (Close is a no-op) — the constructor hotchunk +// uses to compose this facade over the shared multi-CF DB (decision (a)). The +// store must have LedgersCF registered. +func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) *HotStore { return &HotStore{ store: store, chunkID: chunkID, @@ -91,26 +90,25 @@ func OpenHotStore(path string, chunkID chunk.ID, logger *supportlog.Entry) (*Hot compPool: sync.Pool{ New: func() any { return zstd.NewCompressor() }, }, - }, nil + } } -// Close releases the underlying RocksDB store. Idempotent — -// delegates to rocksdb.Store.Close. Must not be called concurrently -// with in-flight reads/writes on this HotStore. -func (h *HotStore) Close() error { return h.store.Close() } +// Close releases the store IF this HotStore owns it (standalone OpenHotStore); +// a no-op when wrapping the shared per-chunk DB (NewWithStore), which hotchunk.DB +// closes once. Idempotent; not safe to call alongside in-flight reads/writes. +func (h *HotStore) Close() error { + if !h.ownsStore { + return nil + } + return h.store.Close() +} // ChunkID returns the chunk this store is bound to (constructor-supplied; // never reads the store). func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } -// AddLedgers writes (seq, raw-bytes) entries to rocksdb. Bytes is -// the uncompressed ledger payload; AddLedgers compresses each -// entry with zstd before write. Variadic so callers can pass -// individual entries (h.AddLedgers(e)), a literal batch -// (h.AddLedgers(e1, e2, e3)), or a slice (h.AddLedgers(entries...)). -// Zero entries is a no-op; one entry uses Store.Put; multiple -// entries use Store.Batch (one atomic write, one fsync — versus N -// fsyncs for N Put calls). +// AddLedgers compresses and writes (seq, raw-bytes) entries. Zero entries is a +// no-op; one uses Store.Put; multiple use one Store.Batch (one fsync, not N). func (h *HotStore) AddLedgers(entries ...Entry) error { if h.store.IsClosed() { return stores.ErrStoreClosed @@ -127,12 +125,10 @@ func (h *HotStore) AddLedgers(entries ...Entry) error { if err != nil { return err } - return translateRocksErr(h.store.Put("", rocksdb.EncodeUint32(e.Seq), compressed)) + return translateRocksErr(h.store.Put(LedgersCF, rocksdb.EncodeUint32(e.Seq), compressed)) } - // Multi-entry path: compress each into its own fresh slice so - // the batch can hold them all simultaneously (the compressor's - // internal buffer would otherwise be overwritten on the next - // Encode call). + // Compress each into its own fresh slice so the batch can hold them all at + // once (the compressor's internal buffer is overwritten on the next Encode). compressed := make([][]byte, len(entries)) for i, e := range entries { out, err := c.Encode(nil, e.Bytes) @@ -143,19 +139,38 @@ func (h *HotStore) AddLedgers(entries ...Entry) error { } return translateRocksErr(h.store.Batch(func(b *rocksdb.BatchWriter) error { for i, e := range entries { - b.Put("", rocksdb.EncodeUint32(e.Seq), compressed[i]) + b.Put(LedgersCF, rocksdb.EncodeUint32(e.Seq), compressed[i]) } return nil })) } +// AddLedgerToBatch compresses one ledger and queues its Put into b on LedgersCF +// — the building block hotchunk uses to fold the ledger write into the one +// shared per-ledger WriteBatch (decision (a)). Does not commit (caller owns the +// batch). Compresses into a fresh buffer BatchWriter.Put copies, so e.Bytes need +// not outlive this call. +func (h *HotStore) AddLedgerToBatch(b *rocksdb.BatchWriter, e Entry) error { + if h.store.IsClosed() { + return stores.ErrStoreClosed + } + c, _ := h.compPool.Get().(*zstd.Compressor) + defer h.compPool.Put(c) + compressed, err := c.Encode(nil, e.Bytes) + if err != nil { + return err + } + b.Put(LedgersCF, rocksdb.EncodeUint32(e.Seq), compressed) + return nil +} + // GetLedgerRaw decodes the ledger stored under seq into a fresh, // caller-owned buffer, or returns stores.ErrNotFound on miss. A zstd // decode failure surfaces as stores.ErrCorrupt. Sequential bulk readers // should prefer IterateLedgers, which yields borrows without the // per-ledger decode allocation. func (h *HotStore) GetLedgerRaw(seq uint32) ([]byte, error) { - v, found, err := h.store.Get("", rocksdb.EncodeUint32(seq)) + v, found, err := h.store.Get(LedgersCF, rocksdb.EncodeUint32(seq)) if err != nil { return nil, translateRocksErr(err) } @@ -184,7 +199,7 @@ func (h *HotStore) edgeSeq(last bool) (uint32, bool, error) { if last { edge = h.store.LastKey } - k, ok, err := edge("") + k, ok, err := edge(LedgersCF) if err != nil { return 0, false, translateRocksErr(err) } @@ -207,19 +222,16 @@ func (h *HotStore) IterateLedgers(start, end uint32) iter.Seq2[Entry, error] { if start > end { return } - // scratch is the reused decompression buffer; Entry.Bytes aliases it - // and is therefore BORROWED — valid only until the next iteration step - // decodes the following ledger into it. Copy it if you need to retain - // it past the loop body. The read benches consume each ledger in-scope, - // so this avoids a per-ledger decode allocation. + // scratch is the reused decompression buffer; Entry.Bytes aliases it and + // is BORROWED — valid only until the next step decodes into it. Copy to + // retain past the loop body. Avoids a per-ledger decode allocation. var scratch []byte - for e, err := range h.store.IterateRange("", rocksdb.EncodeUint32(start), rocksdb.EncodeUint32(end)) { + for e, err := range h.store.IterateRange(LedgersCF, rocksdb.EncodeUint32(start), rocksdb.EncodeUint32(end)) { if err != nil { yield(Entry{}, translateRocksErr(err)) return } - // e.Value is itself a zero-copy ref into the iterator's internal - // buffer; decompress it into the reused scratch buffer. + // e.Value is a zero-copy ref into the iterator buffer; decode into scratch. seq := rocksdb.DecodeUint32(e.Key) decoded, derr := h.dec.Decode(scratch[:0], e.Value) if derr != nil { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go index 18bfa4420..23faa2bd8 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go @@ -31,20 +31,19 @@ type Entry struct { LedgerSeq uint32 } -// HotStore — RocksDB-backed hot transaction-hash store. 16 CFs named -// cf-0..cf-f; each hash routes to cf-{txhash[0]>>4}; ledgerSeq -// encoded big-endian. Routing, CF names, and encoding are internal. -// -// Like every hot store, a HotStore instance is chunk-bound: it -// accumulates exactly one chunk's (txhash → seq) tuples before being -// frozen into the chunk's cold .bin artifact. The binding is recorded -// at open time (ChunkID) so the ingest driver can reject a store -// bound to a different chunk than it is ingesting; the store does not -// itself range-check writes (the driver's drain loop already -// validates every ledger sequence against the chunk). +// HotStore — RocksDB-backed hot transaction-hash store. 16 CFs cf-0..cf-f; each +// hash routes to cf-{txhash[0]>>4}; ledgerSeq BE-encoded (all internal). +// Chunk-bound like every hot store: accumulates one chunk's (txhash → seq) +// tuples before freezing, with the binding recorded at open time (ChunkID) so +// the ingest driver can reject a mismatched store. The store does not itself +// range-check writes (the driver's drain loop already validates every sequence). type HotStore struct { store *rocksdb.Store chunkID chunk.ID + // ownsStore is true on the standalone NewHotStore path; false when wrapping + // the SHARED per-chunk DB via NewWithStore (decision (a)), which + // hotchunk.DB owns and closes once. + ownsStore bool } // NewHotStore validates inputs and returns an open HotStore bound to @@ -65,9 +64,25 @@ func NewHotStore(path string, chunkID chunk.ID, logger *supportlog.Entry) (*HotS if err != nil { return nil, err } - return &HotStore{store: store, chunkID: chunkID}, nil + return &HotStore{store: store, chunkID: chunkID, ownsStore: true}, nil +} + +// NewWithStore wraps an ALREADY-OPEN rocksdb.Store as a txhash HotStore on the +// 16 nibble-routed CFs (CFNames()). The store is NOT owned (Close is a no-op) — +// the constructor hotchunk uses to compose this facade over the shared per-chunk +// DB. The store must have CFNames() registered. +func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) *HotStore { + return &HotStore{store: store, chunkID: chunkID} } +// CFNames returns the 16 nibble-routed CF names this facade owns. Exported so +// the hotchunk shared-DB opener can register them alongside the other CFs. +func CFNames() []string { return cfNames() } + +// Tuning returns this facade's RocksDB tuning, applied to the shared per-chunk +// DB by the hotchunk opener. +func Tuning() rocksdb.Tuning { return tuning() } + func cfNames() []string { out := make([]string, numCFs) copy(out, cfNameByNibble[:]) @@ -78,68 +93,57 @@ func cfNameForTxHash(hash [32]byte) string { return cfNameByNibble[hash[0]>>4] } -// tuning — the hot txhash workload is write-once / point-lookup over -// 16 CFs; the cross-knob interactions below are non-obvious enough -// that they get an explicit per-stanza rationale. The other facades -// ride on RocksDB defaults by contrast — only this workload earned -// the calibration. +// tuning — calibrated for the hot txhash workload (write-once / point-lookup over +// 16 CFs). Cross-knob interactions are non-obvious, hence the per-stanza WHY; +// the other facades ride on defaults. func tuning() rocksdb.Tuning { return rocksdb.Tuning{ - // Per-CF memtable budget × 16 CFs (64 MB × 16 = 1024 MB) - // matches the MaxTotalWalSizeMB cap below. Memtable-fill - // cadence and WAL-cap cadence align under uniform writes; - // either trigger fires at roughly the same time and produces - // ~64 MB SSTs. + // Per-CF memtable budget × 16 (64 MB × 16 = 1024 MB) matches + // MaxTotalWalSizeMB below, so memtable-fill and WAL-cap cadence align + // and each flush produces a ~64 MB SST. WriteBufferMB: 64, MaxWriteBufferNumber: 2, - // L0 triggers pinned high + DisableAutoCompactions=true: - // compaction would re-write the same data with no reordering - // benefit (txhash is write-once, random-key, point-lookup). - // The L0 999s match DisableAutoCompactions so even if a future - // flush somehow exceeded the trigger, the engine still - // wouldn't try to compact. NOTE: DisableAutoCompactions and - // MaxBackgroundJobs are orthogonal — the former turns - // compaction off entirely, the latter only caps the thread - // budget for background work. + // Compaction off: write-once random-key data gains no reordering + // benefit. L0 999s match DisableAutoCompactions so even an over-trigger + // flush won't compact. (DisableAutoCompactions and MaxBackgroundJobs are + // orthogonal — off vs. thread budget.) Level0FileNumCompactionTrigger: 999, Level0SlowdownWritesTrigger: 999, Level0StopWritesTrigger: 999, DisableAutoCompactions: true, - // 64 MB target file matches WriteBufferMB so one memtable - // flush produces one ~64 MB SST — fewer bloom checks per - // query at no-compaction scale. - // MaxBytesForLevelBaseMB is set explicitly even though it's - // irrelevant under DisableAutoCompactions (compaction never - // promotes past L0); explicit > implicit so a future reader - // doesn't have to derive that it's a no-op. + // 64 MB target file matches WriteBufferMB (one flush → one SST). MaxBytes + // is a no-op under DisableAutoCompactions but set explicitly so a reader + // needn't derive that. TargetFileSizeMB: 64, MaxBytesForLevelBaseMB: 256, - // High background-job budget for the periodic memtable - // flushes across 16 CFs. + // High background-job budget for periodic flushes across 16 CFs. MaxBackgroundJobs: 8, MaxOpenFiles: 10_000, - // 512 MB block cache — bloom-filter blocks are the hot - // working set; the cache needs to hold recently-touched - // bloom blocks at scale. - // 12 bits/key bloom (~0.4% false-positive) is tighter than - // the standard 10 bits/key because every false positive at - // no-compaction SST count costs a disk seek across many SSTs. + // 512 MB block cache holds the hot working set (bloom-filter blocks). + // 12 bits/key (~0.4% FP) is tighter than the standard 10 because each FP + // costs a disk seek across many no-compaction SSTs. BlockCacheMB: 512, BloomFilterBitsPerKey: 12, - // 1 GB WAL cap matches the natural memtable budget above. - // Graceful Close auto-Flushes (see rocksdb.Store.Close), so - // this cap only bounds ungraceful-shutdown recovery (kernel - // panic, power loss, OOM kill). + // 1 GB WAL cap matches the memtable budget. Graceful Close auto-Flushes, + // so this only bounds ungraceful-recovery (panic / power loss / OOM). MaxTotalWalSizeMB: 1024, } } -func (h *HotStore) Close() error { return h.store.Close() } +// Close releases the store IF this HotStore owns it (standalone NewHotStore); +// a no-op when wrapping the shared per-chunk DB (NewWithStore), which hotchunk.DB +// closes once. Idempotent. +func (h *HotStore) Close() error { + if !h.ownsStore { + return nil + } + return h.store.Close() +} // ChunkID returns the chunk this store is bound to (constructor-supplied; // never reads the store). @@ -168,6 +172,20 @@ func (h *HotStore) AddEntries(entries []Entry) error { } } +// AddEntriesToBatch queues each (txhash → ledgerSeq) Put into b on its +// nibble-routed CF — the building block hotchunk uses to fold the tx-hash writes +// into the one shared per-ledger WriteBatch (decision (a)). Does not commit +// (caller owns the batch). A closed store returns ErrStoreClosed. +func (h *HotStore) AddEntriesToBatch(b *rocksdb.BatchWriter, entries []Entry) error { + if h.store.IsClosed() { + return rocksdb.ErrStoreClosed + } + for _, e := range entries { + b.Put(cfNameForTxHash(e.Hash), e.Hash[:], rocksdb.EncodeUint32(e.LedgerSeq)) + } + return nil +} + // Get returns the ledger sequence the hash was committed in, or // (0, stores.ErrNotFound) on miss. Only the routed CF is queried. func (h *HotStore) Get(hash [32]byte) (uint32, error) { diff --git a/cmd/stellar-rpc/internal/fullhistory/progress.go b/cmd/stellar-rpc/internal/fullhistory/progress.go deleted file mode 100644 index 2ab6ba375..000000000 --- a/cmd/stellar-rpc/internal/fullhistory/progress.go +++ /dev/null @@ -1,121 +0,0 @@ -package fullhistory - -import ( - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" -) - -// Progress is derived, never stored: every consumer recomputes from durable keys. -// "Highest complete chunk" arithmetic runs in int64 (-1 = "nothing complete") to -// avoid uint32 wraparound on the pre-genesis sentinel. - -// lastCommittedLedger derives the highest durably committed ledger: the max of the -// floor term (EarliestLedger()-1) and the cold term (the highest fully-durable -// chunk's last ledger). Computed signed so a fresh/unpinned store doesn't underflow, -// then floored at the pre-genesis base (FirstLedgerSeq-1) — the "ingest from -// genesis, nothing committed" base. -func lastCommittedLedger(cat *catalog.Catalog) (uint32, error) { - cold, err := highestDurableChunk(cat) - if err != nil { - return 0, err - } - earliest, ok, err := cat.EarliestLedger() - if err != nil { - return 0, err - } - - through := int64(chunk.FirstLedgerSeq) - 1 // pre-genesis base - if ok { - through = max(through, int64(earliest)-1) - } - if cold >= 0 { - through = max(through, int64(chunk.ID(cold).LastLedger())) //nolint:gosec // cold >= 0, a real chunk id - } - return uint32(through), nil // through >= FirstLedgerSeq-1 >= 0 -} - -// highestDurableChunk returns the highest chunk id with all artifacts durable -// (ledgers frozen AND events frozen AND (txhash frozen OR covered by a frozen -// index)), or -1 on a fresh start. A partially-frozen tip chunk is excluded — -// counting it would open reads over a partial artifact; backfill repairs it. -func highestDurableChunk(cat *catalog.Catalog) (int64, error) { - refs, err := cat.ChunkArtifactKeys() - if err != nil { - return 0, err - } - - // Frozen per-kind state per chunk. - type kinds struct{ ledgers, events, txhash bool } - frozen := map[chunk.ID]*kinds{} - for _, ref := range refs { - if ref.State != geometry.StateFrozen { - continue - } - k := frozen[ref.Chunk] - if k == nil { - k = &kinds{} - frozen[ref.Chunk] = k - } - switch ref.Kind { - case geometry.KindLedgers: - k.ledgers = true - case geometry.KindEvents: - k.events = true - case geometry.KindTxHash: - k.txhash = true - } - } - - // A frozen index coverage satisfies a chunk's txhash even after its .bin was demoted. - covered, err := frozenCoverageContains(cat) - if err != nil { - return 0, err - } - - highest := int64(-1) - for c, k := range frozen { - if !k.ledgers || !k.events { - continue - } - if !k.txhash && !covered(c) { - continue - } - if id := int64(c); id > highest { - highest = id - } - } - return highest, nil -} - -// frozenCoverageContains returns a predicate reporting whether a chunk falls in -// some frozen index coverage [Lo, Hi]; coverages are read once up front. -func frozenCoverageContains(cat *catalog.Catalog) (func(chunk.ID) bool, error) { - covs, err := cat.AllTxHashIndexKeys() - if err != nil { - return nil, err - } - var frozen []geometry.TxHashIndexCoverage - for _, cov := range covs { - if cov.State == geometry.StateFrozen { - frozen = append(frozen, cov) - } - } - return func(c chunk.ID) bool { - for _, cov := range frozen { - if cov.Lo <= c && c <= cov.Hi { - return true - } - } - return false - }, nil -} - -// chunkIDOfLedger maps a ledger to its chunk, signed so a sub-genesis ledger -// yields -1 instead of panicking like chunk.IDFromLedger. -func chunkIDOfLedger(ledger uint32) int64 { - if ledger < chunk.FirstLedgerSeq { - return -1 - } - return int64(chunk.IDFromLedger(ledger)) -} diff --git a/cmd/stellar-rpc/internal/fullhistory/progress_test.go b/cmd/stellar-rpc/internal/fullhistory/progress_test.go deleted file mode 100644 index 6fc469049..000000000 --- a/cmd/stellar-rpc/internal/fullhistory/progress_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package fullhistory - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" -) - -// --------------------------------------------------------------------------- -// progress derivation test helpers. -// --------------------------------------------------------------------------- - -// makeChunkDurable freezes ledgers+events+txhash for a chunk — the durable state -// highestDurableChunk counts. -func makeChunkDurable(t *testing.T, cat *catalog.Catalog, c chunk.ID) { - t.Helper() - freezeKinds(t, cat, c, geometry.KindLedgers, geometry.KindEvents, geometry.KindTxHash) -} - -// --------------------------------------------------------------------------- -// lastCommittedLedger — chunk-granularity bound, pure catalog read. -// --------------------------------------------------------------------------- - -func TestLastCommittedLedger(t *testing.T) { - t.Run("fresh store => pre-genesis sentinel, never MaxUint32", func(t *testing.T) { - // Every term is -1; the signed domain must yield FirstLedgerSeq-1, not wrap. - cat, _ := testCatalog(t) - got, err := lastCommittedLedger(cat) - require.NoError(t, err) - require.Equal(t, preGenesisLedger, got) - }) - - t.Run("cold term leads: highest fully-durable chunk", func(t *testing.T) { - cat, _ := testCatalog(t) - makeChunkDurable(t, cat, 0) - makeChunkDurable(t, cat, 1) - makeChunkDurable(t, cat, 2) - got, err := lastCommittedLedger(cat) - require.NoError(t, err) - require.Equal(t, chunk.ID(2).LastLedger(), got) - }) - - t.Run("incompletely-frozen tip degrades the bound (ledgers frozen, events freezing)", func(t *testing.T) { - cat, _ := testCatalog(t) - makeChunkDurable(t, cat, 0) - makeChunkDurable(t, cat, 1) - // Chunk 2 mid-freeze (events only "freezing") must NOT count: bound stays at 1. - freezeKinds(t, cat, 2, geometry.KindLedgers, geometry.KindTxHash) - require.NoError(t, cat.MarkChunkFreezing(2, geometry.KindEvents)) - got, err := lastCommittedLedger(cat) - require.NoError(t, err) - require.Equal(t, chunk.ID(1).LastLedger(), got) - }) - - t.Run("txhash satisfied by a frozen index coverage (post-finalization demote)", func(t *testing.T) { - cat, _ := testCatalog(t) - // Chunk 7: txhash demoted but a frozen index coverage spans it ⇒ still durable. - freezeKinds(t, cat, 7, geometry.KindLedgers, geometry.KindEvents) - freezeCoverage(t, cat, cat.TxHashIndexLayout().TxHashIndexID(7), 0, 999) // window 0 covers chunk 7 - got, err := lastCommittedLedger(cat) - require.NoError(t, err) - require.Equal(t, chunk.ID(7).LastLedger(), got) - }) - - t.Run("chunk NOT covered by any frozen index and no frozen txhash does not count", func(t *testing.T) { - cat, _ := testCatalog(t) - makeChunkDurable(t, cat, 0) - // Chunk 1: ledgers+events frozen, no txhash, no covering index. - freezeKinds(t, cat, 1, geometry.KindLedgers, geometry.KindEvents) - got, err := lastCommittedLedger(cat) - require.NoError(t, err) - require.Equal(t, chunk.ID(0).LastLedger(), got, "chunk 1 not durable; bound stays at chunk 0") - }) - - t.Run("earliest pin floor leads when above the cold term", func(t *testing.T) { - cat, _ := testCatalog(t) - // Floor pinned mid-chain, no chunks durable, no hot keys. - const floor = 50000 - require.NoError(t, cat.PinEarliestLedger(floor)) - got, err := lastCommittedLedger(cat) - require.NoError(t, err) - require.Equal(t, uint32(floor-1), got) - }) - - t.Run("earliest pin == genesis (2) does not underflow", func(t *testing.T) { - cat, _ := testCatalog(t) - require.NoError(t, cat.PinEarliestLedger(chunk.FirstLedgerSeq)) - got, err := lastCommittedLedger(cat) - require.NoError(t, err) - require.Equal(t, preGenesisLedger, got, "earliest 2 - 1 = 1, not MaxUint32") - }) - - t.Run("max of the cold term and the earliest floor", func(t *testing.T) { - cat, _ := testCatalog(t) - makeChunkDurable(t, cat, 3) // cold => chunk 3 last ledger (the higher term) - require.NoError(t, cat.PinEarliestLedger(2)) - got, err := lastCommittedLedger(cat) - require.NoError(t, err) - require.Equal(t, chunk.ID(3).LastLedger(), got) - }) -} diff --git a/cmd/stellar-rpc/internal/fullhistory/retention.go b/cmd/stellar-rpc/internal/fullhistory/retention.go deleted file mode 100644 index 2b0462390..000000000 --- a/cmd/stellar-rpc/internal/fullhistory/retention.go +++ /dev/null @@ -1,48 +0,0 @@ -package fullhistory - -import ( - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" -) - -// RetentionFloor is the lowest chunk still within retention; any chunk below it -// is eligible for discard/prune. It is the reader-side retention contract -// (design "Reader retention contract", gettx §8.2 / §8.5): availability is -// decided by retention, not the on-disk file set, which lets prune/sweep unlink -// a chunk the instant it passes the floor without coordinating with the index -// lifecycle (a stale .idx pointing at a pruned .pack is masked). The floor may -// err LOW harmlessly — a wrongly-retained chunk still hits the reader's -// missing-file rule — so it anchors on the same live completeThrough the prune -// scan uses; widening history is backfill's job, not the floor's. -type RetentionFloor struct { - chunk chunk.ID // lowest in-retention chunk -} - -// NewRetentionFloor pins the floor for one (through, retentionChunks, earliest) -// snapshot. A shortened retentionChunks raises the floor at once — no per-chunk -// state to migrate. -func NewRetentionFloor(through, retentionChunks, earliest uint32) RetentionFloor { - return RetentionFloor{chunk: retentionFloorChunk(through, retentionChunks, earliest)} -} - -// Excludes reports whether chunk c is below the floor — past retention, eligible -// for discard/prune. The discard and prune scans (eligibility.go) use it on a -// chunk directly and, since an index is below the floor exactly when its last -// chunk is, as Excludes(layout.LastChunk(idx)) for a whole tx-hash index. (The -// reader's seq-level admit predicate and the ledger-seq floor for §8.2 coverage -// filtering return with the read path, #772.) -func (f RetentionFloor) Excludes(c chunk.ID) bool { return c < f.chunk } - -// retentionFloorChunk is the retention window's lower bound as a chunk id (the -// design's retentionFloorChunk): the HIGHER of the sliding floor (retentionChunks -// back from the last complete chunk) and the fixed earliest_ledger. slidingChunk is -// signed so a young store / large retentionChunks clamps to chunk 0 instead of -// underflowing. Both terms are chunk-first-ledgers, so IDFromLedger is exact. -func retentionFloorChunk(upperBound, retentionChunks, earliest uint32) chunk.ID { - sliding := uint32(chunk.FirstLedgerSeq) // GenesisLedger - if retentionChunks > 0 { - slidingChunk := geometry.LastCompleteChunkAt(upperBound) - int64(retentionChunks) + 1 - sliding = geometry.ChunkFirstLedger(max(slidingChunk, 0)) - } - return chunk.IDFromLedger(max(sliding, earliest)) -} diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index 34e49dd46..6952ae8df 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -10,6 +10,7 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/observability" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) @@ -37,13 +38,14 @@ func run(ctx context.Context, cfg StartConfig) error { } // Derived, never stored: highest durably-committed ledger, clamped by earliest-1. - lastCommitted, err := lastCommittedLedger(cat) + // nil probe — catch-up startup reads the cold tier at chunk granularity (no hot DB). + lastCommitted, err := lifecycle.LastCommittedLedger(cat, nil) if err != nil { return fmt.Errorf("startup derive last-committed: %w", err) } metrics := observability.MetricsOrNop(cfg.Exec.Metrics) - metrics.LastCommitted(lastCommitted, retentionFloorChunk(lastCommitted, cfg.RetentionChunks, earliest).FirstLedger()) + metrics.LastCommitted(lastCommitted, lifecycle.EffectiveRetentionFloor(lastCommitted, cfg.RetentionChunks, earliest)) logger.WithField("last_committed", lastCommitted). WithField("earliest", earliest). WithField("pinned", pinned). @@ -103,7 +105,7 @@ func backfillToTip(ctx context.Context, cfg StartConfig, lastCommitted, earliest // max() guards a lagging bulk tip: the tip alone could regress the floor below // pruning or drop a complete last-committed chunk. anchor := max(tip, lastCommitted) - rangeStart := retentionFloorChunk(anchor, retentionChunks, earliest) + rangeStart := chunk.IDFromLedger(lifecycle.EffectiveRetentionFloor(anchor, retentionChunks, earliest)) // Same anchor for rangeEnd: a complete last-committed chunk above a lagging tip // still folds in; chunks beyond the tip are durable and self-skip. @@ -112,7 +114,7 @@ func backfillToTip(ctx context.Context, cfg StartConfig, lastCommitted, earliest // Mid-chunk resume exclusion: a mid-chunk last-committed within one chunk of the tip // leaves the partial resume chunk to ingestion. Signed so genesis reads as a boundary. if withinOneChunkOfTip(tip, lastCommitted) && lastCommittedMidChunk(lastCommitted) { - rangeEndSigned = chunkIDOfLedger(lastCommitted) - 1 // one short of the live chunk + rangeEndSigned = lifecycle.ChunkIDOfLedger(lastCommitted) - 1 // one short of the live chunk } // Break on an empty or non-advancing range. @@ -139,7 +141,14 @@ func backfillToTip(ctx context.Context, cfg StartConfig, lastCommitted, earliest metrics.BackfillPass(passDuration) // Refresh the derived gauges as last-committed advances and the floor rises with it. - metrics.LastCommitted(lastCommitted, retentionFloorChunk(lastCommitted, retentionChunks, earliest).FirstLedger()) + metrics.LastCommitted(lastCommitted, lifecycle.EffectiveRetentionFloor(lastCommitted, retentionChunks, earliest)) + // Sample the cold-tier footprint once per pass (a full tree-walk is too costly + // per-chunk); a walk error just leaves the gauge at its last value. + if footprint, cerr := observability.MeasureColdTierBytes(cfg.Exec.Catalog.Layout()); cerr == nil { + metrics.ColdTierBytes(footprint) + } else { + logger.WithError(cerr).Debug("cold-tier footprint sample failed; skipping gauge") + } logger.WithField("range_lo", rangeStart.String()). WithField("range_hi", rangeEnd.String()). WithField("last_committed", lastCommitted). @@ -156,14 +165,10 @@ func withinOneChunkOfTip(tip, lastCommitted uint32) bool { } // lastCommittedMidChunk reports whether lastCommitted falls strictly inside a chunk. -// The only sub-genesis value it sees is the fresh-start sentinel preGenesisLedger, -// where chunkIDOfLedger yields -1 and chunk.ID(-1).LastLedger() wraps (MaxUint32+1 -// overflows to 0) back to exactly preGenesisLedger — so the comparison reports a -// boundary (false) without a special case. +// The genesis sentinel reads as a boundary, never mid-chunk. func lastCommittedMidChunk(lastCommitted uint32) bool { - c := chunkIDOfLedger(lastCommitted) - //nolint:gosec // c is -1 (wraps to preGenesisLedger) or a real chunk id - return lastCommitted != chunk.ID(c).LastLedger() + c := lifecycle.ChunkIDOfLedger(lastCommitted) + return lastCommitted != lifecycle.CompleteThrough(c) } // ErrFirstStartNoTip is the first-start FATAL: no local progress and no reachable From 7e687c152bc074758951c63929a570ee1ca54cd8 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Tue, 30 Jun 2026 00:20:12 -0400 Subject: [PATCH 02/55] fullhistory: rename ingest.go -> hotloop.go to disambiguate from ingest pkg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The file is the daemon's hot-tier ingestion loop (package fullhistory) and sat next to — and imported — the ingest package, reading as a duplicate. Pure file rename; no identifier or package changes. --- cmd/stellar-rpc/internal/fullhistory/{ingest.go => hotloop.go} | 0 .../internal/fullhistory/{ingest_test.go => hotloop_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename cmd/stellar-rpc/internal/fullhistory/{ingest.go => hotloop.go} (100%) rename cmd/stellar-rpc/internal/fullhistory/{ingest_test.go => hotloop_test.go} (100%) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go similarity index 100% rename from cmd/stellar-rpc/internal/fullhistory/ingest.go rename to cmd/stellar-rpc/internal/fullhistory/hotloop.go diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go similarity index 100% rename from cmd/stellar-rpc/internal/fullhistory/ingest_test.go rename to cmd/stellar-rpc/internal/fullhistory/hotloop_test.go From 96d0e44cd07968e80815351c67c19f209e840e86 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Tue, 30 Jun 2026 00:57:56 -0400 Subject: [PATCH 03/55] fullhistory: drop gratuitous comment churn in untouched sections change (e.g. State* lifecycle comments, eventstore method docs, the cold ingest drivers), inflating the diff and risking conflicts with sibling PRs. Revert comment-only changes to base wording; keep comments that document genuinely new/changed code (hot store, lifecycle, hotchunk). No code changes. --- .../internal/fullhistory/catalog/artifacts.go | 5 +- .../internal/fullhistory/catalog/catalog.go | 71 ++-- .../fullhistory/catalog/catalog_protocol.go | 76 ++-- .../internal/fullhistory/geometry/keys.go | 39 +- .../internal/fullhistory/geometry/paths.go | 82 +++-- .../internal/fullhistory/ingest/doc.go | 93 +++-- .../internal/fullhistory/ingest/driver.go | 61 ++-- .../internal/fullhistory/ingest/ingester.go | 46 ++- .../internal/fullhistory/ingest/service.go | 46 ++- .../pkg/stores/eventstore/hot_store.go | 337 ++++++++++++------ .../pkg/stores/ledger/hot_store.go | 56 ++- .../pkg/stores/txhash/hot_store.go | 74 ++-- 12 files changed, 638 insertions(+), 348 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/artifacts.go b/cmd/stellar-rpc/internal/fullhistory/catalog/artifacts.go index f63d5608a..9d8b89681 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/artifacts.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/artifacts.go @@ -12,9 +12,8 @@ import ( // processChunk narrows it by dropping already-frozen kinds (per-kind // idempotency). // -// The representation is a fixed-width bitmask over allKinds' canonical order, so -// Kinds() yields kinds in that order (the canonical ledgers→txhash→events order -// the cold ingesters build in) and membership tests are allocation-free. +// Backed by a fixed-width bitmask over allKinds' canonical order, so Kinds() +// yields that order (matching buildColdIngesters) and membership is alloc-free. type ArtifactSet struct { mask uint8 } diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go index 6df51bbaf..1f8cb3d04 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go @@ -13,20 +13,27 @@ import ( ) // Catalog is the streaming daemon's view of durable state. It WRAPS -// metastore.Store (never reaching around it to RocksDB) and, on top of geometry -// (key schema, key↔path bijection, index arithmetic, fsync helpers), adds the -// one-write protocol (catalog_protocol.go) and the key-driven sweeps -// (catalog_sweep.go). Every key names a file/dir state or a config pin; progress -// is derived, never stored. The read-then-act sequences carry no concurrency -// guard — the design guarantees one writer per key (see catalog_protocol.go). +// metastore.Store — the merged RocksDB KV store with sync Put/Delete, atomic +// Batch, and PrefixScan — never reaching around it to RocksDB directly. On top +// of the geometry package (the key schema + its bijection to disk paths, the +// tx-hash-index arithmetic, and the fsync helpers) it adds the one-write +// protocol (catalog_protocol.go) and the key-driven sweeps (catalog_sweep.go). +// +// Every key names a file/dir state or a config pin; progress is derived, never +// stored. +// +// The read-then-act sequences in the write protocol and the sweeps carry no +// concurrency guard: the design's Concurrency model guarantees one writer per +// key (see the header note in catalog_protocol.go). type Catalog struct { store *metastore.Store layout geometry.Layout txhashIndex geometry.TxHashIndexLayout } -// NewCatalog binds a catalog to an open (caller-owned) metastore.Store, the -// layout, and the index arithmetic. The catalog never closes the store. +// NewCatalog binds a catalog to an open metastore.Store, the on-disk layout, +// and the tx-hash-index arithmetic. The store is caller-owned; the catalog never +// closes it. func NewCatalog(store *metastore.Store, layout geometry.Layout, txhashIndex geometry.TxHashIndexLayout) *Catalog { return &Catalog{store: store, layout: layout, txhashIndex: txhashIndex} } @@ -35,10 +42,12 @@ func (c *Catalog) Layout() geometry.Layout { return c.layout } func (c *Catalog) TxHashIndexLayout() geometry.TxHashIndexLayout { return c.txhashIndex } -// --- Typed artifact-state accessors --- +// --------------------------------------------------------------------------- +// Typed artifact-state accessors. +// --------------------------------------------------------------------------- // State returns the lifecycle State of a per-chunk artifact key, or the empty -// State when the key is absent. +// State when the key is absent — neither file nor in-progress write exists. func (c *Catalog) State(chunkID chunk.ID, kind geometry.Kind) (geometry.State, error) { v, ok, err := c.get(geometry.ChunkKey(chunkID, kind)) if err != nil || !ok { @@ -58,8 +67,11 @@ func (c *Catalog) HotState(chunkID chunk.ID) (geometry.HotState, error) { return geometry.HotState(v), nil } -// --- Scans. Every "find work" iterates keys via PrefixScan (never lists a -// directory); results are returned sorted so callers need no second pass. --- +// --------------------------------------------------------------------------- +// Scans. Every "find work" operation iterates keys via PrefixScan; nothing +// lists a directory. Results are returned sorted so callers need no second +// pass. +// --------------------------------------------------------------------------- // ChunkArtifactKeys returns every per-chunk artifact key with its value, sorted // by key — the deletion/audit surface for chunk:* keys. @@ -102,9 +114,10 @@ func (c *Catalog) AllTxHashIndexKeys() ([]geometry.TxHashIndexCoverage, error) { return c.txhashIndexKeysByPrefix(geometry.TxHashIndexPrefix) } -// FrozenTxHashIndex returns the index's UNIQUE "frozen" coverage (what readers -// resolve as "the index"), or ok=false if none yet. Asserts INV-2 (at most one -// frozen coverage per index) by erroring on two — a detectable bug, not a tie. +// FrozenTxHashIndex returns the index's UNIQUE "frozen" coverage — the key +// readers resolve as "the index" — or ok=false if the index has none +// yet. It asserts INV-2 (at most one frozen coverage per index at any moment) +// by erroring if it observes two — a detectable bug, not a tie-break to resolve. func (c *Catalog) FrozenTxHashIndex(w geometry.TxHashIndexID) (geometry.TxHashIndexCoverage, bool, error) { covs, err := c.TxHashIndexKeys(w) if err != nil { @@ -130,22 +143,27 @@ func (c *Catalog) FrozenTxHashIndex(w geometry.TxHashIndexID) (geometry.TxHashIn return frozen, found, nil } -// --- Config pins. Written once on first start, immutable thereafter. --- +// --------------------------------------------------------------------------- +// Config pins. Written once on first start, immutable thereafter. +// --------------------------------------------------------------------------- -// EarliestLedger returns the pinned config:earliest_ledger (chunk-aligned). ok is -// false if not yet written (a pristine store). +// EarliestLedger returns the pinned config:earliest_ledger (chunk-aligned). ok +// is false if the pin has not been written yet (a pristine store). func (c *Catalog) EarliestLedger() (uint32, bool, error) { return c.uint32Pin(geometry.ConfigEarliestLedger) } -// PinEarliestLedger commits the config:earliest_ledger pin in one synced write. -// Its presence is the sentinel that a prior first start completed; once written -// it is immutable and validated-or-abort on every restart. +// PinEarliestLedger commits the config:earliest_ledger pin in one synced write — +// the first-start commit validateConfig mandates. Its presence is the sentinel +// that a prior first start completed: once written, earliest_ledger is immutable +// and validated-or-abort on every restart. chunks_per_txhash_index is no longer +// pinned — it is the fixed geometry.ChunksPerTxhashIndex constant. func (c *Catalog) PinEarliestLedger(earliestLedger uint32) error { return c.store.Put(geometry.ConfigEarliestLedger, strconv.FormatUint(uint64(earliestLedger), 10)) } -// ArtifactRef is the (chunk, kind, State) unit the sweeps and resolver pass around. +// ArtifactRef names one per-chunk artifact and the State observed for it — the +// (chunk, kind, State) unit the sweeps and resolver pass around. type ArtifactRef struct { Chunk chunk.ID Kind geometry.Kind @@ -154,10 +172,13 @@ type ArtifactRef struct { func (r ArtifactRef) Key() string { return geometry.ChunkKey(r.Chunk, r.Kind) } -// --- Unexported helpers backing the scans and pin getters above. --- +// --------------------------------------------------------------------------- +// Unexported helpers backing the scans and pin getters above. +// --------------------------------------------------------------------------- -// get returns the value at key; ok is false (err nil) on a clean miss, -// distinguishing "absent" from a backing-store error. +// get returns the value at key. The bool is false (err nil) on a clean miss, +// distinguishing "absent" from a backing-store error — the value-blind primitive +// the typed reads above build on. func (c *Catalog) get(key string) (string, bool, error) { v, err := c.store.Get(key) if errors.Is(err, stores.ErrNotFound) { diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol.go index a3f849a35..85d3f44dd 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol.go @@ -11,24 +11,30 @@ import ( // The one write protocol — mark-then-write. Every durable artifact (per-chunk // file or index coverage) flows through here: // -// 1. Put the key "freezing" BEFORE any I/O. -// 2. Caller writes the file. -// 3. Caller fsyncs the file + parent dirent (+ grandparent on a new parent dir) -// — geometry.BarrierNewFile. -// 4. Flip to "frozen" — one Put per-chunk, or one atomic Batch for the index. +// 1. Put the key "freezing" via metastore BEFORE any I/O. +// 2. The caller writes the file. +// 3. The caller fsyncs the FILE + its PARENT dirent (+ the GRANDPARENT dirent +// when the parent dir was just created) — geometry.BarrierNewFile. +// 4. Flip to "frozen": a single Put for per-chunk artifacts, or one atomic +// Batch for the index (see CommitTxHashIndex). // // "frozen" is the only transition readers trust. The catalog owns steps 1 and 4 // (meta writes); the caller owns 2 and 3 (I/O). // -// One writer per key: the read-then-act sequences here and in the sweeps are -// UNGUARDED against a racing second writer because the design rules it out -// (ingest and lifecycle never write the same key; resolve emits one build per -// window; one lifecycle run at a time). Crash re-runs stay safe because every -// step is idempotent and resolve re-plans from durable state. +// One writer per key. The read-then-act sequences here and in the sweeps +// (catalog_sweep.go) — read a key/coverage, then Put or unlink based on it — are +// deliberately UNGUARDED against a second writer racing the same key, because the +// design's Concurrency model rules that out: the ingest thread and the lifecycle +// thread write the catalog at the same time but NEVER the same key; resolve emits +// exactly one index build per window (so even concurrent backfill never has two +// builds for one index); and only one lifecycle run executes at a time. Crash +// re-runs stay safe not because of any in-method guard but because every step is +// idempotent and resolve re-plans from durable state. See the design's +// "Concurrency model" section. // MarkChunkFreezing is step 1 for every requested kind. Re-marking a -// freezing/pruning/absent key is idempotent; skipping an already-frozen kind is -// the caller's job. +// "freezing"/"pruning"/absent key is idempotent re-materialization; skipping an +// already-"frozen" kind (per-kind idempotency) is the caller's job. func (c *Catalog) MarkChunkFreezing(chunkID chunk.ID, kinds ...geometry.Kind) error { if len(kinds) == 0 { return errors.New("streaming: MarkChunkFreezing requires at least one kind") @@ -41,8 +47,9 @@ func (c *Catalog) MarkChunkFreezing(chunkID chunk.ID, kinds ...geometry.Kind) er }) } -// FlipChunkFrozen is step 4 for per-chunk artifacts: flips every kind to -// "frozen". The caller MUST have completed BarrierNewFile for every file first. +// FlipChunkFrozen is step 4 for per-chunk artifacts: flips every requested kind +// to "frozen". The caller MUST have completed geometry.BarrierNewFile for every +// file first. func (c *Catalog) FlipChunkFrozen(chunkID chunk.ID, kinds ...geometry.Kind) error { if len(kinds) == 0 { return errors.New("streaming: FlipChunkFrozen requires at least one kind") @@ -55,8 +62,8 @@ func (c *Catalog) FlipChunkFrozen(chunkID chunk.ID, kinds ...geometry.Kind) erro }) } -// MarkTxHashIndexFreezing is step 1 for the index, returning the coverage for -// CommitTxHashIndex. lo > hi panics (TxHashIndexKey enforces it). +// MarkTxHashIndexFreezing is step 1 for the index, returning the TxHashIndexCoverage for +// CommitTxHashIndex. lo > hi panics (geometry.TxHashIndexKey enforces it). func (c *Catalog) MarkTxHashIndexFreezing( w geometry.TxHashIndexID, lo, hi chunk.ID, ) (geometry.TxHashIndexCoverage, error) { @@ -76,24 +83,31 @@ func (c *Catalog) MarkTxHashIndexFreezing( // CommitTxHashIndex is step 4 for the index. In one atomic batch it: // // - promotes cov ("freezing" -> "frozen"); -// - demotes the predecessor frozen coverage (if any) to "pruning"; -// - iff terminal (cov.Hi == index's last chunk), demotes every chunk:{c}:txhash -// key in cov's [Lo, Hi] range to "pruning". +// - demotes the index's predecessor frozen coverage (if any) to "pruning"; +// - iff this build is terminal (cov.Hi == index's last chunk), demotes the +// chunk:{c}:txhash key of every chunk in cov's [Lo, Hi] range to "pruning". // -// The batch only DEMOTES (file deletion is the sweeps' job), so there is never an -// instant with two frozen coverages, an unreachable live index, or a "frozen" -// chunk:c:txhash whose .bin was deleted. A re-commit is an idempotent overwrite -// (crash re-run). The caller MUST have fsynced the .idx and its dir first; the -// predecessor is re-read from durable state, so this is crash-safe. +// The batch only DEMOTES keys — file deletion is the sweeps' job. So there is no +// instant with two frozen coverages, no live index unreachable, and no "frozen" +// chunk:c:txhash whose .bin was deleted. +// +// A re-commit of the already-frozen coverage is an idempotent overwrite — the +// crash-re-run case. There is no guard against an out-of-order or duplicate build +// for the same index: the design's Concurrency model precludes it (resolve emits +// one build per window; one lifecycle run at a time — see the header note). +// +// The caller MUST have fsynced the .idx file and its dir first. The predecessor +// is re-read from durable state, so this is safe to call after a crash. func (c *Catalog) CommitTxHashIndex(cov geometry.TxHashIndexCoverage) error { - // Compose demotions against durable state BEFORE the batch, so the batch body - // is a pure sequence of puts. + // Compose demotions against durable state BEFORE opening the batch, so the + // batch body is a pure sequence of puts. prev, hasPrev, err := c.FrozenTxHashIndex(cov.Index) if err != nil { return err } if hasPrev && prev.Key == cov.Key { - // Re-commit of an already-landed batch: nothing to demote against itself. + // Re-commit of an already-landed batch: nothing to demote against itself; + // the promote below is an idempotent overwrite. hasPrev = false } @@ -118,9 +132,11 @@ func (c *Catalog) CommitTxHashIndex(cov geometry.TxHashIndexCoverage) error { } // txhashIndexChunkKeysPresent returns the chunk:{c}:txhash keys that EXIST in -// [lo, hi]. A terminal commit passes cov's own range, so it demotes only the .bin -// inputs the new .idx covers — never a key below cov.Lo (whose .bin must survive -// for its own index) nor a chunk whose .bin was never produced. +// the inclusive chunk range [lo, hi]. A terminal commit passes cov's own range, +// so it demotes only the .bin inputs the new .idx actually covers — never a key +// below cov.Lo (whose ledgers the new index cannot answer, and whose .bin must +// survive for its own index's build) and never a chunk whose .bin was never +// produced (the spec's cat.Has guard). func (c *Catalog) txhashIndexChunkKeysPresent(lo, hi chunk.ID) ([]string, error) { var keys []string for cid := lo; cid <= hi; cid++ { diff --git a/cmd/stellar-rpc/internal/fullhistory/geometry/keys.go b/cmd/stellar-rpc/internal/fullhistory/geometry/keys.go index fb684d788..e1fc33ff8 100644 --- a/cmd/stellar-rpc/internal/fullhistory/geometry/keys.go +++ b/cmd/stellar-rpc/internal/fullhistory/geometry/keys.go @@ -15,12 +15,15 @@ import ( type State string const ( - // StateFreezing — file being written. Set BEFORE any I/O (mark-then-write), - // so a crash mid-write is detectable and every file is reachable from a key. + // StateFreezing — the immutable file is being written. Set BEFORE any I/O + // (mark-then-write), so a crash mid-write is detectable from the key alone + // and every on-disk file is reachable from a key. StateFreezing State = "freezing" - // StateFrozen — file + dirent fsynced and durable. Trusted blindly by readers. + // StateFrozen — file and dirent are fsynced and durable. Trusted blindly by + // readers, the resolver, and buildTxhashIndex's precondition. StateFrozen State = "frozen" - // StatePruning — queued for removal. A sweep unlinks, then deletes the key. + // StatePruning — file queued for removal, may or may not still be on disk. + // A sweep finishes the unlink, then deletes the key. StatePruning State = "pruning" ) @@ -68,15 +71,18 @@ type TxHashIndexID uint32 // chunk ids, matching the {idx:08d} segment in keys and paths. func (i TxHashIndexID) String() string { return fmt.Sprintf("%08d", uint32(i)) } -// --- Key prefixes and constructors — single source of truth for the key↔path -// bijection (paths.go holds the inverse). --- +// --------------------------------------------------------------------------- +// Key prefixes and constructors — the single source of truth for the +// key<->path bijection (paths.go holds the inverse). +// --------------------------------------------------------------------------- const ( ChunkPrefix = "chunk:" HotChunkPrefix = "hot:chunk:" TxHashIndexPrefix = "txhash_index:" - // ConfigEarliestLedger is the sole config pin key. + // ConfigEarliestLedger is the sole config pin key. (chunks_per_txhash_index is + // the fixed ChunksPerTxhashIndex constant, not a pin.) ConfigEarliestLedger = "config:earliest_ledger" ) @@ -91,9 +97,9 @@ func HotChunkKey(c chunk.ID) string { return HotChunkPrefix + c.String() } -// TxHashIndexKey returns txhash_index:{idx:08d}:{lo:08d}:{hi:08d}. The coverage -// [lo, hi] lives in the key NAME; the value is pure lifecycle state. lo > hi -// panics (programmer error). +// TxHashIndexKey returns the index coverage key txhash_index:{idx:08d}:{lo:08d}:{hi:08d}. +// The coverage [lo, hi] lives in the key NAME; the value is pure lifecycle +// state. lo > hi is a programmer error, surfaced loudly via panic. func TxHashIndexKey(idx TxHashIndexID, lo, hi chunk.ID) string { if lo > hi { panic(fmt.Sprintf("streaming: TxHashIndexKey lo %s > hi %s", lo, hi)) @@ -101,16 +107,19 @@ func TxHashIndexKey(idx TxHashIndexID, lo, hi chunk.ID) string { return TxHashIndexPrefix + idx.String() + ":" + lo.String() + ":" + hi.String() } -// TxHashIndexPrefixFor returns the scan prefix txhash_index:{idx:08d}: that -// enumerates all coverage keys of one index. +// TxHashIndexPrefixFor returns the scan prefix txhash_index:{idx:08d}: that enumerates +// all coverage keys of one index. func TxHashIndexPrefixFor(idx TxHashIndexID) string { return TxHashIndexPrefix + idx.String() + ":" } -// --- Key parsing — each parser is the reverse bijection of one constructor. --- +// --------------------------------------------------------------------------- +// Key parsing — each parser is the reverse bijection of exactly one +// constructor above. +// --------------------------------------------------------------------------- -// TxHashIndexCoverage is one parsed index coverage key: the index, range [Lo, -// Hi], the full key string, and its lifecycle State. +// TxHashIndexCoverage is one parsed index coverage key: the index, the covered +// chunk range [Lo, Hi], the full key string, and its lifecycle State. type TxHashIndexCoverage struct { Index TxHashIndexID Lo, Hi chunk.ID diff --git a/cmd/stellar-rpc/internal/fullhistory/geometry/paths.go b/cmd/stellar-rpc/internal/fullhistory/geometry/paths.go index 60c07cef7..58eb6752b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/geometry/paths.go +++ b/cmd/stellar-rpc/internal/fullhistory/geometry/paths.go @@ -10,10 +10,10 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash" ) -// Layout is the SINGLE source of truth for storage paths: a fixed key↔path -// bijection holding one root per artifact tree, so a Layout plus a key finds any -// file without listing a directory. NewLayout defaults all roots under one data -// dir: +// Layout is the SINGLE source of truth for storage paths: a fixed key<->path +// bijection (design-docs/full-history-streaming-workflow.md "Directory layout") +// holding one root PER artifact tree, so a Layout plus a key finds any file +// without listing a directory. NewLayout defaults all roots under one data dir: // // {root}/ // ├── catalog/rocksdb/ @@ -24,8 +24,8 @@ import ( // ├── raw/{bucket:05d}/{chunk:08d}.bin // └── index/{idx:08d}/{lo:08d}-{hi:08d}.idx // -// Each root is independently settable (NewLayoutFromRoots) for [storage] -// overrides. Bucket ids never appear in meta-store keys. +// Each root is independently settable (NewLayoutFromRoots) for the [storage] +// path overrides. Bucket ids never appear in meta-store keys. type Layout struct { catalogRoot string // meta-store RocksDB dir (a leaf, not a tree root) hotRoot string @@ -49,8 +49,10 @@ func NewLayout(root string) Layout { } // NewLayoutFromRoots binds a Layout to explicit per-tree roots — the resolved, -// overridable storage paths. Taking strings (not the config Paths struct) keeps -// geometry free of a config dependency; NewLayoutFromPaths adapts a Paths to this. +// independently-overridable storage paths the daemon flocks and opens. Taking +// strings (rather than the config Paths struct) keeps geometry free of any +// config dependency; the streaming package's NewLayoutFromPaths adapts a Paths +// to this so lock and data location can never disagree. func NewLayoutFromRoots(catalogRoot, hotRoot, ledgersRoot, eventsRoot, txhashRawRoot, txhashIndexRoot string) Layout { return Layout{ catalogRoot: catalogRoot, @@ -74,7 +76,8 @@ func (l Layout) HotChunkPath(c chunk.ID) string { } // LedgerPackPath is a chunk's ledger pack. Layout composes the bucket dir; the -// leaf is owned by ledger.PackName. EventsPaths/TxHashBinPath split the same way. +// leaf is owned by ledger.PackName (shared with the cold writer and reader). +// EventsPaths/TxHashBinPath follow the same split. func (l Layout) LedgerPackPath(c chunk.ID) string { return filepath.Join(l.ledgersRoot, c.BucketID(), ledger.PackName(c)) } @@ -101,9 +104,9 @@ func (l Layout) LedgersRoot() string { return l.ledgersRoot } // EventsRoot is the root EventsPaths composes under. func (l Layout) EventsRoot() string { return l.eventsRoot } -// TxHashRawRoot is the root under which per-chunk raw txhash runs are bucketed -// (matches TxHashBinPath). Its own root because the cold pipeline takes an -// explicit per-kind root (ingest.ColdDirs), not a coldDir/ derivation. +// TxHashRawRoot is its own root because the cold pipeline takes an explicit +// per-kind root (ingest.ColdDirs) rather than the single coldDir/ +// layout RunCold derives. func (l Layout) TxHashRawRoot() string { return l.txhashRawRoot } // TxHashIndexRoot is the root TxHashIndexDir composes under. @@ -121,8 +124,8 @@ func (l Layout) TxHashIndexFilePath(cov TxHashIndexCoverage) string { return filepath.Join(l.TxHashIndexDir(cov.Index), name) } -// ArtifactPaths is the single (chunk, kind)->files map, so the sweep and freeze -// writer agree on what a kind owns on disk. +// ArtifactPaths is the single (chunk, kind)->files map, so the sweep and the +// freeze writer agree on what a kind owns on disk. func (l Layout) ArtifactPaths(c chunk.ID, kind Kind) []string { switch kind { case KindLedgers: @@ -136,12 +139,16 @@ func (l Layout) ArtifactPaths(c chunk.ID, kind Kind) []string { } } -// --- fsync barriers for the one-write protocol and sweeps. A creation is -// durable only once the file's data AND the dirent naming it are fsynced; a -// freshly created dir needs its own parent fsynced too. --- +// --------------------------------------------------------------------------- +// fsync barriers — the os-level durability primitives the one-write protocol and +// the sweeps depend on. A creation is durable only once both the file's data AND +// the directory entry naming it are fsynced; a freshly created directory needs +// its own parent fsynced too. See the One write protocol section: "the key never +// outlives the file's creation". +// --------------------------------------------------------------------------- -// syncAndClose fsyncs an open handle then closes it, preferring the sync error so -// a durability failure is never masked. +// syncAndClose fsyncs an open file/dir handle then closes it, preferring the +// sync error over the close error so a durability failure is never masked. func syncAndClose(f *os.File) error { syncErr := f.Sync() closeErr := f.Close() @@ -161,9 +168,10 @@ func fsyncFile(path string) error { return syncAndClose(f) } -// FsyncDir fsyncs a directory entry, making creations/unlinks within it durable. -// A missing dir is not an error: a sweep may run where the file (and its -// on-demand dir) was never created. +// FsyncDir fsyncs a directory entry, making creations and unlinks within it +// durable. A missing directory is not an error: a sweep may run where the file +// (and its on-demand bucket/index dir) was never created, so there is no dirent +// to make durable. func FsyncDir(dir string) error { f, err := os.Open(dir) if os.IsNotExist(err) { @@ -203,9 +211,10 @@ func FsyncParentDirs(paths []string) error { // BarrierNewFile applies the two-level barrier to a freshly written file: fsync // the file, its parent dir, then the grandparent dirent. The grandparent fsync -// persists the parent's own dirent — load-bearing when the write just created the -// parent (a new bucket every 1000th chunk); on an unchanged grandparent it is -// nearly free, so it runs unconditionally rather than tracking parent-newness. +// persists the parent's own directory entry, which matters when the write just +// created the parent (e.g. a new bucket every 1000th chunk). On an unchanged +// grandparent it has no dirty metadata to flush and is nearly free, so the +// barrier runs it unconditionally rather than tracking whether the parent is new. func BarrierNewFile(path string) error { if err := fsyncFile(path); err != nil { return err @@ -217,8 +226,9 @@ func BarrierNewFile(path string) error { return FsyncDir(filepath.Dir(parent)) } -// DeepestExistingDir returns the deepest on-disk ancestor of path (path itself if -// it exists), bounding FsyncNewDirs to only the dirs a subsequent MkdirAll creates. +// DeepestExistingDir returns the deepest ancestor of path (path itself when it +// already exists) present on disk, walking up until a stat succeeds. It bounds +// FsyncNewDirs to only the directories a subsequent MkdirAll actually creates. func DeepestExistingDir(path string) string { for { if _, err := os.Stat(path); err == nil { @@ -232,14 +242,16 @@ func DeepestExistingDir(path string) string { } } -// FsyncNewDirs makes a dir chain freshly produced by MkdirAll durable. MkdirAll -// fsyncs neither the new dirs nor their direntries, so on a fresh deployment a -// crash can lose a whole storage subtree while the synced catalog advertises a -// "frozen" artifact under it (BarrierNewFile's grandparent fsync reaches a root's -// CONTENTS, never the root's own link). Given existingAncestor (from -// DeepestExistingDir before the MkdirAll), this fsyncs createdLeaf up to and -// including it. When nothing was created it costs one harmless fsync. Run once -// per root at startup. +// FsyncNewDirs makes a directory chain freshly produced by MkdirAll durable. +// MkdirAll fsyncs neither the new directories nor the direntries naming them, so +// on a fresh deployment a crash can lose a whole storage subtree while the synced +// catalog still advertises a "frozen" artifact under it — BarrierNewFile's +// grandparent fsync reaches a storage root's CONTENTS, never the root's own link +// in its parent. Given existingAncestor (the deepest dir that already existed, +// from DeepestExistingDir before the MkdirAll), this fsyncs createdLeaf and every +// ancestor up to and including existingAncestor, persisting each new dirent. When +// nothing was created (existingAncestor == createdLeaf) it costs one harmless dir +// fsync. Run once per root at startup. func FsyncNewDirs(existingAncestor, createdLeaf string) error { for d := createdLeaf; ; d = filepath.Dir(d) { if err := FsyncDir(d); err != nil { diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go b/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go index 126f1ac61..5667214d9 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go @@ -1,44 +1,67 @@ -// Package ingest drives full-history ingestion: it pulls raw ledgers from an -// injected stream and writes the three data types (ledgers, txhashes, contract -// events) into the full-history stores, one chunk at a time, via the SDK's -// zero-copy view extractors and events.LCMViewToPayloads. +// Package ingest drives full-history ingestion: it pulls raw ledgers +// (wire-format xdr.LedgerCloseMeta bytes) from an injected ledger stream +// and writes the three data types — ledgers, txhashes, contract events — +// into the full-history stores, one chunk at a time, via the zero-copy +// view extractors in the go-stellar-sdk ingest package and the RPC-side +// events.LCMViewToPayloads emitter. // -// Two tiers share the per-ledger extraction: +// Two tiers share the per-ledger extraction but differ in everything +// else: // -// - Hot (HotService): one ledger into the per-chunk shared multi-CF hot DB -// (hotchunk.DB), committed as ONE atomic synced WriteBatch across all enabled -// CFs — a ledger is fully present or fully absent (decision (a)), no fan-out. -// - Cold (WriteColdChunk): one chunk into per-chunk cold artifacts (ledger -// .pack, txhash .bin, events pack+index). SOURCE-BLIND — the caller resolves -// the ledger source and passes the raw iterator, so the materializer never -// learns whether the bytes came from a local .pack or the bulk backend. Each -// cold ingester opens its own writer; Finalize publishes, Close drops -// partials on failure (ColdService orchestrates). +// - Hot (RunHot): one chunk into the long-lived, caller-owned hot +// stores, from an injected ledgerbackend.LedgerStream. The stores +// are INJECTED and never opened or closed here, and neither is the +// stream; each ledger is durable before the next is pulled. +// Per-ledger fan-out across the enabled ingesters is concurrent +// (HotService). +// - Cold (WriteColdChunk): one chunk into per-chunk cold artifacts +// (ledger .pack, txhash .bin, events pack+index). It is +// SOURCE-BLIND — the caller resolves the chunk's ledger source and +// passes the raw ledger iterator, so the materializer never learns +// whether the bytes came from a local .pack or the bulk backend. +// Each cold ingester OPENS its own per-chunk writer; Finalize +// publishes the artifact and Close drops partials on the failure +// path (ColdService orchestrates). // -// Artifact model (cold) — the contract every layer relies on: +// Artifact model (cold) — the contract every layer here relies on: // -// - Cold artifacts are NOT authoritative alone. The orchestrator's completion -// record — written only after every Finalize returned and its data is durable -// — is the single source of truth for whether a chunk exists. -// - Nothing consumes cold artifacts by scanning directories; consumers take +// - Cold artifacts are NOT authoritative on their own. The +// orchestrator's completion record — written only after every +// enabled ingester's Finalize returned and its data is durable +// (writers fsync before reporting success) — is the single source +// of truth for whether a chunk exists. +// - Nothing may consume cold artifacts by scanning directories. A +// consumer (the serving tier, the deferred index build) takes // explicit paths composed from the completion record. -// - A chunk attempt owns its paths exclusively and overwrites freely. Disk is -// scratch until the completion record says otherwise: partial/stale files -// from a failed attempt are inert and the retry's overwrite is the cleanup — -// so no writer needs tmp+rename, no pre-clean, no rollback. +// - A chunk attempt owns its chunk's paths exclusively and +// overwrites freely. Disk under the cold roots is scratch until the +// completion record says otherwise: stale or partial files from a +// failed or crashed attempt are inert, and the retry's overwrite +// is the cleanup. No writer needs tmp+rename atomicity, no +// constructor needs to pre-clean, and no failure path needs to +// roll committed siblings back. // -// Failure semantics (cold) follow: a chunk fully finalizes (then the orchestrator -// records completion) or is abandoned and re-run from scratch — no mid-chunk -// resume. A failed Ingest releases via Close (Finalize must not run); a failed -// Finalize stops ColdService at the first error. +// Failure semantics (cold) follow from the model: a chunk either fully +// finalizes — and only then may the orchestrator record completion — or +// the attempt is abandoned and re-run from scratch; there is no +// mid-chunk resume. Once any Ingest fails, the chunk is released via +// Close (Finalize must not run; see the ColdIngester contract). Once +// any Finalize fails, ColdService stops at the first error; whatever +// the earlier ingesters already wrote stays on disk as inert scratch. // -// Types are processed in canonical ledgers→txhash→events order (buildColdIngesters -// is the single definition site). On-disk formats/filenames are owned by the store -// packages; this package only composes the {bucketID:05d}/ bucket dirs. +// Data types are processed in canonical ledgers→txhash→events order; +// the constructor table in buildColdIngesters is the order's single +// definition site. The on-disk formats and per-chunk filenames are +// owned by the store packages (ledger.PackName, txhash.ColdBinName + +// its .bin codec, eventstore's cold-format helpers); this package only +// composes the {bucketID:05d}/ bucket directories around them. // -// Inputs are borrowed: every Ingest gets a view valid only until the next pull, -// and each ingester copies what it retains. The raw iterator yields an error on -// ctx cancellation, so the drain loop needn't poll ctx. Metrics flow through -// MetricSink; the cold invariant is exactly one ColdChunkTotal per chunk attempt, -// including pre-service failures. +// Inputs are borrowed: every Ingest receives a view over the source +// stream's buffer, valid only until the next ledger is pulled, and +// each ingester copies what it retains (see HotIngester). The raw +// ledger iterator's contract includes yielding an error on ctx +// cancellation — the drain loop relies on it for cancellation rather +// than polling ctx itself. Metrics flow through MetricSink (Prometheus in prod, +// recorders in tests); the cold tier's invariant is exactly one +// ColdChunkTotal per chunk attempt, including pre-service failures. package ingest diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go index a38b747b6..cc7820d9b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go @@ -19,15 +19,19 @@ import ( // that ingested nothing. var errColdBuildAborted = errors.New("ingest: cold ingester build aborted (sibling constructor failed)") -// coldAborter lets the rollback path mark an ingester's metric aborted before -// Close emits it. Optional — a non-implementer just gets its normal emission. +// coldAborter is implemented by the concrete cold ingesters so the +// constructor-rollback path can mark their per-chunk metric as aborted before +// Close emits it, turning what would be a phantom success into a recorded +// abort. Optional: an ingester that does not implement it just gets its normal +// Close emission. type coldAborter interface { abortMetric(err error) } -// closeColdAll closes every ingester built so far (joining errors), first marking -// each aborted so the deferred Close emit is not a phantom success. Used when a -// LATER constructor fails mid-build. +// closeColdAll closes every cold ingester built so far, joining each Close error +// into err. Used when a LATER constructor fails mid-build: the already-built +// ingesters never ingested anything, so each one's metric is first marked +// aborted (so the deferred Close emit is not a phantom success). func closeColdAll(ings []ColdIngester, err error) error { for _, ing := range ings { if a, ok := ing.(coldAborter); ok { @@ -51,18 +55,21 @@ func drain(ctx context.Context, ledgers iter.Seq2[[]byte, error], chunkID chunk. if serr != nil { return fmt.Errorf("RawLedgers(%d): %w", seq, serr) } - // Reject an overrun before ingesting it: without this, the post-loop - // count check would only trip AFTER the extra ledgers were durably - // written (the ledger/txhash hot stores accept any seq). Guards custom - // iterators; in-repo sources self-bound. + // Reject a stream that runs PAST the chunk before ingesting anything + // out-of-chunk. Without this, an in-order overrun would only trip the + // post-loop count check after the extra ledgers were durably ingested + // (the ledger and txhash hot stores accept any sequence). All in-repo + // sources bound themselves; this guards custom iterators. if seq > last { return fmt.Errorf("ingest: stream for chunk %d yielded a ledger past %d (chunk overrun)", uint32(chunkID), last) } lcm := xdr.LedgerCloseMetaView(raw) - // Validate the actual seq before ingesting: the count check only catches - // short/long streams, so a duplicate or out-of-order ledger with the - // right total count would otherwise pass silently. + // Validate the actual ledger sequence before ingesting. The final + // count check below only catches a short/long stream; a source that + // yields a duplicate or out-of-order ledger with the right total + // count would otherwise pass silently (e.g. on the txhash and + // ledger-hot paths, which key on the LCM's own seq). actual, aerr := lcm.LedgerSequence() if aerr != nil { return fmt.Errorf("ingest: stream for chunk %d: ledger sequence at expected %d: %w", @@ -72,8 +79,8 @@ func drain(ctx context.Context, ledgers iter.Seq2[[]byte, error], chunkID chunk. return fmt.Errorf("ingest: stream for chunk %d yielded ledger %d, expected %d", uint32(chunkID), actual, seq) } - // seq is now VALIDATED as lcm's sequence — pass it through so ingesters - // needn't each re-derive it. + // seq is now VALIDATED as lcm's sequence — pass it through so the + // ingesters consume it instead of each re-deriving it from the view. if err := ing.Ingest(ctx, seq, lcm); err != nil { return err } @@ -123,16 +130,20 @@ func buildColdIngesters(dirs ColdDirs, chunkID chunk.ID, sink MetricSink, cfg Co return ings, nil } -// WriteColdChunk materializes ONE chunk's cold artifacts into dirs in a single -// pass from the raw ledger iterator. SOURCE-BLIND: the caller resolves the ledger -// source (local .pack or bulk backend) and hands its iterator here, so the -// materializer never learns where the bytes came from. Ingesters overwrite any -// crashed partial (the freeze protocol's re-materialization); on failure the -// attempt is abandoned, leftover files inert (package doc's artifact model). +// WriteColdChunk materializes ONE chunk's cold artifacts into the roots named by +// dirs, in a single pass, from the already-opened raw ledger iterator. It is +// SOURCE-BLIND: the caller (backfill) resolves the chunk's ledger source — the +// local frozen .pack or the bulk backend — and hands its RawLedgers iterator here, +// so the cold materializer never learns where the bytes came from and is faked in +// tests with a literal slice iterator. The ingesters overwrite any crashed +// partial, so this is the freeze protocol's re-materialization. On any failure the +// attempt is abandoned — leftover files are inert scratch (see the package doc's +// artifact model) and a retry's overwrite is the cleanup. // // Source resolution (pack-stat, coverage wait) runs in the caller BEFORE this, so -// the only pre-service failures left to meter here are a canceled ctx and a -// constructor failure. +// a pack-missing or coverage-timeout failure is metered there rather than as a +// ColdChunkTotal attempt here. The only pre-service failures left to meter here +// are a canceled ctx and a cold-ingester constructor failure. func WriteColdChunk( ctx context.Context, logger *supportlog.Entry, @@ -147,8 +158,10 @@ func WriteColdChunk( } sink = orNop(sink) - // Pre-service failures emit the chunk's single ColdChunkTotal here (the owning - // ColdService isn't built yet) — invariant: one ColdChunkTotal per attempt. + // Pre-service failures (ctx and the constructor failure below) emit the + // chunk's single ColdChunkTotal here: the ColdService that normally owns that + // aggregate isn't built yet, but the invariant is "exactly one ColdChunkTotal + // per chunk attempt, including failures." start := time.Now() if cerr := ctx.Err(); cerr != nil { sink.ColdChunkTotal(time.Since(start)) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go index 801db2b88..d59453293 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go @@ -8,31 +8,45 @@ import ( // HotIngester ingests one data type for one ledger into a long-lived hot store. // -// Ownership: the store is INJECTED and caller-owned (the daemon); the ingester -// does NOT open or close it — Close is intentionally absent. +// Ownership: the hot store is INJECTED into the ingester's constructor and owned +// by the caller (the daemon). The ingester does NOT open the store and does NOT +// close it — Close is intentionally absent from this interface. // -// Input: seq is the DRIVER-VALIDATED sequence of lcm (drain already checked it -// against the chunk's expected position), so ingesters consume it directly. lcm -// is a zero-copy view over the source's BORROWED buffer, valid only this step — -// an ingester must copy what it retains. The view is consumed synchronously -// within Ingest, so it is never read past its lifetime. +// Input: seq is the DRIVER-VALIDATED ledger sequence of lcm — the drain loop +// has already read it off the view and checked it against the chunk's expected +// position (duplicate / out-of-order / overrun), so ingesters consume it +// directly instead of each re-deriving and re-error-handling it. lcm is a +// zero-copy xdr.LedgerCloseMetaView (a []byte alias over the source stream's +// BORROWED buffer), valid only for the current iteration step; an ingester +// must copy any bytes it retains. The hot fan-out (HotService) waits for all +// ingesters to finish a ledger before the source pulls the next one, so +// synchronous consumption inside Ingest is safe. +// +// Concurrency: distinct HotIngester instances are run concurrently for the same +// ledger (HotService fans out via errgroup); each instance touches only its own +// store plus the read-only view. type HotIngester interface { Ingest(ctx context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error } // ColdIngester ingests one data type for one chunk into a per-chunk cold writer. // -// Ownership: the ingester OPENS its own writer and owns its lifecycle. Finalize -// commits the artifact (explicit, error-checked); Close is deferred + idempotent -// and drops any partial on the failure path. +// Ownership: the ingester OPENS its own per-chunk writer in its constructor and +// owns its lifecycle. Finalize commits the chunk's artifact (explicit, +// error-checked, never deferred). Close is always deferred and idempotent; on +// the failure path (Finalize never ran) it drops any partial file. // -// Contract: Finalize must NOT be called after a failed Ingest — the chunk is -// abandoned via Close and retried from scratch. Partial per-ledger state may -// already be committed, so a post-failure Finalize could publish an inconsistent -// artifact; implementations should latch the failure and refuse (eventsCold does). +// Contract: Finalize must NOT be called after a failed Ingest — once any +// Ingest errors, the chunk is abandoned via Close and retried from scratch. +// Implementations may have committed partial per-ledger state before the +// error (e.g. the events ingester's mirror/pack run ahead of its offsets +// commit point), so a post-failure Finalize could publish an inconsistent +// artifact; implementations are encouraged to latch the failure and refuse +// (eventsCold does). // -// Input: same driver-validated-seq + borrowed-view contract as HotIngester; -// ColdService drives Ingest sequentially. +// Input: same driver-validated-seq and borrowed-view contract as HotIngester. +// ColdService drives the per-ledger Ingest calls sequentially, so each view is +// fully consumed before the next. type ColdIngester interface { Ingest(ctx context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error Finalize(ctx context.Context) error diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go index 00205af05..0b7b09f07 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go @@ -77,10 +77,14 @@ func itemsOnSuccess(n int, err error) int { } // ColdService drives a set of ColdIngesters for one chunk: sequential per-ledger -// Ingest, then Finalize on each. It emits the aggregate ColdChunkTotal exactly -// once — in Finalize on success, else in Close on failure; totalEmitted prevents -// the double-emit. (Pre-service ctx/constructor failures are metered directly by -// WriteColdChunk.) +// Ingest, then Finalize on each. It times from the first Ingest (or, if none ran, +// from the Finalize/Close call) and emits the aggregate ColdChunkTotal exactly +// once for the chunk — in Finalize on the success path, otherwise in Close on the +// failure path (an Ingest error or short stream short-circuits before Finalize). +// The totalEmitted flag prevents a double-emit: Finalize sets it so the caller's +// deferred Close is a no-op for the aggregate. (A ctx or constructor failure +// happens before the service is built — WriteColdChunk emits that chunk's single +// ColdChunkTotal directly.) type ColdService struct { ingesters []ColdIngester sink MetricSink @@ -88,14 +92,18 @@ type ColdService struct { totalEmitted bool } -// NewColdService builds a ColdService over the enabled cold ingesters (nil sink -// → NopSink). The per-chunk aggregate timer starts here. +// NewColdService builds a ColdService over the enabled cold ingesters. A nil +// sink defaults to NopSink. The per-chunk aggregate timer starts here; the only +// case where no Ingest follows is an already-errored short/empty stream, where +// the timing sample is meaningless anyway. func NewColdService(ingesters []ColdIngester, sink MetricSink) *ColdService { return &ColdService{ingesters: ingesters, sink: orNop(sink), start: time.Now()} } // Ingest runs every cold ingester on lcm sequentially (each owns mutable -// per-chunk state). The first error aborts the ledger. +// per-chunk state, so no concurrency within the service). seq is the +// driver-validated sequence of lcm, passed through unchanged. The first error +// aborts the ledger. func (s *ColdService) Ingest(ctx context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { for _, ing := range s.ingesters { if err := ing.Ingest(ctx, seq, lcm); err != nil { @@ -105,11 +113,14 @@ func (s *ColdService) Ingest(ctx context.Context, seq uint32, lcm xdr.LedgerClos return nil } -// Finalize commits each cold ingester's chunk artifact (explicit, error-checked). -// The first error STOPS the loop: unfinalized ingesters are released by the -// caller's deferred Close, and the failed attempt is never recorded complete by -// the orchestrator — earlier-written artifacts stay as inert scratch (package -// doc's artifact model). Emits ColdChunkTotal here on the success path. +// Finalize commits each cold ingester's chunk artifact (explicit, error-checked, +// never deferred). The first Finalize error STOPS the loop: the remaining +// (unfinalized) ingesters are released by the caller's deferred Close, and the +// failed chunk attempt is reported to the orchestrator, which never records +// completion for it. Artifacts the earlier ingesters already wrote are left in +// place — without the orchestrator's completion record they are inert scratch +// (see the package doc's artifact model), and the retry's overwrite is the +// cleanup. The per-chunk ColdChunkTotal is emitted here on the success path. func (s *ColdService) Finalize(ctx context.Context) error { var ferr error for _, ing := range s.ingesters { @@ -122,11 +133,12 @@ func (s *ColdService) Finalize(ctx context.Context) error { return ferr } -// Close closes every cold ingester (joining errors) and emits ColdChunkTotal if -// Finalize never reached it (failure path). Each ingester's Close in turn emits -// its own ColdIngest if its Finalize never ran, so a failed chunk still produces -// one per-ingester signal + one aggregate. Idempotent: on failure a writer's -// Close drops its partial; after a successful Finalize all emissions are no-ops. +// Close closes every cold ingester, joining each Close error, and emits the +// aggregate ColdChunkTotal if Finalize never reached it (the failure path). Each +// ingester's own Close in turn emits that ingester's per-chunk ColdIngest if its +// Finalize never ran, so a failed chunk still produces one per-ingester signal +// and one aggregate. Idempotent: on the failure path a writer's Close drops its +// partial file; after a successful Finalize all emissions are no-ops. func (s *ColdService) Close() error { var err error for _, ing := range s.ingesters { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index 60292340b..8b2031cc9 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -50,13 +50,18 @@ func RemoveHotChunkDir(dataDir string, chunkID chunk.ID) error { return os.RemoveAll(HotChunkDir(dataDir, chunkID)) } -// Per-CF tuning passed via rocksdb.Config.PerCFOptions: +// Per-CF tuning for the hot store, passed via rocksdb.Config.PerCFOptions: // -// - DataCF holds XDR payloads (compressible, batch-read) — larger blocks give -// zstd more context and align with batch-fetch shapes. -// - IndexCF holds 20-byte keys / empty values — small blocks cut wasted I/O per -// random Lookup (one block per key). -// - OffsetsCF holds 8-byte rows — same shape as IndexCF. +// - DataCF holds XDR-encoded event payloads: compressible (zstd +// typically 2-3× on XDR) and read in batches via +// BatchedMultiGetCF. Larger blocks give zstd more context per +// compression unit and align with batch-fetch shapes. +// - IndexCF stores 20-byte (term_hash || event_id) keys with +// empty values — nothing in the values to compress, and small +// blocks reduce wasted I/O per random Lookup miss (each Lookup +// reads one block to find one key). +// - OffsetsCF stores 8-byte (ledger_seq -> event_count) rows in +// the tens-of-thousands per chunk — same shape as IndexCF. const ( dataCFBlockSize = 32 * 1024 indexCFBlockSize = 4 * 1024 @@ -147,8 +152,10 @@ type HotStore struct { // Compile-time guard: *HotStore satisfies Reader. var _ Reader = (*HotStore)(nil) -// OpenHotStore opens (or creates) chunkID's hot DB, warms up the mirror + -// offsets from disk, and returns a ready HotStore that owns its chunkStore. +// OpenHotStore opens (or creates) chunkID's hot DB at +// HotChunkDir(dataDir, chunkID), warms up the in-memory mirror and +// offsets from disk, and returns a ready-to-use HotStore. The +// returned store owns its chunkStore; Close releases it. func OpenHotStore( dataDir string, chunkID chunk.ID, @@ -207,9 +214,11 @@ func (h *HotStore) Close() error { // ChunkID returns the chunk this store serves. func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } -// EventCount is the total events committed to this Chunk so far (== the next -// event-id). Returns (0, ErrClosed) after Close. The fallible signature is for -// the Reader interface (ColdReader's lazy load); the hot value is always live. +// EventCount is the total number of events committed to this Chunk +// so far. Equal to the next event-id IngestLedgerEvents would assign. +// Returns (0, ErrClosed) after Close. The Reader interface signature +// is fallible to accommodate ColdReader's lazy metadata load; on the +// hot side the value is always live and the error is only ErrClosed. func (h *HotStore) EventCount() (uint32, error) { if h.chunkStore.IsClosed() { return 0, ErrClosed @@ -217,16 +226,30 @@ func (h *HotStore) EventCount() (uint32, error) { return h.offsets.TotalEvents(), nil } -// NextEventID is the next chunk-relative event ID IngestLedgerEvents will -// assign. Same value as EventCount; exposed under both names for the ingest- and -// reader-side mental models. Infallible (hot-only, not on Reader). +// NextEventID is the next chunk-relative event ID IngestLedgerEvents +// will assign. Returns the same value as EventCount on the hot side +// and is exposed under both names for the ingest-side and reader-side +// mental models. Infallible at the type level (hot-only API, not on +// the Reader interface). func (h *HotStore) NextEventID() uint32 { return h.offsets.TotalEvents() } -// Offsets returns a point-in-time, read-only view of the ledger-offset cache -// (see Reader.Offsets), capped at the count visible at call time. A concurrent -// ingest may extend the backing past the cap; the view's slice stays bounded. -// The view shares the live backing array — Append on it would silently fork it, -// so the contract is read-only. Returns (nil, ErrClosed) after Close. +// Offsets returns a point-in-time view of the ledger-offset cache. +// The coordinator uses this to stitch a multi-ledger query range +// into chunk-relative event-id ranges (see Reader.Offsets). +// +// Implementation: returns a *LedgerOffsets sharing the live +// backing array, capped at the count visible at call time +// (~24-byte allocation per Query). Concurrent IngestLedgerEvents +// may extend the backing past the cap, but the returned view's +// slice stays bounded to what was visible when Offsets returned. +// Callers (Query) take the view once at entry and pass it through +// their helpers. +// +// Read-only: the returned view's underlying slice shares memory +// with the live backing array. Calling Append on the view would +// silently fork it from the live data; the contract is read-only. +// +// Returns (nil, ErrClosed) after Close. func (h *HotStore) Offsets() (*events.LedgerOffsets, error) { if h.chunkStore.IsClosed() { return nil, ErrClosed @@ -234,15 +257,24 @@ func (h *HotStore) Offsets() (*events.LedgerOffsets, error) { return h.offsets.View(), nil } -// Index returns the in-memory term mirror, used by the freezer to snapshot every -// (TermKey, bitmap) pair without rebuilding from RocksDB. Callers typically use -// h.Index().Snapshot() for a uniquely owned Bitmaps. +// Index returns the in-memory term mirror. Used by the freezer to +// snapshot every (events.TermKey, bitmap) pair into WriteColdIndex +// without rebuilding from RocksDB. Callers should typically call +// h.Index().Snapshot() to get a uniquely owned Bitmaps for +// serialization. func (h *HotStore) Index() *events.ConcurrentBitmaps { return h.mirror } -// Lookup returns the bitmap of event IDs in this Chunk matching key. The bitmap -// is an immutable snapshot of the live mirror (writers publish via atomic.Store) -// — callers MUST NOT mutate it. Returns (nil, ErrTermNotFound) on no match, -// (nil, ErrClosed) after Close. ctx is a fast guard; the hot path does no I/O. +// Lookup returns the bitmap of event IDs in this Chunk that match +// the given term. The returned bitmap is an immutable snapshot of +// the live mirror — writers publish new pointers via atomic.Store +// (see ConcurrentBitmaps), so the caller never observes a mutating +// bitmap. Callers MUST NOT mutate it themselves. See Reader.Lookup +// and ConcurrentBitmaps.Get for the full contract. Returns +// (nil, ErrTermNotFound) when the term has no matching events. +// Returns (nil, ErrClosed) after Close. +// +// ctx is checked as a fast guard but the hot path does no blocking +// I/O — the bitmap comes from the in-memory mirror. func (h *HotStore) Lookup(ctx context.Context, key events.TermKey) (*roaring.Bitmap, error) { if h.chunkStore.IsClosed() { return nil, ErrClosed @@ -260,9 +292,15 @@ func (h *HotStore) Lookup(ctx context.Context, key events.TermKey) (*roaring.Bit return bm, nil } -// LookupKeys returns bitmaps positionally aligned with keys; result[i] is nil on -// no match. See Reader.LookupKeys (borrowed-bitmap contract: callers must not -// mutate). Hot-side is just N mirror lookups, exposed only to satisfy Reader. +// LookupKeys returns bitmaps for each key, aligned positionally with +// the input slice. result[i] is nil if keys[i] has no matching +// events. See Reader.LookupKeys for the semantics — in particular +// the borrowed-bitmap contract (callers must not mutate). +// +// Hot-side implementation is N in-memory mirror lookups — no I/O +// to batch — but exposing this method satisfies the Reader +// interface so callers can program against batched lookups +// uniformly. func (h *HotStore) LookupKeys(ctx context.Context, keys []events.TermKey) ([]*roaring.Bitmap, error) { if h.chunkStore.IsClosed() { return nil, ErrClosed @@ -284,15 +322,26 @@ func (h *HotStore) LookupKeys(ctx context.Context, keys []events.TermKey) ([]*ro return results, nil } -// FetchEvents decodes the events_data row for each eventID, positionally aligned -// with the input. See Reader.FetchEvents for the sorted-input precondition -// (violations return wrapped ErrUnsortedEventIDs). One BatchMultiGet crosses CGO -// once regardless of count and enables async_io (a win on high-latency storage); -// ctx is honored at entry but the CGO call is not cancellable mid-flight. +// FetchEvents decodes the events_data row for each provided eventID +// and returns them positionally aligned with the input slice. See +// Reader.FetchEvents for the sorted-input precondition. +// +// Implementation: validates eventIDs are sorted ascending with no +// duplicates (returns wrapped ErrUnsortedEventIDs otherwise — same +// shape as the cold side), encodes them to BE-uint32 keys, then +// calls rocksdb.Store.BatchMultiGet once with sortedInput=true. +// The batched API crosses CGO a single time regardless of key count +// and enables async_io so the kernel can overlap SST page reads — +// a meaningful win on EBS / high-random-latency storage. ctx is +// honored at the top of the call; the underlying CGO call is not +// cancellable mid-flight. // -// A missing row is an error, not a normal miss: IDs only reach here via Lookup, -// which only returns IDs the mirror (hence RocksDB) has — a miss means -// corruption. Returns ErrClosed after Close. +// A missing row is an error: eventIDs only reach this path through +// Lookup, which only returns IDs the mirror knows about — implying +// RocksDB also has them. A miss indicates corruption or a +// writer/reader mismatch, not a normal not-found case. +// +// After Close, returns ErrClosed. func (h *HotStore) FetchEvents(ctx context.Context, eventIDs []uint32) ([]events.Payload, error) { if h.chunkStore.IsClosed() { return nil, ErrClosed @@ -329,8 +378,10 @@ func (h *HotStore) FetchEvents(ctx context.Context, eventIDs []uint32) ([]events if v == nil { return nil, fmt.Errorf("events: event %d missing from chunk %s", id, h.chunkID) } - // BatchMultiGet already copies out of rocksdb's pinned pages; v is - // Go-owned, so Unmarshal's alias is safe without an extra clone. + // BatchMultiGet already copies out of rocksdb's pinned pages + // (see rocksdb.Store.BatchMultiGet); v is Go-owned and outlives + // the returned Payload, so Unmarshal's alias is safe without + // an extra clone. if err := results[i].Unmarshal(v); err != nil { return nil, fmt.Errorf("events: decode event %d from chunk %s: %w", id, h.chunkID, err) } @@ -338,15 +389,27 @@ func (h *HotStore) FetchEvents(ctx context.Context, eventIDs []uint32) ([]events return results, nil } -// FetchRange streams count events from chunk-relative event ID start, ascending -// (see Reader.FetchRange). Yielded Payloads are borrowed: ContractEventBytes -// aliases the iteration buffer, valid only until the next step — clone to retain. -// After Close yields (zero, ErrClosed). ctx is checked at entry and between -// steps; IterateRange takes no ctx, so a slow Next can block past a cancel. +// FetchRange streams count events starting at chunk-relative event +// ID start, in ascending eventID order. See Reader.FetchRange for +// semantics; the hot path drives rocksdb.Store.IterateRange over +// DataCF with start and end keys derived from encodeDataKey. +// +// Yielded Payloads are borrowed: ContractEventBytes aliases the iteration +// buffer and is valid only until the next step — clone to retain. +// +// After Close, yields (zero Payload, ErrClosed) and stops. +// ctx is checked at entry and between iterator steps — +// rocksdb.Store.IterateRange does not itself accept a ctx, so a +// very slow Next() can block past a cancellation until the next +// yielded entry observes the cancel. // -// Out-of-range args yield an error and stop: count==0 is a no-op; start+count > -// NextEventID is out-of-bounds; a short scan (fewer rows than count) signals -// corruption (the CF should be dense in [0, NextEventID)). +// Out-of-range arguments yield an error and stop: +// - count == 0 is a natural no-op (no yields). +// - start+count > NextEventID (overflow-safe via uint64) yields a +// wrapped out-of-bounds error. +// - A short scan (fewer DataCF rows than count) yields a wrapped +// error after the partial stream — the CF should be dense in +// [0, NextEventID), so a hole indicates corruption. func (h *HotStore) FetchRange(ctx context.Context, start, count uint32) iter.Seq2[events.Payload, error] { return func(yield func(events.Payload, error) bool) { if h.chunkStore.IsClosed() { @@ -378,8 +441,10 @@ func (h *HotStore) FetchRange(ctx context.Context, start, count uint32) iter.Seq return } var p events.Payload - // entry.Value is a zero-copy ref valid only this step; Unmarshal - // aliases it into p, so the yielded Payload is borrowed (clone to retain). + // entry.Value is a zero-copy ref into the IterateRange + // iterator buffer, valid only for this step; Unmarshal aliases + // it into p.ContractEventBytes, so the yielded Payload is + // borrowed (see the FetchRange doc). A retaining consumer clones. if err := p.Unmarshal(entry.Value); err != nil { yield(events.Payload{}, fmt.Errorf("events: decode event from chunk %s: %w", h.chunkID, err)) @@ -398,11 +463,16 @@ func (h *HotStore) FetchRange(ctx context.Context, start, count uint32) iter.Seq } } -// All streams every event in this Chunk in eventID order — used by the freeze -// loop to dump a hot Chunk without buffering. Thin wrapper over FetchRange (same -// borrowed-Payload contract). NextEventID is read inside the closure, so an -// ingest between All returning and the first range step is included. After Close -// yields (zero, ErrClosed). +// All streams every event in this Chunk in chunk-relative eventID +// order. Used by the freeze loop to dump a hot Chunk into a +// ColdWriter without buffering. Thin wrapper over FetchRange; its +// yielded Payloads are likewise borrowed (valid only for the step). +// +// NextEventID is read inside the returned closure body, so a +// concurrent ingest between r.All(ctx) returning the Seq2 and the +// consumer's first range step is included in the snapshot. +// +// After Close, yields (zero Payload, ErrClosed) and stops. func (h *HotStore) All(ctx context.Context) iter.Seq2[events.Payload, error] { return func(yield func(events.Payload, error) bool) { // FetchRange stops iterating after yielding an error; we @@ -415,22 +485,37 @@ func (h *HotStore) All(ctx context.Context) iter.Seq2[events.Payload, error] { } } -// IngestLedgerEvents commits one ledger's events to the chunk store atomically, -// then updates the in-memory mirrors. payloads (from LCMViewToPayloads) arrive in -// getEvents cursor order; write order here IS the cursor contract (event IDs are -// assigned by arrival position). Terms are derived internally via TermsForBytes. -// -// Sequence validation runs up front (before any write or mirror mutation), so a -// rejected call leaves all state untouched: -// - out of [chunkID.FirstLedger(), LastLedger()] → ErrLedgerOutOfRange. -// - == expected (StartLedger + LedgerCount) → appended. -// - < expected (already ingested) → idempotent no-op nil (a restarted ingester -// can blindly re-deliver; the re-delivered events are not re-verified). -// - > expected (a gap) → ErrLedgerOutOfOrder. -// -// Post-batch atomicity: once the batch commits, the mirror + offsets updates are -// infallible — a failure there panics rather than leaving on-disk state ahead of -// in-memory with no clean recovery short of close + reopen. +// IngestLedgerEvents commits one ledger's events to the chunk store +// atomically and then updates the in-memory mirrors. +// +// payloads is produced by events.LCMViewToPayloads, which emits each ledger's +// events in ascending getEvents cursor order — write order here IS the +// cursor contract (event IDs are assigned by arrival position). Terms are +// derived internally via events.TermsForBytes on each payload's +// ContractEventBytes. +// +// Sequence validation is performed up front, before any RocksDB +// write or mirror mutation: +// +// - ledgerSeq must lie within [chunkID.FirstLedger(), +// chunkID.LastLedger()] — out-of-range returns ErrLedgerOutOfRange. +// - ledgerSeq == the next expected ledger (StartLedger + LedgerCount) +// is appended normally. +// - ledgerSeq < expected (an already-ingested ledger) is an idempotent +// no-op returning nil, so a restarted ingester can blindly re-deliver +// the in-flight ledger; the re-delivered events are not re-verified. +// - ledgerSeq > expected (a gap) returns ErrLedgerOutOfOrder. +// +// A rejected call (out-of-range or gap) completes its checks before +// marshaling, leaving the chunk store and in-memory mirrors untouched. +// +// Post-batch atomicity: once the RocksDB batch commits, the in-memory +// mirror + offsets updates are infallible by construction. Any +// failure there panics rather than returning an error, because a +// returned error would leave on-disk state ahead of in-memory state +// with no clean recovery short of close + reopen. +// +//nolint:cyclop // sequential pipeline: validate -> marshal -> batch -> mirror updates func (h *HotStore) IngestLedgerEvents(ledgerSeq uint32, payloads []events.Payload) error { if h.chunkStore.IsClosed() { return ErrClosed @@ -545,9 +630,11 @@ func (h *HotStore) prepareLedger(ledgerSeq uint32, payloads []events.Payload) (* ErrLedgerOutOfOrder, expected, ledgerSeq) } - // Pre-derive term keys so the post-commit mirror update needn't re-hash. A - // TermsForBytes error rejects the ledger without touching the batch (a decode - // failure on core-validated XDR is a corruption signal worth aborting on). + // Pre-derive term keys per payload so the post-commit mirror + // update doesn't re-hash. Surfacing TermsForBytes errors here + // (pre-batch) cleanly rejects the ledger commit without touching disk — + // a decode failure on stellar-core-validated XDR is a corruption + // signal worth aborting on. termKeys := make([][]events.TermKey, len(payloads)) for i := range payloads { keys, err := events.TermsForBytes(payloads[i].ContractEventBytes) @@ -612,12 +699,24 @@ func (h *HotStore) applyLedger(p *preparedLedger) { h.offsets.Append(uint32(len(p.blobs))) } -// Warmup — reconstructs the in-memory mirror + offsets from the on-disk CFs. +// ────────────────────────────────────────────────────────────────── +// Warmup — reconstructs the in-memory mirror + offsets from the +// per-Chunk DB's on-disk CFs. Called only by OpenHotStore. +// ────────────────────────────────────────────────────────────────── -// warmup rebuilds chunkID's in-memory mirrors by scanning the two on-disk caches -// once each: events_index → ConcurrentBitmaps, events_offsets → -// ConcurrentLedgerOffsets. chunkID seeds StartLedger for empty chunks; both -// mirrors are empty for fresh chunks. +// warmup rebuilds the in-memory mirrors for chunkID by prefix-scanning +// the chunk's two on-disk caches once each: +// +// - events_index → *events.ConcurrentBitmaps — every +// (events.TermKey, eventID) row replayed into a fresh in-memory +// bitmap mirror. +// - events_offsets → *events.ConcurrentLedgerOffsets — every +// (ledger_seq, per_ledger_count) row replayed into a fresh +// offset cache. +// +// chunkID seeds events.ConcurrentLedgerOffsets.StartLedger for empty +// chunks; on-disk rows carry the full ledger sequence themselves. +// Both mirrors are empty for fresh chunks. func warmup( chunkStore *rocksdb.Store, chunkID chunk.ID, ) (*events.ConcurrentBitmaps, *events.ConcurrentLedgerOffsets, error) { @@ -635,19 +734,25 @@ func warmup( return mirror, offsets, nil } -// verifyChunkConsistency cross-checks the three on-disk CFs after warmup, turning -// a torn/tampered chunk into a loud open failure. A cheap open-time tripwire, not -// load-bearing correctness (the atomic batch makes violations impossible for the -// writer; only a bug/corruption trips it): +// verifyChunkConsistency cross-checks the three on-disk CFs after warmup, +// turning a torn or tampered chunk into a loud open failure instead of a +// silently inconsistent in-memory cache. The CFs are written in one +// atomic batch, so under normal operation these invariants always hold; +// a violation means a bug or external corruption. // -// - index may not reference an event offsets don't account for: -// indexUpperBound (max indexed id + 1, 0 if none) <= total. -// - data tail matches total: id total-1 present (when total > 0) and nothing at -// id >= total — together pinning the max data id to total-1. +// - the index may not reference an event the offsets don't account for: +// indexUpperBound (max indexed event ID + 1, 0 if none) <= total. +// - the data tail matches total: event total-1 present (when total > 0) +// and no data row at any id >= total. Together those pin the max data +// id to exactly total-1 — one Get plus one bounded seek. // -// Not detected (would need a full scan): interior data holes, under-indexed -// terms, wrong per-ledger boundaries. An interior hole that did appear is caught -// lazily by FetchRange's short-scan check. +// Not detected here: interior data holes (a missing id within 0..total-2, +// masked by a higher present id), under-indexed terms, and wrong +// per-ledger boundaries — each would need a full scan. The atomic batch +// makes all of them impossible for the writer; an interior hole that did +// appear (corruption/tamper) is caught lazily by FetchRange's short-scan +// check on first read. This is a cheap open-time tripwire on denormalized +// state, not load-bearing correctness. func verifyChunkConsistency(chunkStore *rocksdb.Store, total, indexUpperBound uint32) error { if indexUpperBound > total { return fmt.Errorf("events: corrupt chunk: index references event %d but only %d committed", @@ -663,8 +768,9 @@ func verifyChunkConsistency(chunkStore *rocksdb.Store, total, indexUpperBound ui total, total-1) } } - // Nothing may live at or beyond total. Reaching the loop body (no iteration - // error) means an orphan row is present. + // Nothing may live at or beyond total. The bounded seek lands on the + // first such row if one exists; reaching the loop body at all (with no + // iteration error) means an orphan is present — at total or far past it. for _, err := range chunkStore.IterateRange(DataCF, encodeDataKey(total), nil) { if err != nil { return fmt.Errorf("events: verify data tail: %w", err) @@ -674,13 +780,22 @@ func verifyChunkConsistency(chunkStore *rocksdb.Store, total, indexUpperBound ui return nil } -// warmupIndex replays every (TermKey, eventID) row of events_index into a fresh -// ConcurrentBitmaps. Builds into a single-threaded Bitmaps via per-term batching -// (byte-sorted iteration groups a term's rows, flushed on term change), then -// converts at the end — avoiding the per-row Clone the concurrent AddTo would do -// for popular terms (a 10M-event chunk would otherwise do ~50M Clones, saturating -// GC). Also returns the exclusive upper bound of indexed IDs (max + 1, 0 if -// empty) for warmup's cross-check against the committed count. +// warmupIndex scans the events_index CF and replays every +// (events.TermKey, eventID) row into a fresh events.ConcurrentBitmaps. +// Design doc §12 step 3. +// +// Implementation: build into a single-threaded events.Bitmaps via +// per-term batching (rocksdb's byte-sorted iteration delivers all +// rows for term K consecutively, so a small buffer flushes when the +// term changes), then convert to ConcurrentBitmaps at the end. This +// avoids paying the per-row Clone cost the concurrent ConcurrentBitmaps.AddTo +// would do for popular terms — without batching, warmup of a +// 10M-event chunk does ~50M Clones (one per index row) and saturates +// GC for many minutes. +// +// Also returns the exclusive upper bound of indexed event IDs (max + 1, +// or 0 if the index is empty) so warmup can cross-check it against the +// committed event count. func warmupIndex(chunkStore *rocksdb.Store) (*events.ConcurrentBitmaps, uint32, error) { builder := events.NewBitmaps() var ( @@ -723,11 +838,18 @@ func warmupIndex(chunkStore *rocksdb.Store) (*events.ConcurrentBitmaps, uint32, return events.NewConcurrentBitmapsFromBitmaps(builder), indexUpperBound, nil } -// warmupOffsets replays every (ledger_seq, event_count) row of events_offsets -// into a fresh ConcurrentLedgerOffsets. The on-disk shape (per-ledger counts) -// matches Append's input directly. BE-uint32 keys sort in ledger order; on-disk -// rows are untrusted, so each is validated as the next in-chunk ledger before the -// positional Append (the trust boundary moved here — Append no longer checks). +// warmupOffsets scans events_offsets and replays every (ledger_seq, +// event_count) row into a fresh *events.ConcurrentLedgerOffsets. The +// on-disk shape matches the in-memory Append input directly +// (per-ledger counts, not cumulative), so no delta arithmetic is +// needed. +// +// Iteration order is byte-sorted == numeric-sorted under the big-endian +// uint32 key encoding, so rows arrive in ledger order. On-disk rows are +// untrusted, so each is validated as the next in-chunk ledger before the +// positional Append — a gap or stray row is rejected here rather than +// silently mis-attributing counts (ConcurrentLedgerOffsets.Append no +// longer checks the sequence; the trust boundary is here). func warmupOffsets(chunkStore *rocksdb.Store, chunkID chunk.ID) (*events.ConcurrentLedgerOffsets, error) { offsets := events.NewConcurrentLedgerOffsets(chunkID.FirstLedger()) @@ -745,9 +867,10 @@ func warmupOffsets(chunkStore *rocksdb.Store, chunkID chunk.ID) (*events.Concurr } ledger := binary.BigEndian.Uint32(entry.Key) eventCount := binary.BigEndian.Uint32(entry.Value) - // Each row must be the next sequential in-chunk ledger: the first test - // catches a gap/out-of-order/wrong-start, the second an excess row past - // the chunk (which would append past capacity and panic). + // Each row must be the next sequential ledger and within the + // chunk. The first test catches a gap, an out-of-order row, or a + // wrong start; the second catches an excess row past the chunk + // (which would otherwise append past capacity and panic). if expected := offsets.EndLedger(); ledger != expected || ledger > chunkID.LastLedger() { return nil, fmt.Errorf("events: warmup offsets: chunk %s expected ledger %d, got %d", chunkID, expected, ledger) @@ -763,7 +886,9 @@ func warmupOffsets(chunkStore *rocksdb.Store, chunkID chunk.ID) (*events.Concurr return offsets, nil } +// ────────────────────────────────────────────────────────────────── // Key encoding helpers — RocksDB key layouts for the per-Chunk DB. +// ────────────────────────────────────────────────────────────────── func encodeDataKey(eventID uint32) []byte { var key [dataKeyLen]byte diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go index 790f37322..ce246d0d2 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go @@ -36,10 +36,13 @@ type Entry struct { // the ingest driver can reject a mismatched store. The store does not itself // range-check writes (the driver's drain loop already validates every sequence). // -// Concurrency: all methods, including Close, are safe for concurrent use. -// rocksdb.Store.Close drains in-flight ops before releasing; a racing read/write -// either completes first or returns stores.ErrStoreClosed. HotStore adds no -// unguarded state — the compressor pool and decompressor are concurrent-safe. +// Concurrency: all methods, including Close, are safe for concurrent +// use. rocksdb.Store.Close CAS-marks the store closed and then drains +// in-flight ops (each holds an RLock for its duration) before releasing +// resources; a read/write racing Close either completes first or +// observes the closed store and returns stores.ErrStoreClosed. Close is +// idempotent. HotStore adds no unguarded state of its own — the +// compressor pool and decompressor are both concurrent-safe. type HotStore struct { store *rocksdb.Store chunkID chunk.ID @@ -53,11 +56,19 @@ type HotStore struct { compPool sync.Pool } -// OpenHotStore validates inputs and returns an open HotStore bound to chunkID. -// path and logger are required (logger is forwarded to pkg/rocksdb). Rides on -// RocksDB defaults — no block cache, no bloom filter (callers only ask for -// sequences this store holds), no WAL cap (graceful Close flushes; ungraceful -// replay at this scale is sub-second). Re-tune only with a measurement. +// OpenHotStore validates inputs and returns an open HotStore bound +// to chunkID (see the HotStore doc on chunk binding). path and +// logger are both required; logger is forwarded to the +// pkg/rocksdb wrapper (rocksdb writes the on-open state line and +// the close-time Flush warning through it). HotStore itself does +// not emit any logs — the cold store, by contrast, takes no +// logger because packfile is silent. Rides on RocksDB defaults — +// no explicit block cache (RocksDB's per-CF default plus OS page +// cache cover range scans), no bloom filter (callers know in +// advance which sequences this store holds, so it is never asked +// for a key it doesn't have), no WAL cap (graceful Close flushes +// the memtable; ungraceful WAL replay at this scale is sub-second). +// Re-tune only with a workload measurement. func OpenHotStore(path string, chunkID chunk.ID, logger *supportlog.Entry) (*HotStore, error) { if path == "" { return nil, stores.ErrInvalidConfig @@ -107,8 +118,14 @@ func (h *HotStore) Close() error { // never reads the store). func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } -// AddLedgers compresses and writes (seq, raw-bytes) entries. Zero entries is a -// no-op; one uses Store.Put; multiple use one Store.Batch (one fsync, not N). +// AddLedgers writes (seq, raw-bytes) entries to rocksdb. Bytes is +// the uncompressed ledger payload; AddLedgers compresses each +// entry with zstd before write. Variadic so callers can pass +// individual entries (h.AddLedgers(e)), a literal batch +// (h.AddLedgers(e1, e2, e3)), or a slice (h.AddLedgers(entries...)). +// Zero entries is a no-op; one entry uses Store.Put; multiple +// entries use Store.Batch (one atomic write, one fsync — versus N +// fsyncs for N Put calls). func (h *HotStore) AddLedgers(entries ...Entry) error { if h.store.IsClosed() { return stores.ErrStoreClosed @@ -127,8 +144,10 @@ func (h *HotStore) AddLedgers(entries ...Entry) error { } return translateRocksErr(h.store.Put(LedgersCF, rocksdb.EncodeUint32(e.Seq), compressed)) } - // Compress each into its own fresh slice so the batch can hold them all at - // once (the compressor's internal buffer is overwritten on the next Encode). + // Multi-entry path: compress each into its own fresh slice so + // the batch can hold them all simultaneously (the compressor's + // internal buffer would otherwise be overwritten on the next + // Encode call). compressed := make([][]byte, len(entries)) for i, e := range entries { out, err := c.Encode(nil, e.Bytes) @@ -222,16 +241,19 @@ func (h *HotStore) IterateLedgers(start, end uint32) iter.Seq2[Entry, error] { if start > end { return } - // scratch is the reused decompression buffer; Entry.Bytes aliases it and - // is BORROWED — valid only until the next step decodes into it. Copy to - // retain past the loop body. Avoids a per-ledger decode allocation. + // scratch is the reused decompression buffer; Entry.Bytes aliases it + // and is therefore BORROWED — valid only until the next iteration step + // decodes the following ledger into it. Copy it if you need to retain + // it past the loop body. The read benches consume each ledger in-scope, + // so this avoids a per-ledger decode allocation. var scratch []byte for e, err := range h.store.IterateRange(LedgersCF, rocksdb.EncodeUint32(start), rocksdb.EncodeUint32(end)) { if err != nil { yield(Entry{}, translateRocksErr(err)) return } - // e.Value is a zero-copy ref into the iterator buffer; decode into scratch. + // e.Value is itself a zero-copy ref into the iterator's internal + // buffer; decompress it into the reused scratch buffer. seq := rocksdb.DecodeUint32(e.Key) decoded, derr := h.dec.Decode(scratch[:0], e.Value) if derr != nil { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go index 23faa2bd8..6d929eeab 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go @@ -31,12 +31,17 @@ type Entry struct { LedgerSeq uint32 } -// HotStore — RocksDB-backed hot transaction-hash store. 16 CFs cf-0..cf-f; each -// hash routes to cf-{txhash[0]>>4}; ledgerSeq BE-encoded (all internal). -// Chunk-bound like every hot store: accumulates one chunk's (txhash → seq) -// tuples before freezing, with the binding recorded at open time (ChunkID) so -// the ingest driver can reject a mismatched store. The store does not itself -// range-check writes (the driver's drain loop already validates every sequence). +// HotStore — RocksDB-backed hot transaction-hash store. 16 CFs named +// cf-0..cf-f; each hash routes to cf-{txhash[0]>>4}; ledgerSeq +// encoded big-endian. Routing, CF names, and encoding are internal. +// +// Like every hot store, a HotStore instance is chunk-bound: it +// accumulates exactly one chunk's (txhash → seq) tuples before being +// frozen into the chunk's cold .bin artifact. The binding is recorded +// at open time (ChunkID) so the ingest driver can reject a store +// bound to a different chunk than it is ingesting; the store does not +// itself range-check writes (the driver's drain loop already +// validates every ledger sequence against the chunk). type HotStore struct { store *rocksdb.Store chunkID chunk.ID @@ -93,44 +98,63 @@ func cfNameForTxHash(hash [32]byte) string { return cfNameByNibble[hash[0]>>4] } -// tuning — calibrated for the hot txhash workload (write-once / point-lookup over -// 16 CFs). Cross-knob interactions are non-obvious, hence the per-stanza WHY; -// the other facades ride on defaults. +// tuning — the hot txhash workload is write-once / point-lookup over +// 16 CFs; the cross-knob interactions below are non-obvious enough +// that they get an explicit per-stanza rationale. The other facades +// ride on RocksDB defaults by contrast — only this workload earned +// the calibration. func tuning() rocksdb.Tuning { return rocksdb.Tuning{ - // Per-CF memtable budget × 16 (64 MB × 16 = 1024 MB) matches - // MaxTotalWalSizeMB below, so memtable-fill and WAL-cap cadence align - // and each flush produces a ~64 MB SST. + // Per-CF memtable budget × 16 CFs (64 MB × 16 = 1024 MB) + // matches the MaxTotalWalSizeMB cap below. Memtable-fill + // cadence and WAL-cap cadence align under uniform writes; + // either trigger fires at roughly the same time and produces + // ~64 MB SSTs. WriteBufferMB: 64, MaxWriteBufferNumber: 2, - // Compaction off: write-once random-key data gains no reordering - // benefit. L0 999s match DisableAutoCompactions so even an over-trigger - // flush won't compact. (DisableAutoCompactions and MaxBackgroundJobs are - // orthogonal — off vs. thread budget.) + // L0 triggers pinned high + DisableAutoCompactions=true: + // compaction would re-write the same data with no reordering + // benefit (txhash is write-once, random-key, point-lookup). + // The L0 999s match DisableAutoCompactions so even if a future + // flush somehow exceeded the trigger, the engine still + // wouldn't try to compact. NOTE: DisableAutoCompactions and + // MaxBackgroundJobs are orthogonal — the former turns + // compaction off entirely, the latter only caps the thread + // budget for background work. Level0FileNumCompactionTrigger: 999, Level0SlowdownWritesTrigger: 999, Level0StopWritesTrigger: 999, DisableAutoCompactions: true, - // 64 MB target file matches WriteBufferMB (one flush → one SST). MaxBytes - // is a no-op under DisableAutoCompactions but set explicitly so a reader - // needn't derive that. + // 64 MB target file matches WriteBufferMB so one memtable + // flush produces one ~64 MB SST — fewer bloom checks per + // query at no-compaction scale. + // MaxBytesForLevelBaseMB is set explicitly even though it's + // irrelevant under DisableAutoCompactions (compaction never + // promotes past L0); explicit > implicit so a future reader + // doesn't have to derive that it's a no-op. TargetFileSizeMB: 64, MaxBytesForLevelBaseMB: 256, - // High background-job budget for periodic flushes across 16 CFs. + // High background-job budget for the periodic memtable + // flushes across 16 CFs. MaxBackgroundJobs: 8, MaxOpenFiles: 10_000, - // 512 MB block cache holds the hot working set (bloom-filter blocks). - // 12 bits/key (~0.4% FP) is tighter than the standard 10 because each FP - // costs a disk seek across many no-compaction SSTs. + // 512 MB block cache — bloom-filter blocks are the hot + // working set; the cache needs to hold recently-touched + // bloom blocks at scale. + // 12 bits/key bloom (~0.4% false-positive) is tighter than + // the standard 10 bits/key because every false positive at + // no-compaction SST count costs a disk seek across many SSTs. BlockCacheMB: 512, BloomFilterBitsPerKey: 12, - // 1 GB WAL cap matches the memtable budget. Graceful Close auto-Flushes, - // so this only bounds ungraceful-recovery (panic / power loss / OOM). + // 1 GB WAL cap matches the natural memtable budget above. + // Graceful Close auto-Flushes (see rocksdb.Store.Close), so + // this cap only bounds ungraceful-shutdown recovery (kernel + // panic, power loss, OOM kill). MaxTotalWalSizeMB: 1024, } } From b61181a06444220d08ba1f5a129c9bb3b0482e7f Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Tue, 30 Jun 2026 03:07:06 -0400 Subject: [PATCH 04/55] fullhistory: drop redundant pkg-name prefix from PR-added error messages The %w chain and call site already convey origin; the hand-typed "streaming:"/"hotchunk:"/"events:"/"rocksdb:"/"fullhistory:" leaders are noise that also drifts on file/pkg renames. Strip them from the error messages this PR introduces (new files wholesale; pre-existing files only on the lines added here, leaving base-owned lines for their own PR). --- .../internal/fullhistory/catalog/catalog.go | 2 +- .../internal/fullhistory/hotloop.go | 26 +++++++++---------- .../internal/fullhistory/hotsource.go | 2 +- .../internal/fullhistory/lifecycle/discard.go | 10 +++---- .../fullhistory/pkg/rocksdb/rocksdb.go | 2 +- .../pkg/stores/eventstore/hot_store.go | 8 +++--- .../pkg/stores/hotchunk/hotchunk.go | 16 ++++++------ 7 files changed, 33 insertions(+), 33 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go index 1f8cb3d04..292dd444e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go @@ -206,7 +206,7 @@ func (c *Catalog) hotChunkKeysWith(keep func(geometry.HotState) bool) ([]chunk.I } id, ok := geometry.ParseHotChunkKey(e.Key) if !ok { - return nil, fmt.Errorf("streaming: malformed hot key %q", e.Key) + return nil, fmt.Errorf("malformed hot key %q", e.Key) } if keep == nil || keep(geometry.HotState(e.Value)) { ids = append(ids, id) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index 22463fe3e..ed271a391 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -53,7 +53,7 @@ func openHotTierForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *support state, err := cat.HotState(chunkID) if err != nil { - return nil, fmt.Errorf("streaming: read hot state chunk %s: %w", chunkID, err) + return nil, fmt.Errorf("read hot state chunk %s: %w", chunkID, err) } if state == geometry.HotReady { @@ -80,15 +80,15 @@ func openHotTierForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *support // "transient" or absent: wipe any leftover dir, then create fresh under the bracket. if rmErr := os.RemoveAll(dir); rmErr != nil { - return nil, fmt.Errorf("streaming: wipe leftover hot dir %s: %w", dir, rmErr) + return nil, fmt.Errorf("wipe leftover hot dir %s: %w", dir, rmErr) } if putErr := cat.PutHotTransient(chunkID); putErr != nil { - return nil, fmt.Errorf("streaming: mark hot transient chunk %s: %w", chunkID, putErr) + return nil, fmt.Errorf("mark hot transient chunk %s: %w", chunkID, putErr) } db, openErr := hotchunk.Open(dir, chunkID, logger) if openErr != nil { - return nil, fmt.Errorf("streaming: create hot DB chunk %s: %w", chunkID, openErr) + return nil, fmt.Errorf("create hot DB chunk %s: %w", chunkID, openErr) } // The dir + dirent must be durable BEFORE the key flips to "ready", else a @@ -96,15 +96,15 @@ func openHotTierForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *support // dir missing" fatal above for a DB that was actually fine. if syncErr := geometry.FsyncDir(dir); syncErr != nil { _ = db.Close() - return nil, fmt.Errorf("streaming: fsync hot dir %s: %w", dir, syncErr) + return nil, fmt.Errorf("fsync hot dir %s: %w", dir, syncErr) } if syncErr := geometry.FsyncDir(parentDir(dir)); syncErr != nil { _ = db.Close() - return nil, fmt.Errorf("streaming: fsync hot parent dir %s: %w", parentDir(dir), syncErr) + return nil, fmt.Errorf("fsync hot parent dir %s: %w", parentDir(dir), syncErr) } if flipErr := cat.FlipHotReady(chunkID); flipErr != nil { _ = db.Close() - return nil, fmt.Errorf("streaming: flip hot ready chunk %s: %w", chunkID, flipErr) + return nil, fmt.Errorf("flip hot ready chunk %s: %w", chunkID, flipErr) } return db, nil } @@ -151,7 +151,7 @@ func runIngestionLoop( defer func() { if hotDB != nil { if cerr := hotDB.Close(); cerr != nil && err == nil { - err = fmt.Errorf("streaming: close live hot DB: %w", cerr) + err = fmt.Errorf("close live hot DB: %w", cerr) } } }() @@ -160,7 +160,7 @@ func runIngestionLoop( // stored — a re-delivered committed ledger is an idempotent retry). resume, err := nextIngestLedger(hotDB) if err != nil { - return fmt.Errorf("streaming: derive resume ledger: %w", err) + return fmt.Errorf("derive resume ledger: %w", err) } // hotService binds the metrics sink to THIS hotDB instance; the boundary @@ -172,13 +172,13 @@ func runIngestionLoop( for seq := resume; ; seq++ { lcm, gerr := core.GetLedger(ctx, seq) if gerr != nil { - return fmt.Errorf("streaming: get ledger %d: %w", seq, gerr) + return fmt.Errorf("get ledger %d: %w", seq, gerr) } // One atomic synced WriteBatch across all enabled CFs (via // hotDB.IngestLedger), reporting per-type LedgerCounts to the sink. if ierr := hotService.Ingest(ctx, seq, lcm); ierr != nil { - return fmt.Errorf("streaming: ingest ledger %d: %w", seq, ierr) + return fmt.Errorf("ingest ledger %d: %w", seq, ierr) } // Per-ledger liveness gauge — the moving health signal a wedged ingester @@ -194,13 +194,13 @@ func runIngestionLoop( // may then freeze and discard its hot DB — no writer may hold it then). if cerr := hotDB.Close(); cerr != nil { hotDB = nil // closed (failed) — do not double-close in defer - return fmt.Errorf("streaming: close hot DB at boundary chunk %s: %w", closed, cerr) + return fmt.Errorf("close hot DB at boundary chunk %s: %w", closed, cerr) } hotDB = nil // released; reopen below republishes it for the defer nextDB, oerr := openHotTierForChunk(cat, next, logger) if oerr != nil { - return fmt.Errorf("streaming: open hot DB for chunk %s at boundary: %w", next, oerr) + return fmt.Errorf("open hot DB for chunk %s at boundary: %w", next, oerr) } hotDB = nextDB hotService = ingest.NewHotService(hotDB, ingestTypes, sink) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotsource.go b/cmd/stellar-rpc/internal/fullhistory/hotsource.go index b5f048c0b..d6abc55ed 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotsource.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotsource.go @@ -102,7 +102,7 @@ func (st *hotLedgerStream) RawLedgers( ) iter.Seq2[[]byte, error] { return func(yield func([]byte, error) bool) { if st.store == nil { - yield(nil, errors.New("fullhistory: hotLedgerStream has no store")) + yield(nil, errors.New("hotLedgerStream has no store")) return } to := r.To() diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go index 47f507d73..677e0f965 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go @@ -18,26 +18,26 @@ import ( func discardHotTierForChunk(cat *catalog.Catalog, chunkID chunk.ID) error { state, err := cat.HotState(chunkID) if err != nil { - return fmt.Errorf("streaming: read hot key chunk %s: %w", chunkID, err) + return fmt.Errorf("read hot key chunk %s: %w", chunkID, err) } if state == "" { return nil } if putErr := cat.PutHotTransient(chunkID); putErr != nil { - return fmt.Errorf("streaming: mark hot transient chunk %s: %w", chunkID, putErr) + return fmt.Errorf("mark hot transient chunk %s: %w", chunkID, putErr) } dir := cat.Layout().HotChunkPath(chunkID) if rmErr := os.RemoveAll(dir); rmErr != nil { - return fmt.Errorf("streaming: rmdir hot dir %s: %w", dir, rmErr) + return fmt.Errorf("rmdir hot dir %s: %w", dir, rmErr) } // rmdir must be durable BEFORE the key delete: the key outlives the dir, so a // crash re-runs the discard rather than leaving a key-less dir. if syncErr := geometry.FsyncDir(filepath.Dir(dir)); syncErr != nil { - return fmt.Errorf("streaming: fsync hot parent dir %s: %w", filepath.Dir(dir), syncErr) + return fmt.Errorf("fsync hot parent dir %s: %w", filepath.Dir(dir), syncErr) } if delErr := cat.DeleteHotKey(chunkID); delErr != nil { - return fmt.Errorf("streaming: delete hot key chunk %s: %w", chunkID, delErr) + return fmt.Errorf("delete hot key chunk %s: %w", chunkID, delErr) } return nil } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go index 18aa8268b..5465dc0b9 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go @@ -506,7 +506,7 @@ func (s *Store) constructAndOpen() error { // Read-only opens an existing DB; it never creates the directory. if !s.cfg.ReadOnly { if err := os.MkdirAll(abs, dirPerm); err != nil { - return fmt.Errorf("rocksdb: mkdir %s: %w", abs, err) + return fmt.Errorf("mkdir %s: %w", abs, err) } } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index 8b2031cc9..9327bb4fe 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -549,7 +549,7 @@ func (h *HotStore) IngestLedgerToBatchCommit(ledgerSeq uint32, payloads []events if cerr := h.chunkStore.Batch(func(b *rocksdb.BatchWriter) error { return prep.queue(b) }); cerr != nil { - return nil, fmt.Errorf("events: commit ledger %d to chunk %s: %w", ledgerSeq, h.chunkID, cerr) + return nil, fmt.Errorf("commit ledger %d to chunk %s: %w", ledgerSeq, h.chunkID, cerr) } return prep.apply, nil } @@ -639,14 +639,14 @@ func (h *HotStore) prepareLedger(ledgerSeq uint32, payloads []events.Payload) (* for i := range payloads { keys, err := events.TermsForBytes(payloads[i].ContractEventBytes) if err != nil { - return nil, fmt.Errorf("events: derive terms for payload %d in ledger %d: %w", i, ledgerSeq, err) + return nil, fmt.Errorf("derive terms for payload %d in ledger %d: %w", i, ledgerSeq, err) } termKeys[i] = keys } startID := h.offsets.TotalEvents() if uint64(startID)+uint64(len(payloads)) > math.MaxUint32 { - return nil, fmt.Errorf("events: chunk %s would overflow uint32 event-id space at ledger %d", + return nil, fmt.Errorf("chunk %s would overflow uint32 event-id space at ledger %d", h.chunkID, ledgerSeq) } @@ -658,7 +658,7 @@ func (h *HotStore) prepareLedger(ledgerSeq uint32, payloads []events.Payload) (* for i := range payloads { blob, err := payloads[i].MarshalInto(nil) if err != nil { - return nil, fmt.Errorf("events: marshal payload %d for ledger %d: %w", i, ledgerSeq, err) + return nil, fmt.Errorf("marshal payload %d for ledger %d: %w", i, ledgerSeq, err) } blobs[i] = blob } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index 2dca2e546..5b155bfed 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -88,13 +88,13 @@ func open(path string, chunkID chunk.ID, logger *supportlog.Entry, readOnly bool } store, err := rocksdb.New(config(path, logger, readOnly)) if err != nil { - return nil, fmt.Errorf("hotchunk: open chunk %s: %w", chunkID, err) + return nil, fmt.Errorf("open chunk %s: %w", chunkID, err) } es, err := eventstore.NewWithStore(store, chunkID) if err != nil { _ = store.Close() - return nil, fmt.Errorf("hotchunk: compose events facade for chunk %s: %w", chunkID, err) + return nil, fmt.Errorf("compose events facade for chunk %s: %w", chunkID, err) } return &DB{ store: store, @@ -164,7 +164,7 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView, cfg Ingest) ( if cfg.Txhash { hashes, err := sdkingest.ExtractTxHashes(lcm) if err != nil { - return counts, fmt.Errorf("hotchunk: extract tx hashes seq %d: %w", seq, err) + return counts, fmt.Errorf("extract tx hashes seq %d: %w", seq, err) } if len(hashes) > 0 { txEntries = make([]txhash.Entry, len(hashes)) @@ -195,25 +195,25 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView, cfg Ingest) ( cerr := d.store.Batch(func(b *rocksdb.BatchWriter) error { if cfg.Ledgers { if err := d.ledger.AddLedgerToBatch(b, ledger.Entry{Seq: seq, Bytes: []byte(lcm)}); err != nil { - return fmt.Errorf("hotchunk: queue ledger seq %d: %w", seq, err) + return fmt.Errorf("queue ledger seq %d: %w", seq, err) } } if cfg.Txhash && len(txEntries) > 0 { if err := d.txhash.AddEntriesToBatch(b, txEntries); err != nil { - return fmt.Errorf("hotchunk: queue tx hashes seq %d: %w", seq, err) + return fmt.Errorf("queue tx hashes seq %d: %w", seq, err) } } if cfg.Events { apply, err := d.events.IngestLedgerToBatch(b, seq, payloads) if err != nil { - return fmt.Errorf("hotchunk: queue events seq %d: %w", seq, err) + return fmt.Errorf("queue events seq %d: %w", seq, err) } applyEvents = apply } return nil }) if cerr != nil { - return counts, fmt.Errorf("hotchunk: commit ledger %d to chunk %s: %w", seq, d.chunkID, cerr) + return counts, fmt.Errorf("commit ledger %d to chunk %s: %w", seq, d.chunkID, cerr) } // Batch is durable — now and only now apply the events mirror/offsets update. @@ -229,7 +229,7 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView, cfg Ingest) ( func eventPayloads(seq uint32, lcm xdr.LedgerCloseMetaView) ([]events.Payload, error) { payloads, err := events.LCMViewToPayloads(lcm) if err != nil { - return nil, fmt.Errorf("hotchunk: LCMViewToPayloads seq %d: %w", seq, err) + return nil, fmt.Errorf("LCMViewToPayloads seq %d: %w", seq, err) } return payloads, nil } From a0b068cb682fa07d96a8b764240f96fcf8146f44 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Tue, 30 Jun 2026 17:06:32 -0400 Subject: [PATCH 05/55] =?UTF-8?q?streaming(fullhistory):=20Phase=202=20lay?= =?UTF-8?q?er=202=20=E2=80=94=20live=20ingestion=20+=20daemon=20wiring=20(?= =?UTF-8?q?closes=20#816,=20#808)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold the live-ingestion layer (formerly the separate PR #821) into this PR so #820 alone completes the Phase 2 epic #816 (and #808), rather than the original two-PR split. run() now transitions from backfill to a serve+ingest steady state: after catch-up it opens the resume chunk's hot DB, starts captive core (injected CoreOpener), launches the lifecycle goroutine on a doorbell seeded with the last complete chunk, serves reads, then runs runIngestionLoop. The lifecycle goroutine is tied to a per-iteration ctx and joined before run() returns, so the single-lifecycle-goroutine invariant holds across supervisor restarts. - lifecycle: export RunLoop (the event-driven lifecycle goroutine). - startup: CoreOpener seam; StartConfig gains Core + Lifecycle; last-committed derivation now uses the hot probe (lazy ErrHotVolumeLost detection). - daemon: production captiveCoreOpener/backendGetter (captive-core config plumbing deferred to #772); Core threaded through startConfig; supervise treats ErrHotVolumeLost as fatal; test-only seams for cpi + bound-catalog. - tests: run/daemon/supervise adapted to serve+ingest; full-lifecycle E2E (ingest -> freeze -> fold -> discard -> cold+hot lookup -> restart -> prune). --- .../internal/fullhistory/daemon.go | 129 +++- .../internal/fullhistory/daemon_test.go | 39 +- .../internal/fullhistory/e2e_test.go | 597 ++++++++++++++++++ .../fullhistory/lifecycle/lifecycle.go | 4 +- .../lifecycle/lifecycle_loop_test.go | 8 +- .../internal/fullhistory/startup.go | 129 +++- .../internal/fullhistory/startup_test.go | 152 ++++- 7 files changed, 980 insertions(+), 78 deletions(-) create mode 100644 cmd/stellar-rpc/internal/fullhistory/e2e_test.go diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index c2d0d1004..89ef6b79e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -9,13 +9,16 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" supportlog "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/daemon/interfaces" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/ingest" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/observability" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/metastore" ) @@ -34,6 +37,12 @@ type daemonOptions struct { // frontfill-only daemon when no datastore is configured). Tests inject a fakeBackend. Backend backfill.Backend + // Core starts captive core at the resume ledger and yields the live getter the + // ingestion loop polls. nil ⇒ runDaemonWith builds a captiveCoreOpener (whose + // config plumbing is deferred to #772, so production must inject Core until then). + // Tests inject a fake getter. + Core CoreOpener + // ServeReads launches the RPC read server; it must return promptly, not block. // nil ⇒ the #772 no-op placeholder (reads still come from the v1 SQLite daemon). ServeReads func(ctx context.Context) error @@ -49,6 +58,17 @@ type daemonOptions struct { // IngestSink is the per-type cold-path ingest sink; nil ⇒ a *ingest.PrometheusSink. IngestSink ingest.MetricSink + + // chunksPerTxhashIndex overrides the tx-hash index width (test-only). 0 ⇒ the + // fixed geometry.ChunksPerTxhashIndex. Tests set it to 1 so a single chunk's + // freeze is a terminal index (exercising the fold+prune path cheaply). + chunksPerTxhashIndex uint32 + + // onCatalog, when set, receives the daemon's bound Catalog (test-only). The + // metastore is opened RocksDB-primary (exclusive LOCK), so a test cannot open a + // second handle while the daemon runs; this lets it inspect durable state live + // through the daemon's own catalog (safe for concurrent reads). + onCatalog func(*catalog.Catalog) } const defaultRestartBackoff = 5 * time.Second @@ -88,11 +108,18 @@ func runDaemonWith(ctx context.Context, configPath string, opts daemonOptions) e } defer func() { _ = store.Close() }() - txLayout, err := geometry.NewTxHashIndexLayout(geometry.ChunksPerTxhashIndex) + cpi := geometry.ChunksPerTxhashIndex + if opts.chunksPerTxhashIndex != 0 { + cpi = opts.chunksPerTxhashIndex + } + txLayout, err := geometry.NewTxHashIndexLayout(cpi) if err != nil { return err } cat := catalog.NewCatalog(store, NewLayoutFromPaths(paths), txLayout) + if opts.onCatalog != nil { + opts.onCatalog(cat) + } // --- Resolve the backfill backend: injected (tests) or built from // [backfill.datastore] (production; nil ⇒ frontfill-only). Its Tip drives both @@ -130,8 +157,21 @@ func runDaemonWith(ctx context.Context, configPath string, opts daemonOptions) e registry := prometheus.NewRegistry() metrics, sink := buildSinks(opts, registry) + // Resolve the captive-core opener: injected (tests) or built from + // [ingestion].captive_core_config. Production wiring is deferred to #772, so the + // builder errors with a clear pointer — done after validateConfig so config + // errors surface first, and a deployment must inject Core until the cutover. + core := opts.Core + if core == nil { + built, cerr := newCaptiveCoreOpener(cfg.Ingestion.CaptiveCoreConfig, logger) + if cerr != nil { + return cerr + } + core = built + } + // --- Assemble the StartConfig and run the supervised run loop. --- - start := startConfig(cfg, cat, logger, backend, networkTip, serveReads, metrics, sink, tipBackoff, tipMaxAttempts) + start := startConfig(cfg, cat, logger, backend, networkTip, core, serveReads, metrics, sink, tipBackoff, tipMaxAttempts) backoff := opts.RestartBackoff if backoff <= 0 { @@ -140,10 +180,12 @@ func runDaemonWith(ctx context.Context, configPath string, opts daemonOptions) e return supervise(ctx, start, logger, backoff) } -// startConfig assembles the StartConfig run consumes. +// startConfig assembles the StartConfig run consumes. Exec and Lifecycle share +// ONE catalog, worker pool, and retention floor (catch-up and the lifecycle +// goroutine share one set of postconditions), so Lifecycle embeds the same exec. func startConfig( cfg Config, cat *catalog.Catalog, logger *supportlog.Entry, - backend backfill.Backend, networkTip NetworkTipBackend, serveReads func(context.Context) error, + backend backfill.Backend, networkTip NetworkTipBackend, core CoreOpener, serveReads func(context.Context) error, metrics observability.Metrics, sink ingest.MetricSink, tipBackoff time.Duration, tipMaxAttempts int, ) StartConfig { exec := backfill.ExecConfig{ @@ -159,12 +201,16 @@ func startConfig( }, } return StartConfig{ - Exec: exec, - RetentionChunks: deref(cfg.Retention.RetentionChunks), - NetworkTip: networkTip, - ServeReads: serveReads, - TipBackoff: tipBackoff, - TipMaxAttempts: tipMaxAttempts, + Exec: exec, + Lifecycle: lifecycle.LifecycleConfig{ + ExecConfig: exec, + RetentionChunks: deref(cfg.Retention.RetentionChunks), + }, + NetworkTip: networkTip, + Core: core, + ServeReads: serveReads, + TipBackoff: tipBackoff, + TipMaxAttempts: tipMaxAttempts, } } @@ -196,8 +242,9 @@ func supervise( if ctx.Err() != nil { return nil //nolint:nilerr // ctx canceled is a clean shutdown, not a run failure } - // Unrecoverable: a fresh start cannot heal it, so don't spin restarting. - if errors.Is(err, ErrFirstStartNoTip) { + // Unrecoverable: a fresh start cannot heal these, so don't spin restarting — + // surface them up so an operator/supervisor sees them. + if errors.Is(err, backfill.ErrHotVolumeLost) || errors.Is(err, ErrFirstStartNoTip) { return err } logger.WithError(err).Warnf("daemon run failed; restarting in %s", backoff) @@ -245,6 +292,62 @@ func buildBackfillBackend( return backend, cleanup, nil } +// --------------------------------------------------------------------------- +// Production captive-core opener (the live ingestion source). +// --------------------------------------------------------------------------- + +// captiveCoreOpener is the production CoreOpener: it prepares captive core at the +// resume ledger and hands back a LedgerGetter the ingestion loop polls by +// sequence (the design's core.GetLedger(ctx, seq)) plus a closer. +type captiveCoreOpener struct { + backend ledgerbackend.LedgerBackend +} + +// newCaptiveCoreOpener builds the production opener. The captive-core config +// plumbing is deferred to #772, so today it parses the path and errors with a +// clear pointer — a deployment must inject a CoreOpener via daemonOptions until +// the cutover lands. The seam (a LedgerGetter behind CoreOpener) is final. +func newCaptiveCoreOpener(captiveCoreConfigPath string, _ *supportlog.Entry) (*captiveCoreOpener, error) { + if captiveCoreConfigPath == "" { + return nil, errors.New("[ingestion].captive_core_config is required") + } + // TODO(#772): build a ledgerbackend.CaptiveCoreConfig from + // NewCaptiveCoreTomlFromFile(captiveCoreConfigPath, ...) + NewCaptive, then + // PrepareRange(UnboundedRange(resume)) in OpenCore. Only the config plumbing + // is deferred; the seam below is final. + return nil, fmt.Errorf("production captive-core wiring is deferred to #772 "+ + "(config %q parsed; inject a CoreOpener via daemonOptions to run today)", captiveCoreConfigPath) +} + +// OpenCore prepares the backend over the unbounded range from resumeLedger and +// returns a getter wrapping GetLedger plus the backend's Close. +func (c *captiveCoreOpener) OpenCore( + ctx context.Context, resumeLedger uint32, +) (LedgerGetter, func() error, error) { + if err := c.backend.PrepareRange(ctx, ledgerbackend.UnboundedRange(resumeLedger)); err != nil { + return nil, nil, fmt.Errorf("captive core prepare range from %d: %w", resumeLedger, err) + } + return backendGetter{backend: c.backend}, c.backend.Close, nil +} + +// backendGetter adapts a ledgerbackend.LedgerBackend to LedgerGetter: GetLedger +// blocks until the ledger is available and returns its raw wire bytes. +type backendGetter struct { + backend ledgerbackend.LedgerBackend +} + +func (g backendGetter) GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseMetaView, error) { + lcm, err := g.backend.GetLedger(ctx, seq) + if err != nil { + return nil, err + } + raw, err := lcm.MarshalBinary() + if err != nil { + return nil, fmt.Errorf("marshal ledger %d: %w", seq, err) + } + return xdr.LedgerCloseMetaView(raw), nil +} + // resolveNetworkTip adapts the backfill backend to backfill's tip sampler — its Tip // frontier (so the tip and the freeze's coverage frontier are one source) — or the // not-configured placeholder for a frontfill-only daemon (nil backend). @@ -287,6 +390,8 @@ func newLogger(cfg LoggingConfig) (*supportlog.Entry, error) { // compile-time interface checks. var ( + _ CoreOpener = (*captiveCoreOpener)(nil) + _ LedgerGetter = backendGetter{} _ NetworkTipBackend = notConfiguredTip{} _ NetworkTipBackend = backendTip{} ) diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon_test.go b/cmd/stellar-rpc/internal/fullhistory/daemon_test.go index d5f09bff9..f512670d9 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon_test.go @@ -59,26 +59,34 @@ format = "text" // runDaemonWith — the full entrypoint flow against an injected backend. // --------------------------------------------------------------------------- -// Happy path pins earliest_ledger and serves reads once. The injected backend's -// young-network tip (inside chunk 0) ⇒ no-op backfill, no LedgerStream needed. +// Happy path pins earliest_ledger, serves reads once, then ingests. The injected +// backend's young-network tip (inside chunk 0) ⇒ no-op backfill; the injected core +// blocks until ctx cancel (the daemon's steady state), and a ctx cancel is a clean +// shutdown. No LedgerStream needed. func TestRunDaemon_LoadValidateWireStartCleanShutdown(t *testing.T) { configPath, dataDir := writeTempConfig(t, "") var served atomic.Int32 opts := daemonOptions{ Backend: &fakeBackend{tip: chunk.FirstLedgerSeq + 10}, + Core: &fakeCore{}, // default getter blocks until ctx cancel ServeReads: func(context.Context) error { served.Add(1); return nil }, Logger: silentLogger(), } + ctx, cancel := context.WithCancel(context.Background()) errCh := make(chan error, 1) - go func() { errCh <- runDaemonWith(context.Background(), configPath, opts) }() + go func() { errCh <- runDaemonWith(ctx, configPath, opts) }() + + // ServeReads is called after backfill, just before the (blocking) ingestion loop. + require.Eventually(t, func() bool { return served.Load() == 1 }, 3*time.Second, 5*time.Millisecond) + cancel() select { case err := <-errCh: - require.NoError(t, err, "cold backfill + serve returns cleanly") + require.NoError(t, err, "a ctx-cancelled ingestion loop is a clean shutdown") case <-time.After(3 * time.Second): - t.Fatal("runDaemonWith did not return") + t.Fatal("runDaemonWith did not return after ctx cancel") } assert.Equal(t, int32(1), served.Load(), "reads served once") @@ -182,23 +190,36 @@ func TestRunDaemon_BackfillMaterializesAllColdTypesAndIndex(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + // ServeReads runs after backfill completes, just before the blocking ingestion + // loop — so it is the "backfill done" signal. The injected core then blocks until + // the ctx cancel below, and a ctx-cancelled ingestion loop is a clean shutdown. + servedCh := make(chan struct{}, 1) errCh := make(chan error, 1) go func() { errCh <- runDaemonWith(ctx, configPath, daemonOptions{ // Backend's tip is chunk 0's last ledger ⇒ chunk 0 complete, backfill freezes it. // The network tip is derived from this same backend's Tip. Backend: someTxBackend(t), - ServeReads: func(context.Context) error { return nil }, + Core: &fakeCore{}, // default getter blocks until ctx cancel + ServeReads: func(context.Context) error { servedCh <- struct{}{}; return nil }, Logger: silentLogger(), }) }() select { + case <-servedCh: // backfill complete; the daemon is now parked in ingestion case err := <-errCh: - require.NoError(t, err, "daemon backfills to tip then exits cleanly (no-op ServeReads)") + t.Fatalf("daemon returned before backfill completed: %v", err) case <-time.After(60 * time.Second): cancel() t.Fatal("runDaemonWith did not finish backfill within 60s (regressed into a hang/restart loop?)") } + cancel() // request a clean shutdown of the parked ingestion loop + select { + case err := <-errCh: + require.NoError(t, err, "a ctx-cancelled ingestion loop is a clean shutdown") + case <-time.After(10 * time.Second): + t.Fatal("runDaemonWith did not return after ctx cancel") + } // Read the catalog back after the daemon released locks + closed its store. store, err := openMetaAt(t, filepath.Join(dataDir, "catalog", "rocksdb")) @@ -380,7 +401,7 @@ func TestSupervise_RetriesThenCleanShutdown(t *testing.T) { var attempts atomic.Int32 tip := &fakeTipBackend{tips: []uint32{chunk.FirstLedgerSeq + 10}} // young: no backfill - start := startTestConfig(t, cat, tip, nil) + start := startTestConfig(t, cat, tip, &fakeCore{}, nil) // An always-erroring ServeReads makes each attempt a restartable failure. start.ServeReads = func(context.Context) error { attempts.Add(1) @@ -412,7 +433,7 @@ func TestSupervise_FatalSentinelSurfaces(t *testing.T) { pinGenesis(t, cat) // Unreachable tip + no local progress ⇒ fatal ErrFirstStartNoTip. tip := &fakeTipBackend{err: errors.New("unreachable"), errFirst: 99} - start := startTestConfig(t, cat, tip, nil) + start := startTestConfig(t, cat, tip, &fakeCore{}, nil) err := supervise(context.Background(), start, silentLogger(), time.Hour) require.ErrorIs(t, err, ErrFirstStartNoTip, "fatal sentinel surfaces immediately, no retry") diff --git a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go new file mode 100644 index 000000000..ada5d1ac8 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go @@ -0,0 +1,597 @@ +package fullhistory + +// ============================================================================= +// In-process end-to-end integration of the full-history daemon. +// +// WHAT IS REAL HERE +// Everything inside the process is the real production code path: +// - runDaemonWith (the true daemon entrypoint): TOML load + form-validate, +// per-root flock, meta-store open + Catalog bind, the stateful +// validateConfig gate (pins the floor), and the supervised run loop. +// - run → backfillToTip → openHotTierForChunk → runIngestionLoop (the real +// atomic per-ledger WriteBatch across all CFs of the real per-chunk +// hotchunk RocksDB), the real boundary handoff, the real doorbell. +// - lifecycle.RunLoop / runLifecycleTick: the real resolve + executePlan +// freeze (cold artifacts derived FROM the live hot DB), the real txhash +// index fold (a real streamhash .idx on disk), the real discard + prune. +// - The real txhash stores on both sides of a getTransaction-style hash→seq +// lookup: the cold ColdReader over the frozen .idx and the live hot CF. +// +// WHAT IS FAKED (the two EXTERNAL boundaries the daemon injects on purpose) +// - The ledger SOURCE. Production drives ingestion from captive +// stellar-core and backfill from a bulk object-store backend. Here both +// cross their injected interfaces (CoreOpener / backfill.Backend) and are +// fed synthetic-but-well-formed LedgerCloseMeta. No captive core, no +// object store, no network. +// - ServeReads is a no-op recorder (the read cutover is #772). The read PATH +// exercised is the txhash index lookup getTransaction will sit on. +// +// cpi=1 (the chunksPerTxhashIndex test seam) makes every one-chunk window +// terminal the instant its chunk freezes, so the freeze→fold→discard→prune +// sequence completes on a boundary tick without ingesting 1000 chunks. +// ============================================================================= + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stellar/go-stellar-sdk/keypair" + "github.com/stellar/go-stellar-sdk/network" + "github.com/stellar/go-stellar-sdk/xdr" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/observability" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash" +) + +// e2ePassphrase is the network passphrase the synthetic tx hashes are computed +// against. Any stable value works; the index only needs deterministic hashes. +const e2ePassphrase = network.PublicNetworkPassphrase + +// oneTxLCMReturningHash builds a well-formed V2 LedgerCloseMeta carrying exactly +// ONE transaction for seq and returns BOTH the wire bytes and the real, +// network-hashed transaction hash. A non-zero-tx ledger is required somewhere in +// a chunk so its txhash .bin is non-empty (streamhash refuses a zero-key cold +// index, txhash.ErrEmptyBuildSet); returning the hash lets the E2E assert the +// getTransaction-style hash→seq lookup against a hash the daemon really committed. +func oneTxLCMReturningHash(t *testing.T, seq uint32) ([]byte, [32]byte) { + t.Helper() + envelope := xdr.TransactionEnvelope{ + Type: xdr.EnvelopeTypeEnvelopeTypeTx, + V1: &xdr.TransactionV1Envelope{ + Tx: xdr.Transaction{ + SourceAccount: xdr.MustMuxedAddress(keypair.MustRandom().Address()), + Ext: xdr.TransactionExt{V: 1, SorobanData: &xdr.SorobanTransactionData{}}, + }, + }, + } + hash, err := network.HashTransactionInEnvelope(envelope, e2ePassphrase) + require.NoError(t, err) + + comp := []xdr.TxSetComponent{{ + Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, + TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ + Txs: []xdr.TransactionEnvelope{envelope}, + }, + }} + opResults := []xdr.OperationResult{} + lcm := xdr.LedgerCloseMeta{ + V: 2, + V2: &xdr.LedgerCloseMetaV2{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + ScpValue: xdr.StellarValue{CloseTime: xdr.TimePoint(0)}, + LedgerSeq: xdr.Uint32(seq), + }, + }, + TxSet: xdr.GeneralizedTransactionSet{ + V: 1, + V1TxSet: &xdr.TransactionSetV1{Phases: []xdr.TransactionPhase{{V: 0, V0Components: &comp}}}, + }, + TxProcessing: []xdr.TransactionResultMetaV1{{ + TxApplyProcessing: xdr.TransactionMeta{ + V: 4, + V4: &xdr.TransactionMetaV4{Operations: []xdr.OperationMetaV2{}}, + }, + Result: xdr.TransactionResultPair{ + TransactionHash: hash, + Result: xdr.TransactionResult{ + FeeCharged: 100, + Result: xdr.TransactionResultResult{Code: xdr.TransactionResultCodeTxSuccess, Results: &opResults}, + }, + }, + }}, + }, + } + raw, err := lcm.MarshalBinary() + require.NoError(t, err) + return raw, hash +} + +// e2eFrame is one synthetic ledger the fake core serves. +type e2eFrame struct { + seq uint32 + raw []byte +} + +// e2eGetter is the FAKE captive-core ledger getter: a resumable LedgerGetter the +// ingestion loop polls by sequence. It returns the frame for the requested seq +// when it has one, and once the poll runs past the synthetic backlog it blocks +// until ctx is cancelled (a live tip stream ends only on shutdown). It records +// the FIRST seq it was asked for so the restart step can assert the daemon +// re-derived the watermark and resumed with no gap. +type e2eGetter struct { + frames map[uint32][]byte + fromSeen *atomic.Uint32 // first GetLedger seq (for the restart assertion) + delivered *atomic.Uint32 // highest seq actually yielded (test sync) + sawFrom atomic.Bool +} + +var _ LedgerGetter = (*e2eGetter)(nil) + +func (s *e2eGetter) GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseMetaView, error) { + if s.sawFrom.CompareAndSwap(false, true) { + s.fromSeen.Store(seq) + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + if raw, ok := s.frames[seq]; ok { + s.delivered.Store(seq) + return xdr.LedgerCloseMetaView(raw), nil + } + // Past the synthetic backlog: a live tip blocks until shutdown so the loop + // does not see an error that would look like a core crash. + <-ctx.Done() + return nil, ctx.Err() +} + +// e2eCore is the CoreOpener handing back a fresh e2eGetter per daemon run (a +// restart opens core anew). It records the resume ledger every open was driven from. +type e2eCore struct { + frames []e2eFrame + resumeSeen atomic.Uint32 + fromSeen atomic.Uint32 + delivered atomic.Uint32 + opens atomic.Int32 +} + +func (c *e2eCore) OpenCore(_ context.Context, resume uint32) (LedgerGetter, func() error, error) { + c.opens.Add(1) + c.resumeSeen.Store(resume) + byseq := make(map[uint32][]byte, len(c.frames)) + for _, f := range c.frames { + byseq[f.seq] = f.raw + } + getter := &e2eGetter{frames: byseq, fromSeen: &c.fromSeen, delivered: &c.delivered} + return getter, func() error { return nil }, nil +} + +// e2eMetrics is a concurrency-safe observability.Metrics that records the chunk +// boundaries and freezes the daemon emits (the rest discarded via NopMetrics). +type e2eMetrics struct { + observability.NopMetrics + mu sync.Mutex + boundaries []uint32 + freezes int +} + +func (m *e2eMetrics) ChunkBoundary(closed uint32) { + m.mu.Lock() + defer m.mu.Unlock() + m.boundaries = append(m.boundaries, closed) +} + +func (m *e2eMetrics) Freeze(time.Duration) { + m.mu.Lock() + defer m.mu.Unlock() + m.freezes++ +} + +func (m *e2eMetrics) snapshotBoundaries() []uint32 { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]uint32, len(m.boundaries)) + copy(out, m.boundaries) + return out +} + +func (m *e2eMetrics) snapshotFreezeCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.freezes +} + +// e2eConfigPath writes a daemon TOML for an in-process E2E: genesis floor (no +// tip needed to validate/start) and the given retention width. captive_core_config +// is a stub path the test's injected CoreOpener replaces, never opening a real core. +// The one-chunk index window is set via the chunksPerTxhashIndex test seam, not config. +func e2eConfigPath(t *testing.T, dataDir string, retentionChunks uint32) string { + t.Helper() + cfgPath := filepath.Join(t.TempDir(), "daemon.toml") + body := fmt.Sprintf(` +[service] +default_data_dir = %q + +[retention] +earliest_ledger = "genesis" +retention_chunks = %d + +[ingestion] +captive_core_config = "/dev/null" + +[logging] +level = "error" +format = "text" +`, dataDir, retentionChunks) + require.NoError(t, os.WriteFile(cfgPath, []byte(body), 0o644)) + return cfgPath +} + +// runDaemonInBackground starts runDaemonWith on a cancellable ctx and returns a +// cancel func, a channel carrying its (clean-shutdown) return, and a channel +// delivering the daemon's OWN bound *catalog.Catalog (captured via the onCatalog +// seam). The metastore is opened RocksDB-primary (exclusive LOCK), so a test +// cannot open a second handle while the daemon runs — instead it reads durable +// state through the daemon's own catalog (safe for concurrent reads). A young- +// network tip (inside chunk 0) means backfill is a no-op and first-start ingests +// directly from genesis via the fake core. +func runDaemonInBackground( + t *testing.T, cfgPath string, core *e2eCore, served *atomic.Int32, metrics observability.Metrics, +) (context.CancelFunc, <-chan error, <-chan *catalog.Catalog) { + t.Helper() + ctx, cancelFn := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + catChan := make(chan *catalog.Catalog, 1) + opts := daemonOptions{ + Backend: &fakeBackend{tip: chunk.FirstLedgerSeq + 5}, // young: no backfill + Core: core, + ServeReads: func(context.Context) error { served.Add(1); return nil }, + Logger: silentLogger(), + Metrics: metrics, + RestartBackoff: 10 * time.Millisecond, + chunksPerTxhashIndex: 1, + onCatalog: func(cat *catalog.Catalog) { + select { + case catChan <- cat: + default: + } + }, + } + go func() { errCh <- runDaemonWith(ctx, cfgPath, opts) }() + return cancelFn, errCh, catChan +} + +// awaitCatalog waits for the daemon to hand back its bound catalog. +func awaitCatalog(t *testing.T, catCh <-chan *catalog.Catalog) *catalog.Catalog { + t.Helper() + select { + case cat := <-catCh: + return cat + case <-time.After(10 * time.Second): + t.Fatal("daemon did not bind a catalog") + return nil + } +} + +// waitClean cancels the daemon and requires a clean (nil) shutdown. +func waitClean(t *testing.T, cancel context.CancelFunc, done <-chan error) { + t.Helper() + cancel() + select { + case err := <-done: + require.NoError(t, err, "ctx cancel is a clean daemon shutdown") + case <-time.After(60 * time.Second): + // Post-cancel shutdown joins one in-flight lifecycle unit; a mid-flight + // freeze's Finalize fsync + index build is unpreemptible and slow under + // -race + contention — the same reason the boundary-cross budget is 600s. + t.Fatal("daemon did not shut down cleanly after ctx cancel") + } +} + +// hotKeyExists reports whether chunk c's hot:chunk key is present (any non-empty state). +func hotKeyExists(cat *catalog.Catalog, c chunk.ID) (bool, error) { + st, err := cat.HotState(c) + if err != nil { + return false, err + } + return st != geometry.HotState(""), nil +} + +// hashAt builds a deterministic 32-byte hash from n (for the never-committed miss). +func hashAt(n uint64) [32]byte { + var h [32]byte + for i := 0; i < 8; i++ { + h[i] = byte(n >> (8 * i)) + } + return h +} + +// TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune drives the +// whole daemon lifecycle in one process against the real stores and the fake +// ledger source: +// +// first start (genesis, young-network tip ⇒ direct ingest) → +// ingest a FULL chunk + cross into the next (real boundary handoff) → +// lifecycle tick freezes chunk 0 + folds its terminal txhash index + discards +// its hot tier → +// getTransaction-style hash→seq lookup resolves from the cold .idx (chunk 0) +// AND from the live hot CF (chunk 2) → +// clean shutdown → +// RESTART: re-derive the watermark, resume at exactly watermark+1 (no gap) → +// drive retention far enough to prune chunk 0, confirm a pruned read is not-found. +// +// Correctness is asserted at every step. +func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing.T) { + if testing.Short() { + t.Skip("e2e ingests a full 10k-ledger chunk; skipped in -short") + } + + dataDir := t.TempDir() + + const c0 = chunk.ID(0) + const c1 = chunk.ID(1) + const c2 = chunk.ID(2) + + // Cross TWO chunk boundaries so chunks 0 AND 1 both freeze, leaving chunk 2 as + // the live (un-frozen) chunk. That layout lets a later retention_chunks=1 run + // prune chunk 0 (wholly below the floor) while chunk 1 survives. + c0First := c0.FirstLedger() + c1First := c1.FirstLedger() + c2First := c2.FirstLedger() + + coldRaw, coldHash := oneTxLCMReturningHash(t, c0First) // → frozen cold .idx (chunk 0) + hotRaw, hotHash := oneTxLCMReturningHash(t, c2First) // → live hot CF (chunk 2) + // Chunk 1's first ledger also carries a tx so its txhash .bin is non-empty — + // streamhash refuses to build a cold index over zero keys (ErrEmptyBuildSet). + c1Raw, _ := oneTxLCMReturningHash(t, c1First) + + frames := make([]e2eFrame, 0, 2*int(chunk.LedgersPerChunk)+2) + appendLedger := func(seq uint32) { + var raw []byte + switch seq { + case c0First: + raw = coldRaw + case c1First: + raw = c1Raw + case c2First: + raw = hotRaw + default: + raw = zeroTxLCMBytes(t, seq) + } + frames = append(frames, e2eFrame{seq: seq, raw: raw}) + } + // Chunks 0 and 1 in full (both freeze), then chunk 2's first two ledgers. + for seq := c0First; seq <= c1.LastLedger(); seq++ { + appendLedger(seq) + } + appendLedger(c2First) + appendLedger(c2First + 1) + + core := &e2eCore{frames: frames} + var served atomic.Int32 + metrics := &e2eMetrics{} + + // ===================================================================== + // STEP 1 — first start: config → lock → validate (pin genesis) → start → + // direct ingest across the chunk-0 AND chunk-1 boundaries, the lifecycle + // freezing, folding, and discarding each just-closed chunk off the doorbell. + // ===================================================================== + cfgPath := e2eConfigPath(t, dataDir, 0) // retention 0 (full history) for now + cancel, done, catCh := runDaemonInBackground(t, cfgPath, core, &served, metrics) + + // Inspect durable state through the daemon's OWN bound catalog (metastore is + // RocksDB-primary, so a second handle would fail the LOCK). + cat := awaitCatalog(t, catCh) + + // Wait until ingestion crosses BOTH boundaries and commits into chunk 2. + // Delivering c2First proves both boundary handoffs fired (chunks 0 and 1 + // closed, chunk 2 opened) and seeds the live hot-CF lookup. 600s absorbs the + // worst-case contended -race path (per-ledger synced WriteBatches racing the + // freezes that re-read 10k ledgers each). + require.Eventually(t, func() bool { + return core.delivered.Load() >= c2First + }, 600*time.Second, 200*time.Millisecond, "ingestion must cross both boundaries into chunk 2") + + // The boundary doorbells have rung. Per chunk, the durable completion signal is: + // the window has a FROZEN txhash coverage (the .idx) AND the chunk's hot key is + // gone (discarded). + w0 := cat.TxHashIndexLayout().TxHashIndexID(c0) + w1 := cat.TxHashIndexLayout().TxHashIndexID(c1) + require.Eventually(t, func() bool { + for w, c := range map[geometry.TxHashIndexID]chunk.ID{w0: c0, w1: c1} { + _, hasCov, err := cat.FrozenTxHashIndex(w) + if err != nil || !hasCov { + return false + } + has, err := hotKeyExists(cat, c) + if err != nil || has { + return false + } + } + return true + }, 60*time.Second, 50*time.Millisecond, "the boundary ticks must freeze+fold+discard chunks 0 and 1") + + require.GreaterOrEqual(t, served.Load(), int32(1), "reads were served") + require.Equal(t, uint32(c0First), core.resumeSeen.Load(), + "first start resumes captive core at genesis (watermark+1)") + + // --- Correctness: chunks 0 and 1 per-chunk cold artifacts (ledgers + events) froze. --- + for _, c := range []chunk.ID{c0, c1} { + for _, kind := range []geometry.Kind{geometry.KindLedgers, geometry.KindEvents} { + st, err := cat.State(c, kind) + require.NoError(t, err) + assert.Equal(t, geometry.StateFrozen, st, "chunk %s %s is frozen", c, kind) + } + } + // The window's txhash index is a frozen, terminal coverage (the .idx the cold + // getTransaction read resolves against). + frozenCov, ok, err := cat.FrozenTxHashIndex(w0) + require.NoError(t, err) + require.True(t, ok, "chunk 0's window has a frozen txhash coverage") + require.True(t, cat.TxHashIndexLayout().IsTerminalCoverage(frozenCov), "a one-chunk (cpi=1) window is terminal") + + // ===================================================================== + // STEP 2 — getTransaction-style hash→seq lookup, cold tier. + // ===================================================================== + + // Cold .idx — the exact reader getTransaction will sit on for frozen history. + coldReader, err := txhash.OpenColdReader(cat.Layout().TxHashIndexFilePath(frozenCov)) + require.NoError(t, err) + gotSeq, err := coldReader.Get(coldHash) + require.NoError(t, err, "the chunk-0 tx hash must resolve from the frozen cold index") + assert.Equal(t, c0First, gotSeq, "cold lookup returns the ledger the tx was committed in") + // A hash that was never committed misses (not-found, not a wrong answer). + _, missErr := coldReader.Get(hashAt(0xE2EDEADBEEF)) + require.ErrorIs(t, missErr, stores.ErrNotFound, "an uncommitted hash misses the cold index") + require.NoError(t, coldReader.Close()) + + // Observability: the daemon emitted the boundary + freeze phase signals. + assert.GreaterOrEqual(t, len(metrics.snapshotBoundaries()), 1, "at least one chunk boundary was signaled") + assert.GreaterOrEqual(t, metrics.snapshotFreezeCount(), 1, "at least one freeze stage ran") + + // ===================================================================== + // STEP 3 — clean shutdown. The supervised loop returns nil on ctx cancel. + // ===================================================================== + waitClean(t, cancel, done) + + // Bind a fresh inspection catalog on the (now lock-free) data dir for the + // post-shutdown reads. It MUST be closed before the restart reopens the metastore. + postCat, closePost := e2eReadCatalog(t, dataDir) + + // The durable watermark, re-derived from post-shutdown state (the basis for the + // restart's resume-with-no-gap assertion). + wmBeforeRestart := mustDeriveWatermark(t, postCat) + require.GreaterOrEqual(t, wmBeforeRestart, c2First, "watermark advanced into chunk 2") + + // Live hot CF — now the daemon has stopped, chunk 2 (still the un-frozen live + // chunk) is reopenable. Resolve the chunk-2 tx hash through the txhash CF — the + // read path getTransaction uses for live history before a chunk freezes. + hotState, err := postCat.HotState(c2) + require.NoError(t, err) + require.Equal(t, geometry.HotReady, hotState, "chunk 2 is the un-frozen live chunk") + c2lfs, err := postCat.State(c2, geometry.KindLedgers) + require.NoError(t, err) + require.Equal(t, geometry.State(""), c2lfs, "the live chunk has no cold artifacts yet") + + // Retry the open: RocksDB's process-level LOCK can linger momentarily after the + // writer closed (the same transient a production reader retries through). + var liveDB *hotchunk.DB + require.Eventually(t, func() bool { + db, oerr := hotchunk.Open(cat.Layout().HotChunkPath(c2), c2, silentLogger()) + if oerr != nil { + return false + } + liveDB = db + return true + }, 10*time.Second, 50*time.Millisecond, "chunk 2's hot DB must be reopenable after shutdown") + hotSeq, err := liveDB.Txhash().Get(hotHash) + require.NoError(t, err, "the chunk-2 tx hash must resolve from the live hot CF") + assert.Equal(t, c2First, hotSeq, "hot lookup returns the live tx's ledger") + require.NoError(t, liveDB.Close()) // release before the restart reopens it as the live writer + + // ===================================================================== + // STEP 4 — RESTART. A fresh runDaemonWith re-opens everything, re-derives the + // watermark from durable state, and resumes captive core at watermark+1 with no gap. + // ===================================================================== + closePost() // release the inspection metastore handle before the daemon reopens it + core.opens.Store(0) + core.resumeSeen.Store(0) + core.fromSeen.Store(0) + cancel2, done2, _ := runDaemonInBackground(t, cfgPath, core, &served, &e2eMetrics{}) + + require.Eventually(t, func() bool { return core.opens.Load() >= 1 }, 30*time.Second, 20*time.Millisecond, + "the restarted daemon re-opened captive core") + require.Eventually(t, func() bool { return core.fromSeen.Load() != 0 }, 30*time.Second, 20*time.Millisecond, + "the restarted ingestion loop requested a resume range") + + wantResume := wmBeforeRestart + 1 + assert.Equal(t, wantResume, core.resumeSeen.Load(), + "restart resumes captive core at the re-derived watermark+1 (no gap, no re-fetch of the bottom)") + assert.Equal(t, wantResume, core.fromSeen.Load(), + "the ingestion loop streamed from watermark+1 — the durable frontier, re-derived not stored") + + waitClean(t, cancel2, done2) + + // ===================================================================== + // STEP 5 — retention prune. Re-run with retention_chunks = 1: the floor anchors + // at chunk 1, so chunk 0 (frozen + folded) falls WHOLLY below it and the prune + // scan sweeps its files + keys, while chunk 1 (the floor chunk) survives. A read + // of a pruned chunk-0 hash is then not-found (no coverage to resolve it). + // ===================================================================== + prunedCfg := e2eConfigPath(t, dataDir, 1) // retain ~1 chunk + prunedIdxPath := cat.Layout().TxHashIndexFilePath(frozenCov) + require.FileExists(t, prunedIdxPath, "chunk 0's cold index exists before the prune") + + cancel3, done3, catCh3 := runDaemonInBackground(t, prunedCfg, core, &served, &e2eMetrics{}) + pruneCat := awaitCatalog(t, catCh3) // the pruning daemon's own catalog + + // The prune scan runs on the first lifecycle tick (the at-start doorbell ring). + // Poll for chunk 0's per-chunk artifact keys (ledgers + events) to vanish. + require.Eventually(t, func() bool { + ledgers, err := pruneCat.State(c0, geometry.KindLedgers) + if err != nil { + return false + } + ev, err := pruneCat.State(c0, geometry.KindEvents) + if err != nil { + return false + } + return ledgers == geometry.State("") && ev == geometry.State("") + }, 60*time.Second, 50*time.Millisecond, "retention must prune chunk 0's artifact keys") + + // Chunk 1 (the floor chunk) is WITHIN retention and survives the prune. + c1lfs, err := pruneCat.State(c1, geometry.KindLedgers) + require.NoError(t, err) + assert.Equal(t, geometry.StateFrozen, c1lfs, "chunk 1 is at the retention floor and survives") + + // The on-disk cold index file is gone too (prune unlinks the files, not just keys). + require.Eventually(t, func() bool { + _, statErr := os.Stat(prunedIdxPath) + return os.IsNotExist(statErr) + }, 10*time.Second, 50*time.Millisecond, "the pruned cold index file is unlinked") + + // "pruned read is not-found": after prune the window has no frozen coverage + // (ok=false) — the read layer's "no coverage ⇒ not-found" gate. + _, covOK, err := pruneCat.FrozenTxHashIndex(w0) + require.NoError(t, err) + assert.False(t, covOK, "chunk 0's window coverage is pruned ⇒ a chunk-0 hash read is not-found") + + waitClean(t, cancel3, done3) +} + +// e2eReadCatalog binds a Catalog over a SEPARATE metastore handle on the daemon's +// data dir, with the same one-chunk window the daemon's test seam uses, for +// read-only inspection BETWEEN daemon runs (the metastore is RocksDB-primary, so +// this MUST be closed via the returned close func before the next daemon run). +func e2eReadCatalog(t *testing.T, dataDir string) (*catalog.Catalog, func()) { + t.Helper() + paths := Config{Service: ServiceConfig{DefaultDataDir: dataDir}}.WithDefaults().ResolvePaths() + store, err := openMetaAt(t, paths.Catalog) + require.NoError(t, err) + windows, err := geometry.NewTxHashIndexLayout(1) // matches chunksPerTxhashIndex = 1 + require.NoError(t, err) + return catalog.NewCatalog(store, NewLayoutFromPaths(paths), windows), func() { _ = store.Close() } +} + +// mustDeriveWatermark derives the durable watermark through the production probe. +func mustDeriveWatermark(t *testing.T, cat *catalog.Catalog) uint32 { + t.Helper() + wm, err := lifecycle.LastCommittedLedger(cat, NewRocksHotProbe(cat.Layout().HotChunkPath, silentLogger())) + require.NoError(t, err) + return wm +} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index 52bc80d66..c69c45ca6 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -242,12 +242,12 @@ func runLifecycleTick(ctx context.Context, cfg LifecycleConfig, cat *catalog.Cat // this many boundaries behind ingestion, a fatal condition notify() reports. const LifecycleQueueDepth = 8 -// lifecycleLoop is the event-driven lifecycle goroutine. Each notification carries +// RunLoop is the event-driven lifecycle goroutine. Each notification carries // the just-completed chunk id; the loop drains the buffer to the most-recent id // (one tick over [floor, lastChunk] subsumes the rest) and runs one tick. It // selects on both ctx.Done() and the channel, so it never blocks or fatals on // shutdown. -func lifecycleLoop(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog, ch <-chan chunk.ID) { +func RunLoop(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog, ch <-chan chunk.ID) { for { select { case <-ctx.Done(): diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go index dede88cf5..ed2d29e60 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go @@ -12,7 +12,7 @@ import ( ) // --------------------------------------------------------------------------- -// lifecycleLoop: selects on BOTH ctx.Done and the notification channel; drains +// RunLoop: selects on BOTH ctx.Done and the notification channel; drains // to the most-recent queued chunk id. // --------------------------------------------------------------------------- @@ -37,7 +37,7 @@ func TestLifecycleLoop_RunsTickPerNotifyThenStopsOnCtx(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { - lifecycleLoop(ctx, cfg, cat, ch) + RunLoop(ctx, cfg, cat, ch) close(done) }() @@ -77,7 +77,7 @@ func TestLifecycleLoop_DrainsToMostRecent(t *testing.T) { defer cancel() done := make(chan struct{}) go func() { - lifecycleLoop(ctx, cfg, cat, ch) + RunLoop(ctx, cfg, cat, ch) close(done) }() @@ -111,7 +111,7 @@ func TestLifecycleLoop_ReturnsImmediatelyOnAlreadyCancelledCtx(t *testing.T) { ch := make(chan chunk.ID) // unbuffered, never sent to done := make(chan struct{}) go func() { - lifecycleLoop(ctx, cfg, cat, ch) + RunLoop(ctx, cfg, cat, ch) close(done) }() select { diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index 6952ae8df..8a924ed3a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync" "time" "github.com/cenkalti/backoff/v4" @@ -15,9 +16,13 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) -// run is the daemon's startup: backfill to the tip, then serve reads (injected). -// Returns nil only on clean shutdown; any other return is restartable -// (ErrFirstStartNoTip on a first start with no reachable backend). +// run is the daemon's startup, in two steps: (1) CATCH UP via backfill to the +// tip, then (2) SERVE + INGEST — open the resume chunk's hot DB, start captive +// core (injected), launch the lifecycle goroutine on a doorbell, begin serving +// reads (injected), and run the live ingestion loop. Returns nil only on a clean +// shutdown (ctx cancelled mid-run, or the ingestion loop's clean stop); any other +// return is a restartable error the supervisor surfaces (ErrFirstStartNoTip on a +// first start with no reachable backend; a backfill/ingest failure; ErrHotVolumeLost). func run(ctx context.Context, cfg StartConfig) error { if err := cfg.validate(); err != nil { return err @@ -37,38 +42,103 @@ func run(ctx context.Context, cfg StartConfig) error { "(validateConfig pins it before run; not done here)") } - // Derived, never stored: highest durably-committed ledger, clamped by earliest-1. - // nil probe — catch-up startup reads the cold tier at chunk granularity (no hot DB). - lastCommitted, err := lifecycle.LastCommittedLedger(cat, nil) + // Derived, never stored: highest durably-committed ledger (frozen cold artifacts + // vs the highest ready hot DB's max committed seq), clamped by earliest-1. The + // probe does ONE read of the highest ready hot DB and detects hot-volume loss + // LAZILY on that open (ErrHotVolumeLost) before ingestion ever opens a writer. + lastCommitted, err := lifecycle.LastCommittedLedger(cat, cfg.Exec.Process.HotProbe) if err != nil { return fmt.Errorf("startup derive last-committed: %w", err) } metrics := observability.MetricsOrNop(cfg.Exec.Metrics) - metrics.LastCommitted(lastCommitted, lifecycle.EffectiveRetentionFloor(lastCommitted, cfg.RetentionChunks, earliest)) + metrics.LastCommitted(lastCommitted, lifecycle.EffectiveRetentionFloor(lastCommitted, cfg.Lifecycle.RetentionChunks, earliest)) logger.WithField("last_committed", lastCommitted). WithField("earliest", earliest). WithField("pinned", pinned). Info("startup — last-committed derived, beginning backfill") - // Step 1: backfill to the tip. + // Step 1: backfill (catch up) to the tip. lastCommitted, err = backfillToTip(ctx, cfg, lastCommitted, earliest) if err != nil { return err } logger.WithField("last_committed", lastCommitted). - Info("backfill complete — handing off to the read server") + WithField("resume_chunk", chunk.IDFromLedger(lastCommitted+1).String()). + Info("backfill complete — opening resume hot tier and ingesting") - // Step 2: serve (injected). Its error is restartable. + // Step 2: serve + ingest. resumeLedger is one past the watermark — the live + // chunk's next un-committed ledger; runIngestionLoop re-derives the exact resume + // point from durable state, so a mid-chunk and a boundary watermark both resume right. + resumeLedger := lastCommitted + 1 + resumeChunk := chunk.IDFromLedger(resumeLedger) + + hotDB, err := openHotTierForChunk(cat, resumeChunk, logger) + if err != nil { + return fmt.Errorf("startup open resume hot tier chunk %s: %w", resumeChunk, err) + } + + // Start captive core from the resume ledger. On failure the resume hot DB is + // already open; close it so a restart re-opens cleanly (the rocksdb LOCK must + // be released). + core, closeCore, err := cfg.Core.OpenCore(ctx, resumeLedger) + if err != nil { + _ = hotDB.Close() + return fmt.Errorf("startup start captive core at ledger %d: %w", resumeLedger, err) + } + defer func() { + if closeCore != nil { + _ = closeCore() + } + }() + + // The lifecycle goroutine runs one tick per notification carrying the just- + // completed chunk id. Buffered to LifecycleQueueDepth; ingestion sends at each + // boundary. It shares NO in-memory state with ingestion — all derived from durable keys. + lifecycleCh := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) + + // Seed the first tick with the last complete chunk at the resume point so it + // fires at once — clearing crash/downtime leftovers concurrently with serving. + // Skipped on a young network where no chunk is complete (the first real boundary + // triggers the first tick). + if seed := geometry.LastCompleteChunkAt(lastCommitted); seed >= 0 { + lifecycleCh <- chunk.ID(seed) //nolint:gosec // seed >= 0 + } + + // The lifecycle goroutine is tied to a PER-ITERATION child ctx (not the daemon + // ctx) and is cancelled + JOINED before run returns for ANY reason — restoring + // the single-lifecycle-goroutine invariant across supervisor restarts (a + // daemon-ctx-tied loop would survive a restartable return and run a tick + // concurrently with the next iteration's lifecycle + ingestion: two backfill + // passes truncating the same .pack/.idx). RunLoop checks ctx at every step, so + // the join cannot block past the current step. + lifecycleCtx, cancelLifecycle := context.WithCancel(ctx) + var lifecycleWG sync.WaitGroup + lifecycleWG.Add(1) + go func() { + defer lifecycleWG.Done() + lifecycle.RunLoop(lifecycleCtx, cfg.Lifecycle, cat, lifecycleCh) + }() + // The two return paths registered after this defer (the ingestion-loop return + // and the ServeReads error path) have no live sender on lifecycleCh — ingestion + // is a same-goroutine call whose inline notify has stopped, and the serve path + // never starts it — so no send can race the cancel. + defer func() { + cancelLifecycle() + lifecycleWG.Wait() + }() + + // Begin serving reads (injected). It must return promptly (launch, not block). if err := cfg.ServeReads(ctx); err != nil { + _ = hotDB.Close() return fmt.Errorf("startup serve reads: %w", err) } - // TODO(#772): production ServeReads is a no-op until the cutover, so an immediate - // clean exit after backfill is expected, not a misconfig. - logger.WithField("last_committed", lastCommitted). - Info("read server returned — cold-only daemon shutting down cleanly") - return nil + + // The ingestion loop owns hotDB for the rest of its life (closes it on any exit, + // reopens at each boundary). Returns the GetLedger/boundary error; the daemon top + // level classifies a ctx-cancelled return as a clean shutdown. + return runIngestionLoop(ctx, core, hotDB, cat, lifecycleCh, allHotTypes, logger, metrics, cfg.Exec.Process.Sink) } // backfillToTip runs the backfill loop, returning lastCommitted as backfill makes @@ -76,7 +146,7 @@ func run(ctx context.Context, cfg StartConfig) error { // computes [rangeStart, rangeEnd] with the mid-chunk exclusion, and breaks on an // empty or non-advancing range. func backfillToTip(ctx context.Context, cfg StartConfig, lastCommitted, earliest uint32) (uint32, error) { - retentionChunks := cfg.RetentionChunks + retentionChunks := cfg.Lifecycle.RetentionChunks metrics := observability.MetricsOrNop(cfg.Exec.Metrics) logger := cfg.Exec.Logger @@ -180,21 +250,34 @@ var ErrFirstStartNoTip = errors.New("network tip unavailable and no local histor // --------------------------------------------------------------------------- // NetworkTipBackend samples the bulk backend's current network tip during backfill. +// It is consulted only during catch-up; once ingestion runs, captive core is the tip. type NetworkTipBackend interface { NetworkTip(ctx context.Context) (uint32, error) } +// CoreOpener prepares captive core at resumeLedger and hands back a LedgerGetter +// the ingestion loop polls plus a closer the caller defers. Production wraps +// captive core's PrepareRange + GetLedger; tests pass a fake getter. +type CoreOpener interface { + OpenCore(ctx context.Context, resumeLedger uint32) (LedgerGetter, func() error, error) +} + // StartConfig is run's resolved dependency bundle. type StartConfig struct { // Exec drives backfill's RunBackfill; its Catalog/Logger are the shared ones. Exec backfill.ExecConfig - // RetentionChunks is the backfill floor's width; 0 ⇒ the earliest-ledger floor only. - RetentionChunks uint32 + // Lifecycle drives the lifecycle goroutine. Its embedded ExecConfig is the SAME + // wiring as Exec (one catalog, one pool); RetentionChunks is the catch-up floor's + // width too (0 ⇒ the earliest-ledger floor only). + Lifecycle lifecycle.LifecycleConfig // NetworkTip samples the bulk backend's tip during backfill. Required. NetworkTip NetworkTipBackend + // Core starts captive core and yields the ingestion getter. Required. + Core CoreOpener + // ServeReads begins serving reads; it must return promptly, not block. Required. ServeReads func(ctx context.Context) error @@ -212,9 +295,11 @@ const ( defaultTipMaxAttempts = 5 ) -// withDefaults fills the tip-backoff defaults and the embedded ExecConfig defaults. +// withDefaults fills the tip-backoff defaults and the embedded Exec/Lifecycle +// defaults (Workers -> GOMAXPROCS; lifecycle Fatalf -> log.Fatalf). func (cfg StartConfig) withDefaults() StartConfig { cfg.Exec = cfg.Exec.WithDefaults() + cfg.Lifecycle = cfg.Lifecycle.WithLifecycleDefaults() if cfg.TipBackoff <= 0 { cfg.TipBackoff = defaultTipBackoff } @@ -231,9 +316,15 @@ func (cfg StartConfig) validate() error { if cfg.Exec.Logger == nil { return errors.New("nil StartConfig.Exec.Logger") } + if cfg.Exec.Process.HotProbe == nil { + return errors.New("nil StartConfig.Exec.Process.HotProbe (last-committed derivation needs it)") + } if cfg.NetworkTip == nil { return errors.New("nil StartConfig.NetworkTip") } + if cfg.Core == nil { + return errors.New("nil StartConfig.Core") + } if cfg.ServeReads == nil { return errors.New("nil StartConfig.ServeReads") } diff --git a/cmd/stellar-rpc/internal/fullhistory/startup_test.go b/cmd/stellar-rpc/internal/fullhistory/startup_test.go index 5f2e65c94..17aabee47 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup_test.go @@ -13,6 +13,8 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) @@ -75,19 +77,36 @@ func (r *recordingPlan) snapshot() [][2]chunk.ID { return out } -// startTestConfig builds a cold StartConfig over a real catalog with faked -// boundaries; a non-nil recordPlan wires the runBackfill seam to record passes. +// startTestConfig builds a StartConfig over a real catalog with faked boundaries. +// core may be nil for backfillToTip tests (which call backfillToTip directly and +// never reach validate or the ingestion path); run() tests pass a fakeCore. A +// non-nil recordPlan wires the runBackfill seam to record passes without cold I/O. func startTestConfig( - t *testing.T, cat *catalog.Catalog, tip *fakeTipBackend, recordPlan *recordingPlan, + t *testing.T, cat *catalog.Catalog, tip *fakeTipBackend, core *fakeCore, recordPlan *recordingPlan, ) StartConfig { t.Helper() + exec := backfill.ExecConfig{ + Catalog: cat, + Logger: silentLogger(), + Workers: 2, + Process: backfill.ProcessConfig{ + HotProbe: NewRocksHotProbe(cat.Layout().HotChunkPath, silentLogger()), + }, + } cfg := StartConfig{ - Exec: backfill.ExecConfig{Catalog: cat, Logger: silentLogger(), Workers: 2}, - RetentionChunks: 0, - NetworkTip: tip, - ServeReads: func(context.Context) error { return nil }, - TipBackoff: time.Millisecond, - TipMaxAttempts: 3, + Exec: exec, + Lifecycle: lifecycle.LifecycleConfig{ + ExecConfig: exec, + RetentionChunks: 0, + // A tick op failure should fail the test loudly, not kill the process; the + // loop goroutine is joined before run() returns, so t.Errorf is safe here. + Fatalf: func(format string, args ...any) { t.Errorf("unexpected lifecycle fatal: "+format, args...) }, + }, + NetworkTip: tip, + Core: core, + ServeReads: func(context.Context) error { return nil }, + TipBackoff: time.Millisecond, + TipMaxAttempts: 3, } if recordPlan != nil { cfg.runBackfill = func(_ context.Context, _ backfill.ExecConfig, lo, hi chunk.ID) error { @@ -98,6 +117,30 @@ func startTestConfig( return cfg } +// fakeCore is a CoreOpener handing back a programmed LedgerGetter and recording +// the resume ledger it was started from. +type fakeCore struct { + getter LedgerGetter + openErr error + resumeSeen atomic.Uint32 + openedCount atomic.Int32 +} + +func (c *fakeCore) OpenCore(_ context.Context, resumeLedger uint32) (LedgerGetter, func() error, error) { + c.openedCount.Add(1) + c.resumeSeen.Store(resumeLedger) + if c.openErr != nil { + return nil, nil, c.openErr + } + getter := c.getter + if getter == nil { + // Default: a live getter that blocks until ctx is cancelled (the daemon's + // steady state). Tests that need a finite poll set c.getter. + getter = &fakeLedgerGetter{frames: map[uint32][]byte{}, blockOnCtx: true} + } + return getter, func() error { return nil }, nil +} + // pinGenesis pins earliest_ledger to genesis (as validateConfig does for a // "genesis" floor) so the first-start predicate classifies correctly. func pinGenesis(t *testing.T, cat *catalog.Catalog) { @@ -150,7 +193,7 @@ func TestBackfill_FirstStartTipAbsentFatal(t *testing.T) { cat, _ := testCatalog(t) pinGenesis(t, cat) tip := &fakeTipBackend{err: errors.New("backend unreachable"), errFirst: 99} - cfg := startTestConfig(t, cat, tip, &recordingPlan{}) + cfg := startTestConfig(t, cat, tip, nil, &recordingPlan{}) // Empty catalog ⇒ lastCommitted=1 < earliest=2 ⇒ first start with no progress. _, err := backfillToTip(context.Background(), cfg, preGenesisLedger, chunk.FirstLedgerSeq) @@ -167,7 +210,7 @@ func TestBackfill_FirstStartTipPresentComputesRange(t *testing.T) { tipLedger := chunk.ID(3).FirstLedger() + 100 rec := &recordingPlan{} tip := &fakeTipBackend{tips: []uint32{tipLedger}} - cfg := startTestConfig(t, cat, tip, rec) + cfg := startTestConfig(t, cat, tip, nil, rec) last, err := backfillToTip(context.Background(), cfg, preGenesisLedger, chunk.FirstLedgerSeq) require.NoError(t, err) @@ -186,7 +229,7 @@ func TestBackfill_YoungNetworkNoOp(t *testing.T) { // Tip inside chunk 0 (no chunk has fully closed yet). tip := &fakeTipBackend{tips: []uint32{chunk.FirstLedgerSeq + 50}} rec := &recordingPlan{} - cfg := startTestConfig(t, cat, tip, rec) + cfg := startTestConfig(t, cat, tip, nil, rec) last, err := backfillToTip(context.Background(), cfg, preGenesisLedger, chunk.FirstLedgerSeq) require.NoError(t, err) @@ -203,7 +246,7 @@ func TestBackfill_SteadyRestartNoOp(t *testing.T) { tipLedger := chunk.ID(3).FirstLedger() + 10 // last complete chunk == 2 rec := &recordingPlan{} tip := &fakeTipBackend{tips: []uint32{tipLedger}} - cfg := startTestConfig(t, cat, tip, rec) + cfg := startTestConfig(t, cat, tip, nil, rec) last, err := backfillToTip(context.Background(), cfg, lastCommitted, chunk.FirstLedgerSeq) require.NoError(t, err) @@ -224,7 +267,7 @@ func TestBackfill_MidChunkResumeExclusion(t *testing.T) { tipLedger := chunk.ID(5).LastLedger() // within one chunk, chunk 5 complete-at-tip rec := &recordingPlan{} tip := &fakeTipBackend{tips: []uint32{tipLedger}} - cfg := startTestConfig(t, cat, tip, rec) + cfg := startTestConfig(t, cat, tip, nil, rec) last, err := backfillToTip(context.Background(), cfg, lastCommitted, chunk.FirstLedgerSeq) require.NoError(t, err) @@ -251,7 +294,7 @@ func TestBackfill_LongDowntimeRePass(t *testing.T) { chunk.ID(6).FirstLedger() + 1, // last complete 5 }} rec := &recordingPlan{} - cfg := startTestConfig(t, cat, tip, rec) + cfg := startTestConfig(t, cat, tip, nil, rec) last, err := backfillToTip(context.Background(), cfg, preGenesisLedger, chunk.FirstLedgerSeq) require.NoError(t, err) @@ -274,7 +317,7 @@ func TestBackfill_RestartTipUnreachableDegrades(t *testing.T) { lastCommitted := chunk.ID(2).LastLedger() // local progress exists tip := &fakeTipBackend{err: errors.New("backend down"), errFirst: 99} rec := &recordingPlan{} - cfg := startTestConfig(t, cat, tip, rec) + cfg := startTestConfig(t, cat, tip, nil, rec) last, err := backfillToTip(context.Background(), cfg, lastCommitted, chunk.FirstLedgerSeq) require.NoError(t, err, "local progress means no fatal") @@ -295,7 +338,7 @@ func TestBackfill_LaggingBulkTipFoldsLastCommittedChunk(t *testing.T) { tipLedger := chunk.ID(3).FirstLedger() + 10 // lagging bulk tip in chunk 3 (last complete 2) rec := &recordingPlan{} tip := &fakeTipBackend{tips: []uint32{tipLedger}} - cfg := startTestConfig(t, cat, tip, rec) + cfg := startTestConfig(t, cat, tip, nil, rec) last, err := backfillToTip(context.Background(), cfg, lastCommitted, chunk.FirstLedgerSeq) require.NoError(t, err) @@ -308,57 +351,97 @@ func TestBackfill_LaggingBulkTipFoldsLastCommittedChunk(t *testing.T) { } // --------------------------------------------------------------------------- -// run — the backfill + serve flow. +// run — the backfill + serve + ingest flow. // --------------------------------------------------------------------------- -// A young-network first start does no backfill then serves reads once. -func TestRun_FirstStartBackfillThenServe(t *testing.T) { +// A young-network first start does no backfill, opens the resume hot DB, starts +// the (blocking) fake core, serves reads, and runs the ingestion loop — which +// surfaces the ctx-cancelled GetLedger error on a clean shutdown (the daemon top +// level classifies it as clean). The resume ledger is genesis (watermark+1). +func TestRun_FirstStartServeIngestCleanShutdown(t *testing.T) { cat, _ := testCatalog(t) pinGenesis(t, cat) served := atomic.Int32{} + core := &fakeCore{getter: &fakeLedgerGetter{frames: map[uint32][]byte{}, blockOnCtx: true}} tip := &fakeTipBackend{tips: []uint32{chunk.FirstLedgerSeq + 10}} // young: no backfill - cfg := startTestConfig(t, cat, tip, nil) + cfg := startTestConfig(t, cat, tip, core, nil) cfg.ServeReads = func(context.Context) error { served.Add(1); return nil } - require.NoError(t, run(context.Background(), cfg)) + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { errCh <- run(ctx, cfg) }() + + // Wait until the loop has opened the hot DB, started core, served, and parked on + // the blocking getter, then request a clean shutdown. + require.Eventually(t, func() bool { return served.Load() == 1 }, 2*time.Second, 5*time.Millisecond) + cancel() + + select { + case err := <-errCh: + require.ErrorIs(t, err, context.Canceled, "clean shutdown surfaces the ctx-cancelled error") + case <-time.After(3 * time.Second): + t.Fatal("run did not return after ctx cancel") + } + require.Equal(t, int32(1), served.Load(), "reads were served exactly once") + require.Equal(t, int32(1), core.openedCount.Load(), "captive core started once") + require.Equal(t, uint32(chunk.FirstLedgerSeq), core.resumeSeen.Load(), + "resume ledger is genesis on a fresh start (watermark+1)") + + // The resume chunk's hot key is "ready" (opened, boundary never crossed). + state, err := cat.HotState(chunk.IDFromLedger(chunk.FirstLedgerSeq)) + require.NoError(t, err) + assert.Equal(t, geometry.HotReady, state) } -// run surfaces a ServeReads error wrapped, as a restartable failure. +// A ServeReads error is surfaced wrapped as a restartable failure (NOT clean) and +// the already-opened resume hot DB is closed on the way out, so a restart can +// reopen it (the rocksdb LOCK is released). ServeReads runs after the hot DB opens +// and core starts but before the blocking ingestion loop, so run returns here. func TestRun_ServeReadsErrorSurfaces(t *testing.T) { cat, _ := testCatalog(t) pinGenesis(t, cat) + core := &fakeCore{getter: &fakeLedgerGetter{frames: map[uint32][]byte{}, blockOnCtx: true}} tip := &fakeTipBackend{tips: []uint32{chunk.FirstLedgerSeq + 10}} - cfg := startTestConfig(t, cat, tip, nil) + cfg := startTestConfig(t, cat, tip, core, nil) cfg.ServeReads = func(context.Context) error { return errors.New("rpc bind failed") } err := run(context.Background(), cfg) require.Error(t, err) require.Contains(t, err.Error(), "serve reads") + require.NotErrorIs(t, err, context.Canceled, "a ServeReads error is restartable, not a clean shutdown") + require.Equal(t, int32(1), core.openedCount.Load(), "core was started before serving") + + // The resume hot DB was closed on the error path (LOCK released): reopening it succeeds. + db, err := openHotTierForChunk(cat, chunk.IDFromLedger(chunk.FirstLedgerSeq), silentLogger()) + require.NoError(t, err, "the resume hot DB is reopenable — run released its LOCK") + require.NoError(t, db.Close()) } -// run fatals with ErrFirstStartNoTip on a first start with an -// unavailable tip; reads are never served. +// run fatals with ErrFirstStartNoTip on a first start with an unavailable tip; +// reads are never served and ingestion never starts. func TestRun_FirstStartNoTipFatal(t *testing.T) { cat, _ := testCatalog(t) pinGenesis(t, cat) served := atomic.Int32{} + core := &fakeCore{} tip := &fakeTipBackend{err: errors.New("unreachable"), errFirst: 99} - cfg := startTestConfig(t, cat, tip, nil) + cfg := startTestConfig(t, cat, tip, core, nil) cfg.ServeReads = func(context.Context) error { served.Add(1); return nil } err := run(context.Background(), cfg) require.ErrorIs(t, err, ErrFirstStartNoTip) require.Zero(t, served.Load(), "reads are never served when backfill fatals") + require.Zero(t, core.openedCount.Load(), "core never starts when backfill fatals") } -// run surfaces a missing earliest_ledger pin loudly (a wiring error, -// not a first start to mis-classify). +// run surfaces a missing earliest_ledger pin loudly (a wiring error, not a first +// start to mis-classify). func TestRun_RequiresEarliestPin(t *testing.T) { cat, _ := testCatalog(t) // No pinGenesis. - cfg := startTestConfig(t, cat, &fakeTipBackend{tips: []uint32{50_000}}, nil) + cfg := startTestConfig(t, cat, &fakeTipBackend{tips: []uint32{50_000}}, &fakeCore{}, nil) err := run(context.Background(), cfg) require.Error(t, err) require.Contains(t, err.Error(), "earliest_ledger pinned") @@ -367,13 +450,18 @@ func TestRun_RequiresEarliestPin(t *testing.T) { // run validates its injected boundaries. func TestRun_ValidatesConfig(t *testing.T) { cat, _ := testCatalog(t) - base := startTestConfig(t, cat, &fakeTipBackend{tips: []uint32{50_000}}, nil) + base := startTestConfig(t, cat, &fakeTipBackend{tips: []uint32{50_000}}, &fakeCore{}, nil) t.Run("nil NetworkTip", func(t *testing.T) { cfg := base cfg.NetworkTip = nil require.Error(t, run(context.Background(), cfg)) }) + t.Run("nil Core", func(t *testing.T) { + cfg := base + cfg.Core = nil + require.Error(t, run(context.Background(), cfg)) + }) t.Run("nil ServeReads", func(t *testing.T) { cfg := base cfg.ServeReads = nil @@ -436,7 +524,7 @@ func TestBackfill_ReportsPassAndProgress(t *testing.T) { rp := &recordingPlan{} tipLedger := chunk.ID(3).LastLedger() + 5 tip := &fakeTipBackend{tips: []uint32{tipLedger}} - start := startTestConfig(t, cat, tip, rp) + start := startTestConfig(t, cat, tip, nil, rp) metrics := newRecordingMetrics() start.Exec.Metrics = metrics From 4d19700f676f5ad35a2467408b89e4e771e2ef6a Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Tue, 30 Jun 2026 18:03:15 -0400 Subject: [PATCH 06/55] fullhistory/ingest: fix stale comments describing the deleted HotService errgroup fan-out The hot tier now ingests one ledger as a single atomic synced WriteBatch across all CFs (decision (a)); there is no per-type fan-out or errgroup. Update the doc.go tier description, the HotIngester interface doc (now the per-ledger ingest contract, also satisfied by the cold drain's ColdService), the Config doc, and the MetricSink/PrometheusSink help strings to match. Comments only. --- .../internal/fullhistory/ingest/config.go | 3 +- .../internal/fullhistory/ingest/doc.go | 12 +++---- .../internal/fullhistory/ingest/ingester.go | 31 ++++++++++--------- .../internal/fullhistory/ingest/metrics.go | 14 ++++----- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/config.go b/cmd/stellar-rpc/internal/fullhistory/ingest/config.go index 139f70d43..014c554c6 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/config.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/config.go @@ -3,8 +3,7 @@ package ingest import "errors" // Config selects which data types the ingest drivers write. At least one of -// Ledgers/Txhash/Events must be enabled. Per-ledger hot fan-out is always -// parallel; that is not configurable. +// Ledgers/Txhash/Events must be enabled. // // The view-based event path derives payloads from the LedgerCloseMetaView and // needs no network passphrase, so Config carries no passphrase. diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go b/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go index 5667214d9..ae42b0079 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go @@ -8,12 +8,12 @@ // Two tiers share the per-ledger extraction but differ in everything // else: // -// - Hot (RunHot): one chunk into the long-lived, caller-owned hot -// stores, from an injected ledgerbackend.LedgerStream. The stores -// are INJECTED and never opened or closed here, and neither is the -// stream; each ledger is durable before the next is pulled. -// Per-ledger fan-out across the enabled ingesters is concurrent -// (HotService). +// - Hot (HotService): one ledger at a time into the long-lived, +// caller-owned per-chunk hot DB, driven by the daemon's live +// ingestion loop. The DB is INJECTED and never opened or closed +// here. Each ledger is written as ONE atomic synced WriteBatch +// across all column families (decision (a) — no per-type fan-out), +// so a ledger is fully present or absent before the next is pulled. // - Cold (WriteColdChunk): one chunk into per-chunk cold artifacts // (ledger .pack, txhash .bin, events pack+index). It is // SOURCE-BLIND — the caller resolves the chunk's ledger source and diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go index d59453293..55a0e200f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go @@ -6,25 +6,26 @@ import ( "github.com/stellar/go-stellar-sdk/xdr" ) -// HotIngester ingests one data type for one ledger into a long-lived hot store. +// HotIngester ingests one ledger by sequence into a long-lived, caller-owned +// store. The hot tier's HotService implements it — one atomic synced WriteBatch +// across all column families per ledger (decision (a); NO per-type fan-out, no +// errgroup) — and the cold drain loop (drain) consumes the same shape, so +// ColdService satisfies it too. (The "Hot" name is historical: this is just the +// per-ledger ingest contract now.) // -// Ownership: the hot store is INJECTED into the ingester's constructor and owned -// by the caller (the daemon). The ingester does NOT open the store and does NOT -// close it — Close is intentionally absent from this interface. +// Ownership: the store is INJECTED into the implementation's constructor and +// owned by the caller (the daemon). The implementation does NOT open the store +// and does NOT close it — Close is intentionally absent from this interface. // // Input: seq is the DRIVER-VALIDATED ledger sequence of lcm — the drain loop // has already read it off the view and checked it against the chunk's expected -// position (duplicate / out-of-order / overrun), so ingesters consume it -// directly instead of each re-deriving and re-error-handling it. lcm is a -// zero-copy xdr.LedgerCloseMetaView (a []byte alias over the source stream's -// BORROWED buffer), valid only for the current iteration step; an ingester -// must copy any bytes it retains. The hot fan-out (HotService) waits for all -// ingesters to finish a ledger before the source pulls the next one, so -// synchronous consumption inside Ingest is safe. -// -// Concurrency: distinct HotIngester instances are run concurrently for the same -// ledger (HotService fans out via errgroup); each instance touches only its own -// store plus the read-only view. +// position (duplicate / out-of-order / overrun), so implementations consume it +// directly instead of re-deriving and re-error-handling it. lcm is a zero-copy +// xdr.LedgerCloseMetaView (a []byte alias over the source stream's BORROWED +// buffer), valid only for the current iteration step; an implementation must +// copy any bytes it retains. Ledgers are ingested sequentially — the source +// pulls the next only after Ingest returns — so synchronous consumption inside +// Ingest is safe. type HotIngester interface { Ingest(ctx context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error } diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go index 714458e93..86a019deb 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go @@ -37,9 +37,9 @@ const ( // wall-clock. A sink lets the same ingesters/services feed Prometheus in prod, // a CSV recorder in benchmarks, or a test recorder — interchangeably. // -// Implementations must be safe for concurrent use across ALL methods, not just -// HotIngest: the hot fan-out calls HotIngest/HotLedgerTotal from per-ledger -// goroutines, and a caller may freeze several chunks concurrently (each its own +// Implementations must be safe for concurrent use across ALL methods: the live +// hot ingestion loop reports HotIngest/HotLedgerTotal from its own goroutine +// while the lifecycle may freeze several chunks concurrently (each its own // WriteColdChunk), so the cold methods (ColdIngest, ColdChunkTotal) can likewise // be called from several goroutines at once. type MetricSink interface { @@ -52,8 +52,8 @@ type MetricSink interface { // wall-clock plus its Finalize, items the total items written for the chunk, // err the first error (nil on success). ColdIngest(dataType string, d time.Duration, items int, err error) - // HotLedgerTotal reports the per-ledger wall-clock across all hot ingesters - // (the HotService.Ingest fan-out duration). + // HotLedgerTotal reports the per-ledger wall-clock of one HotService.Ingest + // (the single atomic synced WriteBatch across all CFs). HotLedgerTotal(d time.Duration) // ColdChunkTotal reports the per-chunk wall-clock across all cold ingesters' // ingests plus their Finalizes (the ColdService lifetime). @@ -219,7 +219,7 @@ type PrometheusSink struct { coldStage map[string]prometheus.Observer hotStageVec *prometheus.HistogramVec coldStageVec *prometheus.HistogramVec - // Aggregate per-tier wall-clock: hot per-ledger fan-out, cold per-chunk + // Aggregate per-tier wall-clock: hot per-ledger Ingest, cold per-chunk // service lifetime. Separate histograms so each tier gets fitting buckets. hotLedgerTotal prometheus.Observer coldChunkTotal prometheus.Observer @@ -258,7 +258,7 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh hotLedgerTotal := prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: metricsSubsystem, Name: "hot_ledger_duration_seconds", - Help: "aggregate per-ledger wall-clock across all hot ingesters (HotService fan-out)", + Help: "per-ledger wall-clock of one HotService.Ingest (single atomic batch across all CFs)", Buckets: hotBuckets, }) From cfca87a7d65a7100c62157b8daeca4b313bbf001 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Tue, 30 Jun 2026 18:21:46 -0400 Subject: [PATCH 07/55] fullhistory/observability: drop cold_tier_bytes + per-ledger LedgerCommitted (apply #819 metrics cut) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-N review on #819 trimmed the control-plane metrics to a minimal set and explicitly dropped cold_tier_bytes (a ~30k-file tree walk; node_exporter statfs reports per-mount used bytes). The #820 rebase had re-introduced it (plus MeasureColdTierBytes and its per-pass/per-tick call sites) — remove it again. Also drop the per-ledger LedgerCommitted heartbeat: it only re-set the existing last_committed_ledger gauge more often (the lifecycle tick / backfill pass already set it) and added interface surface #816 doesn't require. Kept: live_hot_chunks (actionable discard-backlog signal), chunk_boundaries_total and discarded_hot_chunks_total (now driven by live Phase-2 code). --- .../fullhistory/backfill/recorder_test.go | 2 - .../internal/fullhistory/helpers_test.go | 2 - .../internal/fullhistory/hotloop.go | 4 -- .../fullhistory/lifecycle/lifecycle.go | 5 -- .../observability/observability.go | 63 +------------------ .../internal/fullhistory/startup.go | 7 --- 6 files changed, 1 insertion(+), 82 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go b/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go index 0a194dd2a..8b168cf0e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go @@ -42,11 +42,9 @@ func (r *recordingMetrics) Prune(count int, d time.Duration) { } func (*recordingMetrics) LastCommitted(uint32, uint32) {} -func (*recordingMetrics) LedgerCommitted(uint32) {} func (*recordingMetrics) ChunkBoundary(uint32) {} func (*recordingMetrics) BackfillPass(time.Duration) {} func (*recordingMetrics) LiveHotChunks(int) {} -func (*recordingMetrics) ColdTierBytes(int64) {} func (*recordingMetrics) Discard(int, time.Duration) {} var _ observability.Metrics = (*recordingMetrics)(nil) diff --git a/cmd/stellar-rpc/internal/fullhistory/helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/helpers_test.go index 44759ea07..44fa9c29e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/helpers_test.go @@ -114,13 +114,11 @@ func (r *recordingMetrics) BackfillPass(time.Duration) { r.backfillPasses++ } -func (*recordingMetrics) LedgerCommitted(uint32) {} func (*recordingMetrics) ChunkBoundary(uint32) {} func (*recordingMetrics) Freeze(time.Duration) {} func (*recordingMetrics) Rebuild(time.Duration) {} func (*recordingMetrics) Prune(int, time.Duration) {} func (*recordingMetrics) LiveHotChunks(int) {} -func (*recordingMetrics) ColdTierBytes(int64) {} func (*recordingMetrics) Discard(int, time.Duration) {} var _ observability.Metrics = (*recordingMetrics)(nil) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index ed271a391..b8a10b349 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -181,10 +181,6 @@ func runIngestionLoop( return fmt.Errorf("ingest ledger %d: %w", seq, ierr) } - // Per-ledger liveness gauge — the moving health signal a wedged ingester - // trips between boundaries (the tick-granular LastCommitted can't). - metrics.LedgerCommitted(seq) - // Chunk boundary: this seq is the chunk's last ledger. if seq == chunk.IDFromLedger(seq).LastLedger() { closed := chunk.IDFromLedger(seq) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index c69c45ca6..6327ea882 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -230,11 +230,6 @@ func runLifecycleTick(ctx context.Context, cfg LifecycleConfig, cat *catalog.Cat if logger != nil && len(pruneOps) > 0 { logger.WithField("pruned", len(pruneOps)).Info("streaming: lifecycle prune stage complete") } - - // Cold-tier footprint gauge after the prune stage. - if bytes, berr := observability.MeasureColdTierBytes(cat.Layout()); berr == nil { - metrics.ColdTierBytes(bytes) - } } // LifecycleQueueDepth is the notification buffer depth — far above the at-most-one diff --git a/cmd/stellar-rpc/internal/fullhistory/observability/observability.go b/cmd/stellar-rpc/internal/fullhistory/observability/observability.go index 0a111b85e..186a463be 100644 --- a/cmd/stellar-rpc/internal/fullhistory/observability/observability.go +++ b/cmd/stellar-rpc/internal/fullhistory/observability/observability.go @@ -1,14 +1,9 @@ package observability import ( - "io/fs" - "os" - "path/filepath" "time" "github.com/prometheus/client_golang/prometheus" - - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" ) // Metrics is the daemon's control-plane sink — the derived-progress gauges plus @@ -19,12 +14,6 @@ type Metrics interface { // retention floor (the two advance together each backfill pass / lifecycle tick). LastCommitted(lastCommitted, retentionFloor uint32) - // LedgerCommitted is the per-ledger liveness heartbeat the live-ingestion loop - // emits after each committed ledger: it advances the same last-committed gauge - // between lifecycle ticks (leaving the retention floor untouched mid-chunk), so - // a wedged ingester trips the gauge that the tick-granular LastCommitted can't. - LedgerCommitted(seq uint32) - // ChunkBoundary counts one ingestion chunk-boundary handoff (closedChunk = just-filled chunk id). ChunkBoundary(closedChunk uint32) @@ -33,9 +22,6 @@ type Metrics interface { // stage so the gauge tracks the live + awaiting-discard set. LiveHotChunks(count int) - // ColdTierBytes sets the cold-tier on-disk footprint. - ColdTierBytes(bytes int64) - // BackfillPass records one completed backfill pass's wall-clock. BackfillPass(d time.Duration) // Freeze records one freeze (plan-and-execute) stage's wall-clock. @@ -52,10 +38,8 @@ type Metrics interface { type NopMetrics struct{} func (NopMetrics) LastCommitted(uint32, uint32) {} -func (NopMetrics) LedgerCommitted(uint32) {} func (NopMetrics) ChunkBoundary(uint32) {} func (NopMetrics) LiveHotChunks(int) {} -func (NopMetrics) ColdTierBytes(int64) {} func (NopMetrics) BackfillPass(time.Duration) {} func (NopMetrics) Freeze(time.Duration) {} func (NopMetrics) Rebuild(time.Duration) {} @@ -86,7 +70,6 @@ type PrometheusMetrics struct { lastCommitted prometheus.Gauge retentionFloor prometheus.Gauge liveHotChunks prometheus.Gauge - coldTierBytes prometheus.Gauge // Counters — monotonic tallies. chunkBoundaries prometheus.Counter @@ -123,7 +106,6 @@ func NewPrometheusMetrics(registry *prometheus.Registry, namespace string) *Prom lastCommitted: gauge("last_committed_ledger", "highest ledger durably committed"), retentionFloor: gauge("retention_floor_ledger", "effective retention floor — lowest in-window ledger"), liveHotChunks: gauge("live_hot_chunks", "count of hot-chunk DBs currently on disk"), - coldTierBytes: gauge("cold_tier_bytes", "cold-tier on-disk footprint in bytes"), chunkBoundaries: counter("chunk_boundaries_total", "ingestion chunk-boundary handoffs"), discarded: counter("discarded_hot_chunks_total", "hot DBs retired by the discard stage"), pruned: counter("pruned_ops_total", "artifacts swept after an index build"), @@ -135,7 +117,7 @@ func NewPrometheusMetrics(registry *prometheus.Registry, namespace string) *Prom } registry.MustRegister( - m.lastCommitted, m.retentionFloor, m.liveHotChunks, m.coldTierBytes, + m.lastCommitted, m.retentionFloor, m.liveHotChunks, m.chunkBoundaries, m.discarded, m.pruned, m.phaseDuration, ) @@ -147,14 +129,10 @@ func (m *PrometheusMetrics) LastCommitted(lastCommitted, retentionFloor uint32) m.retentionFloor.Set(float64(retentionFloor)) } -func (m *PrometheusMetrics) LedgerCommitted(seq uint32) { m.lastCommitted.Set(float64(seq)) } - func (m *PrometheusMetrics) ChunkBoundary(uint32) { m.chunkBoundaries.Inc() } func (m *PrometheusMetrics) LiveHotChunks(count int) { m.liveHotChunks.Set(float64(count)) } -func (m *PrometheusMetrics) ColdTierBytes(bytes int64) { m.coldTierBytes.Set(float64(bytes)) } - func (m *PrometheusMetrics) BackfillPass(d time.Duration) { m.phaseDuration.WithLabelValues(phaseBackfillPass).Observe(d.Seconds()) } @@ -183,42 +161,3 @@ func (m *PrometheusMetrics) Prune(count int, d time.Duration) { // compile-time interface check. var _ Metrics = (*PrometheusMetrics)(nil) - -// MeasureColdTierBytes sums the cold tier's on-disk footprint (ledgers/events/txhash-raw/ -// txhash-index trees; hot tier and meta store excluded), walking each root once and -// ignoring missing trees. A per-tree error is non-fatal to the others (caller skips the gauge). -func MeasureColdTierBytes(layout geometry.Layout) (int64, error) { - var total int64 - var firstErr error - for _, root := range []string{ - layout.LedgersRoot(), - layout.EventsRoot(), - layout.TxHashRawRoot(), - layout.TxHashIndexRoot(), - } { - err := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error { - if err != nil { - if os.IsNotExist(err) { - return nil // an un-materialized tree contributes nothing - } - return err - } - if d.IsDir() { - return nil - } - info, ierr := d.Info() - if ierr != nil { - if os.IsNotExist(ierr) { - return nil // raced with a prune unlink — count it as gone - } - return ierr - } - total += info.Size() - return nil - }) - if err != nil && firstErr == nil { - firstErr = err - } - } - return total, firstErr -} diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index 8a924ed3a..b5ce18028 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -212,13 +212,6 @@ func backfillToTip(ctx context.Context, cfg StartConfig, lastCommitted, earliest metrics.BackfillPass(passDuration) // Refresh the derived gauges as last-committed advances and the floor rises with it. metrics.LastCommitted(lastCommitted, lifecycle.EffectiveRetentionFloor(lastCommitted, retentionChunks, earliest)) - // Sample the cold-tier footprint once per pass (a full tree-walk is too costly - // per-chunk); a walk error just leaves the gauge at its last value. - if footprint, cerr := observability.MeasureColdTierBytes(cfg.Exec.Catalog.Layout()); cerr == nil { - metrics.ColdTierBytes(footprint) - } else { - logger.WithError(cerr).Debug("cold-tier footprint sample failed; skipping gauge") - } logger.WithField("range_lo", rangeStart.String()). WithField("range_hi", rangeEnd.String()). WithField("last_committed", lastCommitted). From f8c8cece5c7240a4702a6b4137444b53351cda31 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Tue, 30 Jun 2026 18:38:39 -0400 Subject: [PATCH 08/55] fullhistory: /simplify pass on the layer-2 test code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reuse: oneTxLCMBytes now returns the committed tx hash, so the E2E reuses it instead of the ~50-line duplicate oneTxLCMReturningHash (deleted, with the now-unused e2ePassphrase const + network import). - Simplify: e2eCore holds frames as a seq→raw map directly (drop the e2eFrame slice + the per-open rebuild); e2eGetter takes a *e2eCore back-ref instead of aliased *atomic pointers; e2eMetrics counts boundaries (int) instead of retaining a []uint32 the test only len()s. Test-only; production code unchanged. E2E + backfill-materializes green. --- .../internal/fullhistory/daemon_test.go | 11 +- .../internal/fullhistory/e2e_test.go | 157 +++++------------- 2 files changed, 46 insertions(+), 122 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon_test.go b/cmd/stellar-rpc/internal/fullhistory/daemon_test.go index f512670d9..b263147a9 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon_test.go @@ -114,7 +114,8 @@ func someTxBackend(t *testing.T) *fakeBackend { if seq%2500 != 0 { return zeroTxLCMBytes(t, seq) } - return oneTxLCMBytes(t, seq, src) + raw, _ := oneTxLCMBytes(t, seq, src) + return raw } return &fakeBackend{ LedgerStream: &fullChunkStream{t: t, gen: gen}, @@ -124,8 +125,10 @@ func someTxBackend(t *testing.T) *fakeBackend { } // oneTxLCMBytes is zeroTxLCMBytes plus one tx (per-seq SeqNum ⇒ unique hash) so -// ExtractTxHashes yields exactly one key for seq. -func oneTxLCMBytes(t *testing.T, seq uint32, src xdr.MuxedAccount) []byte { +// ExtractTxHashes yields exactly one key for seq. Returns the wire bytes and the +// real, network-hashed transaction hash (the hash the daemon commits for seq), so +// callers can assert a getTransaction-style hash→seq lookup. +func oneTxLCMBytes(t *testing.T, seq uint32, src xdr.MuxedAccount) ([]byte, [32]byte) { t.Helper() envelope := xdr.TransactionEnvelope{ Type: xdr.EnvelopeTypeEnvelopeTypeTx, @@ -177,7 +180,7 @@ func oneTxLCMBytes(t *testing.T, seq uint32, src xdr.MuxedAccount) []byte { } raw, err := lcm.MarshalBinary() require.NoError(t, err) - return raw + return raw, hash } // #815 acceptance: one TOML boots the daemon and it backfills the complete chunk diff --git a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go index ada5d1ac8..befbe9300 100644 --- a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go @@ -45,7 +45,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stellar/go-stellar-sdk/keypair" - "github.com/stellar/go-stellar-sdk/network" "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" @@ -58,100 +57,45 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash" ) -// e2ePassphrase is the network passphrase the synthetic tx hashes are computed -// against. Any stable value works; the index only needs deterministic hashes. -const e2ePassphrase = network.PublicNetworkPassphrase - -// oneTxLCMReturningHash builds a well-formed V2 LedgerCloseMeta carrying exactly -// ONE transaction for seq and returns BOTH the wire bytes and the real, -// network-hashed transaction hash. A non-zero-tx ledger is required somewhere in -// a chunk so its txhash .bin is non-empty (streamhash refuses a zero-key cold -// index, txhash.ErrEmptyBuildSet); returning the hash lets the E2E assert the -// getTransaction-style hash→seq lookup against a hash the daemon really committed. -func oneTxLCMReturningHash(t *testing.T, seq uint32) ([]byte, [32]byte) { - t.Helper() - envelope := xdr.TransactionEnvelope{ - Type: xdr.EnvelopeTypeEnvelopeTypeTx, - V1: &xdr.TransactionV1Envelope{ - Tx: xdr.Transaction{ - SourceAccount: xdr.MustMuxedAddress(keypair.MustRandom().Address()), - Ext: xdr.TransactionExt{V: 1, SorobanData: &xdr.SorobanTransactionData{}}, - }, - }, - } - hash, err := network.HashTransactionInEnvelope(envelope, e2ePassphrase) - require.NoError(t, err) - - comp := []xdr.TxSetComponent{{ - Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, - TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ - Txs: []xdr.TransactionEnvelope{envelope}, - }, - }} - opResults := []xdr.OperationResult{} - lcm := xdr.LedgerCloseMeta{ - V: 2, - V2: &xdr.LedgerCloseMetaV2{ - LedgerHeader: xdr.LedgerHeaderHistoryEntry{ - Header: xdr.LedgerHeader{ - ScpValue: xdr.StellarValue{CloseTime: xdr.TimePoint(0)}, - LedgerSeq: xdr.Uint32(seq), - }, - }, - TxSet: xdr.GeneralizedTransactionSet{ - V: 1, - V1TxSet: &xdr.TransactionSetV1{Phases: []xdr.TransactionPhase{{V: 0, V0Components: &comp}}}, - }, - TxProcessing: []xdr.TransactionResultMetaV1{{ - TxApplyProcessing: xdr.TransactionMeta{ - V: 4, - V4: &xdr.TransactionMetaV4{Operations: []xdr.OperationMetaV2{}}, - }, - Result: xdr.TransactionResultPair{ - TransactionHash: hash, - Result: xdr.TransactionResult{ - FeeCharged: 100, - Result: xdr.TransactionResultResult{Code: xdr.TransactionResultCodeTxSuccess, Results: &opResults}, - }, - }, - }}, - }, - } - raw, err := lcm.MarshalBinary() - require.NoError(t, err) - return raw, hash +// e2eCore is the CoreOpener handing back a fresh e2eGetter per daemon run (a +// restart opens core anew). frames is the seq→raw backlog every getter serves; +// the atomics aggregate observations across opens for the restart assertions. +type e2eCore struct { + frames map[uint32][]byte + resumeSeen atomic.Uint32 + fromSeen atomic.Uint32 + delivered atomic.Uint32 + opens atomic.Int32 } -// e2eFrame is one synthetic ledger the fake core serves. -type e2eFrame struct { - seq uint32 - raw []byte +func (c *e2eCore) OpenCore(_ context.Context, resume uint32) (LedgerGetter, func() error, error) { + c.opens.Add(1) + c.resumeSeen.Store(resume) + return &e2eGetter{core: c}, func() error { return nil }, nil } // e2eGetter is the FAKE captive-core ledger getter: a resumable LedgerGetter the // ingestion loop polls by sequence. It returns the frame for the requested seq -// when it has one, and once the poll runs past the synthetic backlog it blocks -// until ctx is cancelled (a live tip stream ends only on shutdown). It records -// the FIRST seq it was asked for so the restart step can assert the daemon -// re-derived the watermark and resumed with no gap. +// when its core has one, and once the poll runs past the synthetic backlog it +// blocks until ctx is cancelled (a live tip stream ends only on shutdown). It +// records (into its core) the FIRST seq it was asked for, so the restart step can +// assert the daemon re-derived the watermark and resumed with no gap. type e2eGetter struct { - frames map[uint32][]byte - fromSeen *atomic.Uint32 // first GetLedger seq (for the restart assertion) - delivered *atomic.Uint32 // highest seq actually yielded (test sync) - sawFrom atomic.Bool + core *e2eCore + sawFrom atomic.Bool } var _ LedgerGetter = (*e2eGetter)(nil) func (s *e2eGetter) GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseMetaView, error) { if s.sawFrom.CompareAndSwap(false, true) { - s.fromSeen.Store(seq) + s.core.fromSeen.Store(seq) } if ctx.Err() != nil { return nil, ctx.Err() } - if raw, ok := s.frames[seq]; ok { - s.delivered.Store(seq) + if raw, ok := s.core.frames[seq]; ok { + s.core.delivered.Store(seq) return xdr.LedgerCloseMetaView(raw), nil } // Past the synthetic backlog: a live tip blocks until shutdown so the loop @@ -160,40 +104,19 @@ func (s *e2eGetter) GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseM return nil, ctx.Err() } -// e2eCore is the CoreOpener handing back a fresh e2eGetter per daemon run (a -// restart opens core anew). It records the resume ledger every open was driven from. -type e2eCore struct { - frames []e2eFrame - resumeSeen atomic.Uint32 - fromSeen atomic.Uint32 - delivered atomic.Uint32 - opens atomic.Int32 -} - -func (c *e2eCore) OpenCore(_ context.Context, resume uint32) (LedgerGetter, func() error, error) { - c.opens.Add(1) - c.resumeSeen.Store(resume) - byseq := make(map[uint32][]byte, len(c.frames)) - for _, f := range c.frames { - byseq[f.seq] = f.raw - } - getter := &e2eGetter{frames: byseq, fromSeen: &c.fromSeen, delivered: &c.delivered} - return getter, func() error { return nil }, nil -} - -// e2eMetrics is a concurrency-safe observability.Metrics that records the chunk +// e2eMetrics is a concurrency-safe observability.Metrics that counts the chunk // boundaries and freezes the daemon emits (the rest discarded via NopMetrics). type e2eMetrics struct { observability.NopMetrics mu sync.Mutex - boundaries []uint32 + boundaries int freezes int } -func (m *e2eMetrics) ChunkBoundary(closed uint32) { +func (m *e2eMetrics) ChunkBoundary(uint32) { m.mu.Lock() defer m.mu.Unlock() - m.boundaries = append(m.boundaries, closed) + m.boundaries++ } func (m *e2eMetrics) Freeze(time.Duration) { @@ -202,12 +125,10 @@ func (m *e2eMetrics) Freeze(time.Duration) { m.freezes++ } -func (m *e2eMetrics) snapshotBoundaries() []uint32 { +func (m *e2eMetrics) boundaryCount() int { m.mu.Lock() defer m.mu.Unlock() - out := make([]uint32, len(m.boundaries)) - copy(out, m.boundaries) - return out + return m.boundaries } func (m *e2eMetrics) snapshotFreezeCount() int { @@ -354,26 +275,26 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing c1First := c1.FirstLedger() c2First := c2.FirstLedger() - coldRaw, coldHash := oneTxLCMReturningHash(t, c0First) // → frozen cold .idx (chunk 0) - hotRaw, hotHash := oneTxLCMReturningHash(t, c2First) // → live hot CF (chunk 2) + // One shared source account; the per-seq SeqNum makes each tx hash unique. + src := xdr.MustMuxedAddress(keypair.MustRandom().Address()) + coldRaw, coldHash := oneTxLCMBytes(t, c0First, src) // → frozen cold .idx (chunk 0) + hotRaw, hotHash := oneTxLCMBytes(t, c2First, src) // → live hot CF (chunk 2) // Chunk 1's first ledger also carries a tx so its txhash .bin is non-empty — // streamhash refuses to build a cold index over zero keys (ErrEmptyBuildSet). - c1Raw, _ := oneTxLCMReturningHash(t, c1First) + c1Raw, _ := oneTxLCMBytes(t, c1First, src) - frames := make([]e2eFrame, 0, 2*int(chunk.LedgersPerChunk)+2) + frames := make(map[uint32][]byte, 2*int(chunk.LedgersPerChunk)+2) appendLedger := func(seq uint32) { - var raw []byte switch seq { case c0First: - raw = coldRaw + frames[seq] = coldRaw case c1First: - raw = c1Raw + frames[seq] = c1Raw case c2First: - raw = hotRaw + frames[seq] = hotRaw default: - raw = zeroTxLCMBytes(t, seq) + frames[seq] = zeroTxLCMBytes(t, seq) } - frames = append(frames, e2eFrame{seq: seq, raw: raw}) } // Chunks 0 and 1 in full (both freeze), then chunk 2's first two ledgers. for seq := c0First; seq <= c1.LastLedger(); seq++ { @@ -461,7 +382,7 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing require.NoError(t, coldReader.Close()) // Observability: the daemon emitted the boundary + freeze phase signals. - assert.GreaterOrEqual(t, len(metrics.snapshotBoundaries()), 1, "at least one chunk boundary was signaled") + assert.GreaterOrEqual(t, metrics.boundaryCount(), 1, "at least one chunk boundary was signaled") assert.GreaterOrEqual(t, metrics.snapshotFreezeCount(), 1, "at least one freeze stage ran") // ===================================================================== From 5b497b30c7a553ad6fa3f56df6b4696918c6b91d Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Tue, 30 Jun 2026 18:51:39 -0400 Subject: [PATCH 09/55] fullhistory: align prose to the design's vocabulary (backfill, last-committed) Standardize comment terminology to the design doc's canonical terms (the same catch-up/backfill synonym flagged on the earlier phase): - catch-up -> backfill (the doc uses "backfill" 42x, "catch" 1x) - watermark -> last-committed ledger (the doc's term for the derived bound; "watermark" was used only informally) Comments only; all occurrences were in files #820 creates or already modifies. --- .../internal/fullhistory/backfill/process.go | 4 ++-- .../internal/fullhistory/catalog/catalog.go | 4 ++-- cmd/stellar-rpc/internal/fullhistory/daemon.go | 2 +- cmd/stellar-rpc/internal/fullhistory/hotloop.go | 8 ++++---- cmd/stellar-rpc/internal/fullhistory/hotsource.go | 2 +- .../internal/fullhistory/lifecycle/lifecycle.go | 8 ++++---- .../internal/fullhistory/lifecycle/progress.go | 2 +- .../internal/fullhistory/lifecycle/retention.go | 2 +- .../fullhistory/pkg/stores/hotchunk/hotchunk.go | 4 ++-- cmd/stellar-rpc/internal/fullhistory/startup.go | 12 ++++++------ 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go index 8e0465f02..2ae3a7b25 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go @@ -42,7 +42,7 @@ type HotProbe interface { // HotChunk is one chunk's opened hot tier: the single DB's completeness gate plus // an LCM source over the ledgers CF. type HotChunk interface { - // MaxCommittedSeq is the single authoritative watermark (decision (a)); + // MaxCommittedSeq is the single authoritative last-committed ledger (decision (a)); // ok=false on an empty DB (so the chunk cannot be complete). MaxCommittedSeq() (seq uint32, ok bool, err error) // Source yields the chunk's LCMs from the ledgers CF as a LedgerStream the cold @@ -59,7 +59,7 @@ type ProcessConfig struct { Sink ingest.MetricSink // HotProbe opens the hot tier for backfillSource's hot branch. Nil (cold-only - // catch-up or a hot-less test) skips that branch — pack/backend sources only. + // backfill or a hot-less test) skips that branch — pack/backend sources only. HotProbe HotProbe // Backend is the bulk source for a chunk with no local copy (BSB now, captive diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go index 292dd444e..84536182e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go @@ -58,7 +58,7 @@ func (c *Catalog) State(chunkID chunk.ID, kind geometry.Kind) (geometry.State, e // HotState returns the HotState of a chunk's hot-DB key, or empty (key absent). // The key's mere existence (any value) marks the chunk as owned by ingestion; -// only the watermark derivation cares which value (see ReadyHotChunkKeys). +// only the last-committed ledger derivation cares which value (see ReadyHotChunkKeys). func (c *Catalog) HotState(chunkID chunk.ID) (geometry.HotState, error) { v, ok, err := c.get(geometry.HotChunkKey(chunkID)) if err != nil || !ok { @@ -103,7 +103,7 @@ func (c *Catalog) HotChunkKeys() ([]chunk.ID, error) { } // ReadyHotChunkKeys returns only the chunks whose hot-DB key is "ready", sorted -// ascending. The watermark counts only these — a "transient" key never advances +// ascending. The last-committed ledger counts only these — a "transient" key never advances // the bound, which lets recovery demote any hot key without disturbing it. func (c *Catalog) ReadyHotChunkKeys() ([]chunk.ID, error) { return c.hotChunkKeysWith(func(s geometry.HotState) bool { return s == geometry.HotReady }) diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index 89ef6b79e..41418132d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -181,7 +181,7 @@ func runDaemonWith(ctx context.Context, configPath string, opts daemonOptions) e } // startConfig assembles the StartConfig run consumes. Exec and Lifecycle share -// ONE catalog, worker pool, and retention floor (catch-up and the lifecycle +// ONE catalog, worker pool, and retention floor (backfill and the lifecycle // goroutine share one set of postconditions), so Lifecycle embeds the same exec. func startConfig( cfg Config, cat *catalog.Catalog, logger *supportlog.Entry, diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index b8a10b349..84487414c 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -22,7 +22,7 @@ import ( // The hot-DB ingestion loop (decision (a)). One goroutine polls ledgers by seq // (core.GetLedger) into the per-chunk shared multi-CF hot DB, committing each as // one atomic synced WriteBatch across all CFs. It keeps NO progress variable — -// the last synced batch IS the watermark, re-derived at startup. Its only +// the last synced batch IS the last-committed ledger, re-derived at startup. Its only // coupling to the lifecycle is the channel: at each boundary it sends the // just-completed chunk id (the two goroutines share no memory). Clean-shutdown vs // crash is decided at the daemon top level (a ctx-cancelled return is clean). @@ -113,7 +113,7 @@ func openHotTierForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *support // WriteBatch each), and at each chunk boundary hands the frontier forward by // closing the just-filled DB and opening the next. It never returns nil; the // daemon classifies a ctx-cancelled return as clean shutdown, any other as -// RESTARTABLE (startup re-derives the watermark, losing nothing). +// RESTARTABLE (startup re-derives the last-committed ledger, losing nothing). // // HANDOFF FENCE: the DB is CLOSED before the next chunk's hot:chunk key is // created — that key is what makes THIS chunk complete to the lifecycle, which @@ -156,7 +156,7 @@ func runIngestionLoop( } }() - // Resume point: one past the live chunk's durable watermark (re-derived, not + // Resume point: one past the live chunk's durable last-committed ledger (re-derived, not // stored — a re-delivered committed ledger is an idempotent retry). resume, err := nextIngestLedger(hotDB) if err != nil { @@ -215,7 +215,7 @@ func runIngestionLoop( } // nextIngestLedger is the resume point for a just-opened live hot DB: one past -// its authoritative watermark, or the bound chunk's first ledger on an empty DB. +// its authoritative last-committed ledger, or the bound chunk's first ledger on an empty DB. func nextIngestLedger(db *hotchunk.DB) (uint32, error) { maxSeq, ok, err := db.MaxCommittedSeq() if err != nil { diff --git a/cmd/stellar-rpc/internal/fullhistory/hotsource.go b/cmd/stellar-rpc/internal/fullhistory/hotsource.go index d6abc55ed..c439c4dc8 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotsource.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotsource.go @@ -60,7 +60,7 @@ type rocksHotChunk struct { db *hotchunk.DB } -// MaxCommittedSeq returns the single authoritative watermark (decision (a)): the +// MaxCommittedSeq returns the single authoritative last-committed ledger (decision (a)): the // highest ledger seq the shared DB has durably committed. ok=false on an empty DB. func (h *rocksHotChunk) MaxCommittedSeq() (uint32, bool, error) { seq, ok, err := h.db.MaxCommittedSeq() diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index 6327ea882..e14ca20ce 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -13,7 +13,7 @@ import ( ) // The lifecycle tick runs three stages in order: (1) plan-and-execute (the same -// resolve+executePlan as catch-up, over [floor, lastChunk]); (2) discard scan; +// resolve+executePlan as backfill, over [floor, lastChunk]); (2) discard scan; // (3) prune scan. The tick is a pure function of the catalog — the two goroutines // share no state. // @@ -23,10 +23,10 @@ import ( // PRODUCTION boundary erring low is DANGEROUS (it would plan a build below // existing storage from an unvalidated source). So the plan range never starts // below storage — start is RAISED to lowestMaterializedChunk; extending the -// bottom is catch-up's job, producibility enforced lazily per chunk. +// bottom is backfill's job, producibility enforced lazily per chunk. // LifecycleConfig bundles the tick/loop dependencies. It composes the scheduler's -// ExecConfig (shared postconditions + worker pool with catch-up) plus the +// ExecConfig (shared postconditions + worker pool with backfill) plus the // retention knob and an injectable fatal sink. type LifecycleConfig struct { backfill.ExecConfig @@ -161,7 +161,7 @@ func runLifecycleTick(ctx context.Context, cfg LifecycleConfig, cat *catalog.Cat rangeEnd = highestComplete } if haveComplete && start >= 0 && start <= int64(rangeEnd) { - // Plan-and-execute over [start, rangeEnd] via the same entry point catch-up + // Plan-and-execute over [start, rangeEnd] via the same entry point backfill // uses (resolve → executePlan → Freeze metric, recorded internally). if eerr := backfill.RunBackfill(ctx, cfg.ExecConfig, chunk.ID(start), rangeEnd); eerr != nil { //nolint:gosec // start >= 0 // CLEAN-SHUTDOWN: a cancelled ctx makes RunBackfill return ctx.Err() — diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go index d0553b7c9..bd66d22a8 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go @@ -13,7 +13,7 @@ import ( // int64 (-1 = "nothing complete") to avoid uint32 wraparound on the pre-genesis // sentinel; CompleteThrough is the chokepoint. -// preGenesisLedger is the watermark when nothing is complete (FirstLedgerSeq-1). +// preGenesisLedger is the last-committed ledger when nothing is complete (FirstLedgerSeq-1). const preGenesisLedger uint32 = chunk.FirstLedgerSeq - 1 // CompleteThrough maps a signed chunk index to its "complete through" last ledger: diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention.go index 852b44976..951d54bb7 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention.go @@ -12,7 +12,7 @@ import ( // instant it passes the floor without coordinating with the index lifecycle. The // floor may err LOW harmlessly (a wrongly-retained chunk still hits the reader's // missing-file rule), so it anchors on the live CompleteThrough; widening history -// is catch-up's job, not the floor's. +// is backfill's job, not the floor's. type RetentionFloor struct { chunk chunk.ID // lowest in-retention chunk } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index 5b155bfed..dffe212bf 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -2,7 +2,7 @@ // RocksDB holding the union of every hot data type's CFs (ledger + 3 events + 16 // nibble-routed txhash), and each ledger commits as ONE atomic synced WriteBatch // across ALL of them — so a ledger is fully present or fully absent, with a -// SINGLE per-chunk watermark (max committed seq, from the ledgers CF's last key) +// SINGLE per-chunk last-committed ledger (max committed seq, from the ledgers CF's last key) // and no per-store frontiers / min-of-three. The three typed facades // (ledger/txhash/eventstore HotStore) are composed over the shared store via // NewWithStore; their write paths queue Puts into the one shared batch. @@ -121,7 +121,7 @@ func (d *DB) Events() *eventstore.HotStore { return d.events } // concurrently with in-flight reads/writes. func (d *DB) Close() error { return d.store.Close() } -// MaxCommittedSeq returns the single authoritative per-chunk watermark: the +// MaxCommittedSeq returns the single authoritative per-chunk last-committed ledger: the // highest seq durably committed, from the ledgers CF's last key. Under decision // (a) this one value pins EVERY CF's frontier. ok=false on an empty DB. func (d *DB) MaxCommittedSeq() (seq uint32, ok bool, err error) { diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index b5ce18028..113554453 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -16,7 +16,7 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) -// run is the daemon's startup, in two steps: (1) CATCH UP via backfill to the +// run is the daemon's startup, in two steps: (1) BACKFILL to the // tip, then (2) SERVE + INGEST — open the resume chunk's hot DB, start captive // core (injected), launch the lifecycle goroutine on a doorbell, begin serving // reads (injected), and run the live ingestion loop. Returns nil only on a clean @@ -58,7 +58,7 @@ func run(ctx context.Context, cfg StartConfig) error { WithField("pinned", pinned). Info("startup — last-committed derived, beginning backfill") - // Step 1: backfill (catch up) to the tip. + // Step 1: backfill to the tip. lastCommitted, err = backfillToTip(ctx, cfg, lastCommitted, earliest) if err != nil { return err @@ -68,9 +68,9 @@ func run(ctx context.Context, cfg StartConfig) error { WithField("resume_chunk", chunk.IDFromLedger(lastCommitted+1).String()). Info("backfill complete — opening resume hot tier and ingesting") - // Step 2: serve + ingest. resumeLedger is one past the watermark — the live + // Step 2: serve + ingest. resumeLedger is one past the last-committed ledger — the live // chunk's next un-committed ledger; runIngestionLoop re-derives the exact resume - // point from durable state, so a mid-chunk and a boundary watermark both resume right. + // point from durable state, so a mid-chunk and a boundary last-committed ledger both resume right. resumeLedger := lastCommitted + 1 resumeChunk := chunk.IDFromLedger(resumeLedger) @@ -243,7 +243,7 @@ var ErrFirstStartNoTip = errors.New("network tip unavailable and no local histor // --------------------------------------------------------------------------- // NetworkTipBackend samples the bulk backend's current network tip during backfill. -// It is consulted only during catch-up; once ingestion runs, captive core is the tip. +// It is consulted only during backfill; once ingestion runs, captive core is the tip. type NetworkTipBackend interface { NetworkTip(ctx context.Context) (uint32, error) } @@ -261,7 +261,7 @@ type StartConfig struct { Exec backfill.ExecConfig // Lifecycle drives the lifecycle goroutine. Its embedded ExecConfig is the SAME - // wiring as Exec (one catalog, one pool); RetentionChunks is the catch-up floor's + // wiring as Exec (one catalog, one pool); RetentionChunks is the backfill floor's // width too (0 ⇒ the earliest-ledger floor only). Lifecycle lifecycle.LifecycleConfig From c2471fc89d8f9c099e0ffffac76ea68cc2159e75 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Tue, 30 Jun 2026 20:28:28 -0400 Subject: [PATCH 10/55] fullhistory: rename functions to the design doc's names - openHotTierForChunk / discardHotTierForChunk -> openHotDBForChunk / discardHotDBForChunk (they operate on one chunk's hot DB; the doc says "hot DB") - runLifecycleTick -> runLifecycle (the doc's name) - lifecycle.RunLoop -> lifecycle.Loop (exported form of the doc's lifecycleLoop) Identifier + comment renames only; no behavior change. lifecycle -short + root -short + the full E2E green. --- cmd/stellar-rpc/internal/fullhistory/e2e_test.go | 4 ++-- cmd/stellar-rpc/internal/fullhistory/hotloop.go | 8 ++++---- .../internal/fullhistory/hotloop_test.go | 14 +++++++------- .../internal/fullhistory/lifecycle/discard.go | 4 ++-- .../internal/fullhistory/lifecycle/discard_test.go | 4 ++-- .../internal/fullhistory/lifecycle/eligibility.go | 6 +++--- .../internal/fullhistory/lifecycle/helpers_test.go | 10 +++++----- .../internal/fullhistory/lifecycle/lifecycle.go | 10 +++++----- .../lifecycle/lifecycle_helpers_test.go | 4 ++-- .../fullhistory/lifecycle/lifecycle_loop_test.go | 8 ++++---- .../fullhistory/lifecycle/lifecycle_test.go | 4 ++-- .../fullhistory/lifecycle/progress_test.go | 2 +- cmd/stellar-rpc/internal/fullhistory/startup.go | 6 +++--- .../internal/fullhistory/startup_test.go | 2 +- 14 files changed, 43 insertions(+), 43 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go index befbe9300..6d272117c 100644 --- a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go @@ -8,10 +8,10 @@ package fullhistory // - runDaemonWith (the true daemon entrypoint): TOML load + form-validate, // per-root flock, meta-store open + Catalog bind, the stateful // validateConfig gate (pins the floor), and the supervised run loop. -// - run → backfillToTip → openHotTierForChunk → runIngestionLoop (the real +// - run → backfillToTip → openHotDBForChunk → runIngestionLoop (the real // atomic per-ledger WriteBatch across all CFs of the real per-chunk // hotchunk RocksDB), the real boundary handoff, the real doorbell. -// - lifecycle.RunLoop / runLifecycleTick: the real resolve + executePlan +// - lifecycle.Loop / runLifecycle: the real resolve + executePlan // freeze (cold artifacts derived FROM the live hot DB), the real txhash // index fold (a real streamhash .idx on disk), the real discard + prune. // - The real txhash stores on both sides of a getTransaction-style hash→seq diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index 84487414c..f84b32b32 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -41,14 +41,14 @@ type LedgerGetter interface { //nolint:gochecknoglobals // immutable selection, the production ingest config var allHotTypes = hotchunk.Ingest{Ledgers: true, Txhash: true, Events: true} -// openHotTierForChunk opens/recovers/creates the chunk's shared hot DB, keyed on +// openHotDBForChunk opens/recovers/creates the chunk's shared hot DB, keyed on // the durable hot:chunk state: // - "ready": open it. A MISSING dir is hot-volume loss (the hot DB is the sole // copy of recently-ingested ledgers) — refuse with ErrHotVolumeLost, never auto-heal. // - "transient" or absent: wipe any leftover dir and create fresh // (transient -> fsync dir+parent -> ready), so a crash mid-create can't // fabricate the "ready but dir missing" fatal above. -func openHotTierForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlog.Entry) (*hotchunk.DB, error) { +func openHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlog.Entry) (*hotchunk.DB, error) { dir := cat.Layout().HotChunkPath(chunkID) state, err := cat.HotState(chunkID) @@ -194,13 +194,13 @@ func runIngestionLoop( } hotDB = nil // released; reopen below republishes it for the defer - nextDB, oerr := openHotTierForChunk(cat, next, logger) + nextDB, oerr := openHotDBForChunk(cat, next, logger) if oerr != nil { return fmt.Errorf("open hot DB for chunk %s at boundary: %w", next, oerr) } hotDB = nextDB hotService = ingest.NewHotService(hotDB, ingestTypes, sink) - // next's key (created inside openHotTierForChunk) moved the partition; + // next's key (created inside openHotDBForChunk) moved the partition; // only now notify the lifecycle of the completed chunk. notify(closed) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index 3789b4faf..b2cea2f17 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -94,7 +94,7 @@ func getterForSeqs(t *testing.T, from, to uint32) *fakeLedgerGetter { // production opener, returning the handle and the catalog it lives under. func openLiveHotDB(t *testing.T, cat *catalog.Catalog, c chunk.ID) *hotchunk.DB { t.Helper() - db, err := openHotTierForChunk(cat, c, silentLogger()) + db, err := openHotDBForChunk(cat, c, silentLogger()) require.NoError(t, err) return db } @@ -110,7 +110,7 @@ func seedWatermark(t *testing.T, cat *catalog.Catalog, c chunk.ID, seq uint32) * db := openLiveHotDB(t, cat, c) require.NoError(t, db.Ledgers().AddLedgers(ledgerEntry(t, seq))) require.NoError(t, db.Close()) - reopened, err := openHotTierForChunk(cat, c, silentLogger()) + reopened, err := openHotDBForChunk(cat, c, silentLogger()) require.NoError(t, err) return reopened } @@ -130,7 +130,7 @@ func drainLifecycle(ch chan chunk.ID) []chunk.ID { } // --------------------------------------------------------------------------- -// openHotTierForChunk — the bracket's open end. +// openHotDBForChunk — the bracket's open end. // --------------------------------------------------------------------------- // TestOpenHotTier_CreatesBracketAndDir: a fresh open writes the dir and flips @@ -139,7 +139,7 @@ func TestOpenHotTier_CreatesBracketAndDir(t *testing.T) { cat, _ := testCatalog(t) c := chunk.ID(3) - db, err := openHotTierForChunk(cat, c, silentLogger()) + db, err := openHotDBForChunk(cat, c, silentLogger()) require.NoError(t, err) t.Cleanup(func() { _ = db.Close() }) @@ -163,7 +163,7 @@ func TestOpenHotTier_ReadyButDirMissingIsCase4(t *testing.T) { require.NoError(t, cat.PutHotTransient(c)) require.NoError(t, cat.FlipHotReady(c)) // key says ready, but no dir created - _, err := openHotTierForChunk(cat, c, silentLogger()) + _, err := openHotDBForChunk(cat, c, silentLogger()) require.Error(t, err) require.ErrorIs(t, err, backfill.ErrHotVolumeLost) } @@ -175,7 +175,7 @@ func TestOpenHotTier_TransientRecreatesFresh(t *testing.T) { c := chunk.ID(2) require.NoError(t, cat.PutHotTransient(c)) // a crash left a transient key - db, err := openHotTierForChunk(cat, c, silentLogger()) + db, err := openHotDBForChunk(cat, c, silentLogger()) require.NoError(t, err) t.Cleanup(func() { _ = db.Close() }) @@ -345,7 +345,7 @@ func TestRunIngestionLoop_RestartResumesFromWatermark(t *testing.T) { // Restart: re-open the live DB the way startup would. The resume point must // be watermark+1. - db2, err := openHotTierForChunk(cat, c, silentLogger()) + db2, err := openHotDBForChunk(cat, c, silentLogger()) require.NoError(t, err) resume, err := nextIngestLedger(db2) require.NoError(t, err) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go index 677e0f965..c60f1bfb6 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go @@ -10,12 +10,12 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) -// discardHotTierForChunk retires a chunk's hot DB once its cold artifacts are +// discardHotDBForChunk retires a chunk's hot DB once its cold artifacts are // durable (or it fell past retention): transient -> rmdir+fsync parent -> delete // key. Idempotent — a missing key is a no-op, and a crash mid-discard leaves the // key "transient" for the next scan to finish. The caller must have closed the // write handle (the stage runs after executePlan froze the cold artifacts). -func discardHotTierForChunk(cat *catalog.Catalog, chunkID chunk.ID) error { +func discardHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID) error { state, err := cat.HotState(chunkID) if err != nil { return fmt.Errorf("read hot key chunk %s: %w", chunkID, err) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard_test.go index 8aa6e4564..ce958ad43 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard_test.go @@ -18,7 +18,7 @@ func TestDiscardHotTier_RemovesDirAndKey(t *testing.T) { db := openLiveHotDB(t, cat, c) require.NoError(t, db.Close()) - require.NoError(t, discardHotTierForChunk(cat, c)) + require.NoError(t, discardHotDBForChunk(cat, c)) has, err := hotKeyExists(cat, c) require.NoError(t, err) @@ -26,5 +26,5 @@ func TestDiscardHotTier_RemovesDirAndKey(t *testing.T) { _, statErr := os.Stat(cat.Layout().HotChunkPath(c)) assert.True(t, os.IsNotExist(statErr), "the dir is removed") - require.NoError(t, discardHotTierForChunk(cat, c), "second discard is a no-op") + require.NoError(t, discardHotDBForChunk(cat, c), "second discard is a no-op") } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go index 9c00a9188..240e9974f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go @@ -14,7 +14,7 @@ import ( // fully serve (or that fell past retention). Per chunk: below the floor → discard; // complete (last <= through), nothing pending, and the index covers it → discard; // otherwise (live, or frozen awaiting coverage) → leave alone. -// discardHotTierForChunk is idempotent, so a crash between freeze and discard +// discardHotDBForChunk is idempotent, so a crash between freeze and discard // self-heals next tick. func eligibleDiscardOps(cfg LifecycleConfig, cat *catalog.Catalog, through uint32) ([]func() error, error) { earliest, _, err := cat.EarliestLedger() @@ -36,7 +36,7 @@ func eligibleDiscardOps(cfg LifecycleConfig, cat *catalog.Catalog, through uint3 last := c.LastLedger() switch { case gate.Excludes(c): - ops = append(ops, func() error { return discardHotTierForChunk(cat, c) }) + ops = append(ops, func() error { return discardHotDBForChunk(cat, c) }) case last <= through: pending, perr := pendingArtifacts(c, cfg, cat) if perr != nil { @@ -47,7 +47,7 @@ func eligibleDiscardOps(cfg LifecycleConfig, cat *catalog.Catalog, through uint3 return nil, cerr } if pending.Empty() && covers { - ops = append(ops, func() error { return discardHotTierForChunk(cat, c) }) + ops = append(ops, func() error { return discardHotDBForChunk(cat, c) }) } // else: frozen awaiting coverage, or still producing — leave alone. } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go index 1e41fe195..c35871b22 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go @@ -29,7 +29,7 @@ import ( // This file provides the shared test scaffolding the lifecycle tests need. The // catalog/fixture helpers are copied verbatim from the root fullhistory package's // helpers_test.go (which still serves the root tests). The hot-tier helpers -// (allHotTypes / openHotTierForChunk / openLiveHotDB / NewRocksHotProbe) are +// (allHotTypes / openHotDBForChunk / openLiveHotDB / NewRocksHotProbe) are // test-local equivalents of the production hot-source primitives that live in the // root fullhistory package — the lifecycle package cannot import root (root imports // lifecycle), so the lifecycle tests rebuild them over the same public store APIs. @@ -122,7 +122,7 @@ func zeroTxLCMBytes(t *testing.T, seq uint32) []byte { // --------------------------------------------------------------------------- // Hot-tier test scaffolding: test-local equivalents of the root package's -// production hot-source primitives (ingest.go's openHotTierForChunk/allHotTypes +// production hot-source primitives (ingest.go's openHotDBForChunk/allHotTypes // and hotsource.go's rocksHotProbe/NewRocksHotProbe). They use only the public // hotchunk/ledger/catalog/backfill APIs the production code uses, so a lifecycle // test reads and freezes the SAME on-disk hot DB the real daemon would, without @@ -133,12 +133,12 @@ func zeroTxLCMBytes(t *testing.T, seq uint32) []byte { // production ingest config. var allHotTypes = hotchunk.Ingest{Ledgers: true, Txhash: true, Events: true} -// openHotTierForChunk creates a "ready" shared hot DB for chunkID under the +// openHotDBForChunk creates a "ready" shared hot DB for chunkID under the // hot:chunk bracket (transient -> create -> ready) and returns an open handle the // caller owns. The test equivalent of the production opener, trimmed to the // create branch the lifecycle tests need (no crash-recovery / fsync — those edges // are covered by the root ingest_test.go opener tests). -func openHotTierForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlog.Entry) (*hotchunk.DB, error) { +func openHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlog.Entry) (*hotchunk.DB, error) { dir := cat.Layout().HotChunkPath(chunkID) if err := os.RemoveAll(dir); err != nil { return nil, fmt.Errorf("wipe leftover hot dir %s: %w", dir, err) @@ -161,7 +161,7 @@ func openHotTierForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *support // test opener, returning the handle. func openLiveHotDB(t *testing.T, cat *catalog.Catalog, c chunk.ID) *hotchunk.DB { t.Helper() - db, err := openHotTierForChunk(cat, c, silentLogger()) + db, err := openHotDBForChunk(cat, c, silentLogger()) require.NoError(t, err) return db } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index e14ca20ce..9948b9e6e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -90,7 +90,7 @@ func lowestMaterializedChunk(cat *catalog.Catalog) (chunk.ID, bool, error) { return lowest, found, nil } -// runLifecycleTick runs one tick over the three stages for just-completed chunk +// runLifecycle runs one tick over the three stages for just-completed chunk // lastChunk. through = lastChunk.LastLedger() is the single snapshot every stage // shares, so a boundary committing mid-tick can't make stages contradict (it's // next tick's work). Plan range is [floor, lastChunk] (start raised to storage); @@ -99,7 +99,7 @@ func lowestMaterializedChunk(cat *catalog.Catalog) (chunk.ID, bool, error) { // CLEAN-SHUTDOWN (binding): on an op error with ctx cancelled, return WITHOUT // Fatalf — cancellation is a shutdown, not a failure. Only a genuine failure // (ctx still live) aborts via Fatalf. -func runLifecycleTick(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog, lastChunk chunk.ID) { +func runLifecycle(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog, lastChunk chunk.ID) { metrics := observability.MetricsOrNop(cfg.Metrics) logger := cfg.Logger @@ -237,12 +237,12 @@ func runLifecycleTick(ctx context.Context, cfg LifecycleConfig, cat *catalog.Cat // this many boundaries behind ingestion, a fatal condition notify() reports. const LifecycleQueueDepth = 8 -// RunLoop is the event-driven lifecycle goroutine. Each notification carries +// Loop is the event-driven lifecycle goroutine. Each notification carries // the just-completed chunk id; the loop drains the buffer to the most-recent id // (one tick over [floor, lastChunk] subsumes the rest) and runs one tick. It // selects on both ctx.Done() and the channel, so it never blocks or fatals on // shutdown. -func RunLoop(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog, ch <-chan chunk.ID) { +func Loop(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog, ch <-chan chunk.ID) { for { select { case <-ctx.Done(): @@ -260,7 +260,7 @@ func RunLoop(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog, ch break drain } } - runLifecycleTick(ctx, cfg, cat, lastChunk) + runLifecycle(ctx, cfg, cat, lastChunk) } } } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go index 5df02fa1c..98298e327 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go @@ -152,7 +152,7 @@ func runTickForCatalog(ctx context.Context, t *testing.T, cfg LifecycleConfig, c if !ok { last = 0 } - runLifecycleTick(ctx, cfg, cat, last) + runLifecycle(ctx, cfg, cat, last) } // makeReadyHotDirNoData opens and closes a real (empty) hot DB for c so its dir @@ -160,7 +160,7 @@ func runTickForCatalog(ctx context.Context, t *testing.T, cfg LifecycleConfig, c // without needing a full ingest. func makeReadyHotDirNoData(t *testing.T, cat *catalog.Catalog, c chunk.ID) { t.Helper() - db, err := openHotTierForChunk(cat, c, silentLogger()) + db, err := openHotDBForChunk(cat, c, silentLogger()) require.NoError(t, err) require.NoError(t, db.Close()) } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go index ed2d29e60..b26a770fb 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go @@ -12,7 +12,7 @@ import ( ) // --------------------------------------------------------------------------- -// RunLoop: selects on BOTH ctx.Done and the notification channel; drains +// Loop: selects on BOTH ctx.Done and the notification channel; drains // to the most-recent queued chunk id. // --------------------------------------------------------------------------- @@ -37,7 +37,7 @@ func TestLifecycleLoop_RunsTickPerNotifyThenStopsOnCtx(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { - RunLoop(ctx, cfg, cat, ch) + Loop(ctx, cfg, cat, ch) close(done) }() @@ -77,7 +77,7 @@ func TestLifecycleLoop_DrainsToMostRecent(t *testing.T) { defer cancel() done := make(chan struct{}) go func() { - RunLoop(ctx, cfg, cat, ch) + Loop(ctx, cfg, cat, ch) close(done) }() @@ -111,7 +111,7 @@ func TestLifecycleLoop_ReturnsImmediatelyOnAlreadyCancelledCtx(t *testing.T) { ch := make(chan chunk.ID) // unbuffered, never sent to done := make(chan struct{}) go func() { - RunLoop(ctx, cfg, cat, ch) + Loop(ctx, cfg, cat, ch) close(done) }() select { diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go index eb1a11ffa..d715aa327 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go @@ -215,7 +215,7 @@ func TestRunLifecycleTick_CleanShutdownNoFatal(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // shutdown requested before the tick runs - runLifecycleTick(ctx, cfg, cat, 0) // lastChunk 0: plan range [0,0], build fails under a cancelled ctx + runLifecycle(ctx, cfg, cat, 0) // lastChunk 0: plan range [0,0], build fails under a cancelled ctx require.False(t, rec.fired(), "a cancelled ctx is a clean shutdown, NOT an op failure — no Fatalf") } @@ -224,6 +224,6 @@ func TestRunLifecycleTick_CleanShutdownNoFatal(t *testing.T) { func TestRunLifecycleTick_GenuineFailureAborts(t *testing.T) { cfg, rec, cat := genuineFailureTickSetup(t) - runLifecycleTick(context.Background(), cfg, cat, 0) // lastChunk 0: plan range [0,0], the failing build + runLifecycle(context.Background(), cfg, cat, 0) // lastChunk 0: plan range [0,0], the failing build require.True(t, rec.fired(), "a genuine op failure aborts the daemon") } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go index 34f834fb4..896c7b18c 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go @@ -269,7 +269,7 @@ func TestDeriveWatermark(t *testing.T) { // Two ready keys; the LOWER one's dir is missing. Under the design's lazy // detection (no eager all-ready-keys scan) only the HIGHEST ready chunk is // opened, so the lower key's missing dir is not surfaced here — it surfaces - // later, when ingestion/discard reaches that chunk via openHotTierForChunk. + // later, when ingestion/discard reaches that chunk via openHotDBForChunk. require.NoError(t, cat.PutHotTransient(2)) require.NoError(t, cat.FlipHotReady(2)) // ready key 2, NO dir (not opened here) readyHot(t, cat, 5) // highest ready key 5 WITH dir (opened) diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index 113554453..04cc1d43c 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -74,7 +74,7 @@ func run(ctx context.Context, cfg StartConfig) error { resumeLedger := lastCommitted + 1 resumeChunk := chunk.IDFromLedger(resumeLedger) - hotDB, err := openHotTierForChunk(cat, resumeChunk, logger) + hotDB, err := openHotDBForChunk(cat, resumeChunk, logger) if err != nil { return fmt.Errorf("startup open resume hot tier chunk %s: %w", resumeChunk, err) } @@ -111,14 +111,14 @@ func run(ctx context.Context, cfg StartConfig) error { // the single-lifecycle-goroutine invariant across supervisor restarts (a // daemon-ctx-tied loop would survive a restartable return and run a tick // concurrently with the next iteration's lifecycle + ingestion: two backfill - // passes truncating the same .pack/.idx). RunLoop checks ctx at every step, so + // passes truncating the same .pack/.idx). Loop checks ctx at every step, so // the join cannot block past the current step. lifecycleCtx, cancelLifecycle := context.WithCancel(ctx) var lifecycleWG sync.WaitGroup lifecycleWG.Add(1) go func() { defer lifecycleWG.Done() - lifecycle.RunLoop(lifecycleCtx, cfg.Lifecycle, cat, lifecycleCh) + lifecycle.Loop(lifecycleCtx, cfg.Lifecycle, cat, lifecycleCh) }() // The two return paths registered after this defer (the ingestion-loop return // and the ServeReads error path) have no live sender on lifecycleCh — ingestion diff --git a/cmd/stellar-rpc/internal/fullhistory/startup_test.go b/cmd/stellar-rpc/internal/fullhistory/startup_test.go index 17aabee47..684f0e400 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup_test.go @@ -414,7 +414,7 @@ func TestRun_ServeReadsErrorSurfaces(t *testing.T) { require.Equal(t, int32(1), core.openedCount.Load(), "core was started before serving") // The resume hot DB was closed on the error path (LOCK released): reopening it succeeds. - db, err := openHotTierForChunk(cat, chunk.IDFromLedger(chunk.FirstLedgerSeq), silentLogger()) + db, err := openHotDBForChunk(cat, chunk.IDFromLedger(chunk.FirstLedgerSeq), silentLogger()) require.NoError(t, err, "the resume hot DB is reopenable — run released its LOCK") require.NoError(t, db.Close()) } From f1d602a9d2166f99a9e719e4ad8692f4a437a00e Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Tue, 30 Jun 2026 22:46:44 -0400 Subject: [PATCH 11/55] fullhistory: run captive core for live ingestion via ledgerbackend (#816) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #816 scopes live ingestion FROM captive core (not #772), so wire it here instead of the deferred-to-#772 stub. newCaptiveCoreOpener now builds a real ledgerbackend captive core the same way the RPC daemon (daemon.newCaptiveCore) does — same CaptiveCoreTomlParams (strict, unified events, soroban diagnostic/meta enforcement) and CaptiveCoreConfig — so full-history ingests the same meta the events/txhash stores need. OpenCore builds a FRESH backend per run (restart-safe) and PrepareRange(UnboundedRange(resume)). The captive-core params are added to the full-history config's [ingestion] section (mirroring the RPC daemon's knobs): stellar_core_binary_path, network_passphrase, history_archive_urls, captive_core_storage_path (defaults to {default_data_dir}/captive-core). captive_core_config was already present. Validated in newCaptiveCoreOpener (only reached when no CoreOpener is injected); tests inject a fake core, so build/vet + root -short stay green. --- .../internal/fullhistory/config.go | 19 +++- .../internal/fullhistory/daemon.go | 88 ++++++++++++++----- 2 files changed, 83 insertions(+), 24 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/config.go b/cmd/stellar-rpc/internal/fullhistory/config.go index 6ad1e2c16..93094e80f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/config.go +++ b/cmd/stellar-rpc/internal/fullhistory/config.go @@ -88,10 +88,25 @@ type BackfillConfig struct { BSB ledgerbackend.BufferedStorageBackendConfig `toml:"bsb"` } -// IngestionConfig is [ingestion] — the live-network ingestion settings. +// IngestionConfig is [ingestion] — the live-network ingestion settings. These +// mirror the RPC daemon's captive-core knobs so the full-history daemon runs +// captive core the same way (see newCaptiveCoreOpener). type IngestionConfig struct { - // Path to the CaptiveStellarCore config file. Required. + // CaptiveCoreConfig is the path to the CaptiveStellarCore (stellar-core) config + // file. Required for live ingestion. CaptiveCoreConfig string `toml:"captive_core_config"` + // StellarCoreBinaryPath is the path to the stellar-core binary captive core runs. + // Required for live ingestion. + StellarCoreBinaryPath string `toml:"stellar_core_binary_path"` + // NetworkPassphrase is the Stellar network passphrase captive core connects with. + // Required for live ingestion. + NetworkPassphrase string `toml:"network_passphrase"` + // HistoryArchiveURLs are the history-archive URLs captive core reads checkpoints + // from. Required for live ingestion. + HistoryArchiveURLs []string `toml:"history_archive_urls"` + // CaptiveCoreStoragePath is captive core's BUCKET_DIR_PATH base; defaults to + // {default_data_dir}/captive-core when empty. + CaptiveCoreStoragePath string `toml:"captive_core_storage_path"` } // LoggingConfig is [logging]. diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index 41418132d..b981b02f9 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "time" "github.com/prometheus/client_golang/prometheus" @@ -163,7 +164,7 @@ func runDaemonWith(ctx context.Context, configPath string, opts daemonOptions) e // errors surface first, and a deployment must inject Core until the cutover. core := opts.Core if core == nil { - built, cerr := newCaptiveCoreOpener(cfg.Ingestion.CaptiveCoreConfig, logger) + built, cerr := newCaptiveCoreOpener(cfg.Ingestion, cfg.Service.DefaultDataDir, logger) if cerr != nil { return cerr } @@ -296,38 +297,81 @@ func buildBackfillBackend( // Production captive-core opener (the live ingestion source). // --------------------------------------------------------------------------- -// captiveCoreOpener is the production CoreOpener: it prepares captive core at the -// resume ledger and hands back a LedgerGetter the ingestion loop polls by -// sequence (the design's core.GetLedger(ctx, seq)) plus a closer. +// captiveCoreOpener is the production CoreOpener. It holds a resolved +// CaptiveCoreConfig and builds a FRESH captive-core ledgerbackend per run (each +// supervised restart reopens core anew), prepares it at the resume ledger, and +// hands back a LedgerGetter the ingestion loop polls by sequence plus a closer. +// Construction mirrors the RPC daemon's newCaptiveCore so the full-history daemon +// runs captive core and the ledgerbackend the same way (#772 can unify them at +// the cutover). type captiveCoreOpener struct { - backend ledgerbackend.LedgerBackend + config ledgerbackend.CaptiveCoreConfig } -// newCaptiveCoreOpener builds the production opener. The captive-core config -// plumbing is deferred to #772, so today it parses the path and errors with a -// clear pointer — a deployment must inject a CoreOpener via daemonOptions until -// the cutover lands. The seam (a LedgerGetter behind CoreOpener) is final. -func newCaptiveCoreOpener(captiveCoreConfigPath string, _ *supportlog.Entry) (*captiveCoreOpener, error) { - if captiveCoreConfigPath == "" { - return nil, errors.New("[ingestion].captive_core_config is required") +// newCaptiveCoreOpener resolves the captive-core config from [ingestion] the same +// way the RPC daemon does (same toml params: strict, unified events, soroban +// diagnostic/meta enforcement — the meta the events + txhash ingesters need). +func newCaptiveCoreOpener(ing IngestionConfig, dataDir string, logger *supportlog.Entry) (*captiveCoreOpener, error) { + switch { + case ing.CaptiveCoreConfig == "": + return nil, errors.New("[ingestion].captive_core_config is required for live ingestion") + case ing.StellarCoreBinaryPath == "": + return nil, errors.New("[ingestion].stellar_core_binary_path is required for live ingestion") + case ing.NetworkPassphrase == "": + return nil, errors.New("[ingestion].network_passphrase is required for live ingestion") + case len(ing.HistoryArchiveURLs) == 0: + return nil, errors.New("[ingestion].history_archive_urls is required for live ingestion") + } + + storagePath := ing.CaptiveCoreStoragePath + if storagePath == "" { + storagePath = filepath.Join(dataDir, "captive-core") + } + + toml, err := ledgerbackend.NewCaptiveCoreTomlFromFile(ing.CaptiveCoreConfig, ledgerbackend.CaptiveCoreTomlParams{ + HistoryArchiveURLs: ing.HistoryArchiveURLs, + NetworkPassphrase: ing.NetworkPassphrase, + Strict: true, + EnforceSorobanDiagnosticEvents: true, + EnforceSorobanTransactionMetaExtV1: true, + EmitUnifiedEvents: true, + CoreBinaryPath: ing.StellarCoreBinaryPath, + }) + if err != nil { + return nil, fmt.Errorf("invalid captive-core toml %q: %w", ing.CaptiveCoreConfig, err) } - // TODO(#772): build a ledgerbackend.CaptiveCoreConfig from - // NewCaptiveCoreTomlFromFile(captiveCoreConfigPath, ...) + NewCaptive, then - // PrepareRange(UnboundedRange(resume)) in OpenCore. Only the config plumbing - // is deferred; the seam below is final. - return nil, fmt.Errorf("production captive-core wiring is deferred to #772 "+ - "(config %q parsed; inject a CoreOpener via daemonOptions to run today)", captiveCoreConfigPath) + + return &captiveCoreOpener{ + config: ledgerbackend.CaptiveCoreConfig{ + BinaryPath: ing.StellarCoreBinaryPath, + StoragePath: storagePath, + NetworkPassphrase: ing.NetworkPassphrase, + HistoryArchiveURLs: ing.HistoryArchiveURLs, + Log: logger.WithField("subservice", "stellar-core"), + Toml: toml, + UserAgent: "stellar-rpc-fullhistory", + }, + }, nil } -// OpenCore prepares the backend over the unbounded range from resumeLedger and -// returns a getter wrapping GetLedger plus the backend's Close. +// OpenCore builds a fresh captive-core backend, prepares it over the unbounded +// range from resumeLedger, and returns a getter wrapping GetLedger plus the +// backend's Close. A fresh backend per call keeps supervised restarts clean (the +// prior run's core was closed on its way out). func (c *captiveCoreOpener) OpenCore( ctx context.Context, resumeLedger uint32, ) (LedgerGetter, func() error, error) { - if err := c.backend.PrepareRange(ctx, ledgerbackend.UnboundedRange(resumeLedger)); err != nil { + cfg := c.config + cfg.Context = ctx + backend, err := ledgerbackend.NewCaptive(cfg) + if err != nil { + return nil, nil, fmt.Errorf("build captive core: %w", err) + } + if err := backend.PrepareRange(ctx, ledgerbackend.UnboundedRange(resumeLedger)); err != nil { + _ = backend.Close() return nil, nil, fmt.Errorf("captive core prepare range from %d: %w", resumeLedger, err) } - return backendGetter{backend: c.backend}, c.backend.Close, nil + return backendGetter{backend: backend}, backend.Close, nil } // backendGetter adapts a ledgerbackend.LedgerBackend to LedgerGetter: GetLedger From 4a4e037c8171004b54651d585c5b837743c6b804 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Tue, 30 Jun 2026 23:14:17 -0400 Subject: [PATCH 12/55] fullhistory: source captive-core config from the captive_core_config file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per Simon: the full-history config should point at a captive_core_config file that holds what captive core needs. Read NETWORK_PASSPHRASE back from that file and default the stellar-core binary to the one on PATH (the RPC daemon's default), so those aren't separate keys. The one thing that genuinely can't come from the file stays explicit: history_archive_urls — the SDK's archive client needs plain URLs, but the file's [HISTORY.*] entries are shell (curl) commands. [ingestion] is now: captive_core_config + history_archive_urls (required), stellar_core_binary_path + captive_core_storage_path (optional overrides; default to PATH lookup and {default_data_dir}/captive-core). Construction still mirrors the RPC daemon's ledgerbackend usage. Build/vet + root -short green (tests inject a fake core, so the opener isn't exercised in-test). --- .../internal/fullhistory/config.go | 29 ++++----- .../internal/fullhistory/daemon.go | 62 ++++++++++++++----- 2 files changed, 61 insertions(+), 30 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/config.go b/cmd/stellar-rpc/internal/fullhistory/config.go index 93094e80f..f2edb2b84 100644 --- a/cmd/stellar-rpc/internal/fullhistory/config.go +++ b/cmd/stellar-rpc/internal/fullhistory/config.go @@ -88,24 +88,25 @@ type BackfillConfig struct { BSB ledgerbackend.BufferedStorageBackendConfig `toml:"bsb"` } -// IngestionConfig is [ingestion] — the live-network ingestion settings. These -// mirror the RPC daemon's captive-core knobs so the full-history daemon runs -// captive core the same way (see newCaptiveCoreOpener). +// IngestionConfig is [ingestion] — the live-network ingestion (captive-core) +// settings. The captive-core config FILE is the single source of truth for what +// it can hold (notably NETWORK_PASSPHRASE, read back at startup); only the two +// things that genuinely can't live in that file are separate keys — the plain +// history-archive URLs (the file's [HISTORY.*] entries are shell commands, not +// the URLs the SDK's archive client needs) and, optionally, the binary path. type IngestionConfig struct { // CaptiveCoreConfig is the path to the CaptiveStellarCore (stellar-core) config - // file. Required for live ingestion. + // file. Required for live ingestion. Must define NETWORK_PASSPHRASE. CaptiveCoreConfig string `toml:"captive_core_config"` - // StellarCoreBinaryPath is the path to the stellar-core binary captive core runs. - // Required for live ingestion. - StellarCoreBinaryPath string `toml:"stellar_core_binary_path"` - // NetworkPassphrase is the Stellar network passphrase captive core connects with. - // Required for live ingestion. - NetworkPassphrase string `toml:"network_passphrase"` - // HistoryArchiveURLs are the history-archive URLs captive core reads checkpoints - // from. Required for live ingestion. + // HistoryArchiveURLs are the plain history-archive URLs the SDK reads + // checkpoints from. Required for live ingestion (not derivable from the + // captive-core file's [HISTORY.*] get-commands). HistoryArchiveURLs []string `toml:"history_archive_urls"` - // CaptiveCoreStoragePath is captive core's BUCKET_DIR_PATH base; defaults to - // {default_data_dir}/captive-core when empty. + // StellarCoreBinaryPath is the path to the stellar-core binary. Optional — + // defaults to the "stellar-core" found on PATH. + StellarCoreBinaryPath string `toml:"stellar_core_binary_path"` + // CaptiveCoreStoragePath is captive core's BUCKET_DIR_PATH base; optional, + // defaults to {default_data_dir}/captive-core. CaptiveCoreStoragePath string `toml:"captive_core_storage_path"` } diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index b981b02f9..d975db981 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -4,9 +4,12 @@ import ( "context" "errors" "fmt" + "os" + "os/exec" "path/filepath" "time" + "github.com/pelletier/go-toml" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" @@ -308,34 +311,61 @@ type captiveCoreOpener struct { config ledgerbackend.CaptiveCoreConfig } -// newCaptiveCoreOpener resolves the captive-core config from [ingestion] the same -// way the RPC daemon does (same toml params: strict, unified events, soroban -// diagnostic/meta enforcement — the meta the events + txhash ingesters need). +// newCaptiveCoreOpener resolves the captive-core config, treating the +// captive_core_config FILE as the single source of truth: NETWORK_PASSPHRASE is +// read back from it, and the stellar-core binary defaults to the one on PATH. +// Only the plain history-archive URLs (not derivable from the file's [HISTORY.*] +// get-commands) come from [ingestion].history_archive_urls. The toml params +// mirror the RPC daemon (strict, unified events, soroban diagnostic/meta +// enforcement) so the ingested meta is what the events + txhash stores need. func newCaptiveCoreOpener(ing IngestionConfig, dataDir string, logger *supportlog.Entry) (*captiveCoreOpener, error) { - switch { - case ing.CaptiveCoreConfig == "": + if ing.CaptiveCoreConfig == "" { return nil, errors.New("[ingestion].captive_core_config is required for live ingestion") - case ing.StellarCoreBinaryPath == "": - return nil, errors.New("[ingestion].stellar_core_binary_path is required for live ingestion") - case ing.NetworkPassphrase == "": - return nil, errors.New("[ingestion].network_passphrase is required for live ingestion") - case len(ing.HistoryArchiveURLs) == 0: + } + if len(ing.HistoryArchiveURLs) == 0 { return nil, errors.New("[ingestion].history_archive_urls is required for live ingestion") } + // NETWORK_PASSPHRASE lives in the captive-core file; read it back so the + // operator configures it in one place. (go-toml v1 ignores the other fields.) + data, err := os.ReadFile(ing.CaptiveCoreConfig) + if err != nil { + return nil, fmt.Errorf("read captive_core_config %q: %w", ing.CaptiveCoreConfig, err) + } + var peek struct { + NetworkPassphrase string `toml:"NETWORK_PASSPHRASE"` + } + if perr := toml.Unmarshal(data, &peek); perr != nil { + return nil, fmt.Errorf("parse captive_core_config %q: %w", ing.CaptiveCoreConfig, perr) + } + if peek.NetworkPassphrase == "" { + return nil, fmt.Errorf("captive_core_config %q must define NETWORK_PASSPHRASE", ing.CaptiveCoreConfig) + } + + // stellar-core binary: explicit path, else the one on PATH (RPC daemon default). + binaryPath := ing.StellarCoreBinaryPath + if binaryPath == "" { + found, lerr := exec.LookPath("stellar-core") + if lerr != nil { + return nil, fmt.Errorf( + "[ingestion].stellar_core_binary_path unset and stellar-core not found on PATH: %w", lerr) + } + binaryPath = found + } + storagePath := ing.CaptiveCoreStoragePath if storagePath == "" { storagePath = filepath.Join(dataDir, "captive-core") } - toml, err := ledgerbackend.NewCaptiveCoreTomlFromFile(ing.CaptiveCoreConfig, ledgerbackend.CaptiveCoreTomlParams{ + coreToml, err := ledgerbackend.NewCaptiveCoreTomlFromFile(ing.CaptiveCoreConfig, ledgerbackend.CaptiveCoreTomlParams{ HistoryArchiveURLs: ing.HistoryArchiveURLs, - NetworkPassphrase: ing.NetworkPassphrase, + NetworkPassphrase: peek.NetworkPassphrase, Strict: true, EnforceSorobanDiagnosticEvents: true, EnforceSorobanTransactionMetaExtV1: true, EmitUnifiedEvents: true, - CoreBinaryPath: ing.StellarCoreBinaryPath, + CoreBinaryPath: binaryPath, }) if err != nil { return nil, fmt.Errorf("invalid captive-core toml %q: %w", ing.CaptiveCoreConfig, err) @@ -343,12 +373,12 @@ func newCaptiveCoreOpener(ing IngestionConfig, dataDir string, logger *supportlo return &captiveCoreOpener{ config: ledgerbackend.CaptiveCoreConfig{ - BinaryPath: ing.StellarCoreBinaryPath, + BinaryPath: binaryPath, StoragePath: storagePath, - NetworkPassphrase: ing.NetworkPassphrase, + NetworkPassphrase: peek.NetworkPassphrase, HistoryArchiveURLs: ing.HistoryArchiveURLs, Log: logger.WithField("subservice", "stellar-core"), - Toml: toml, + Toml: coreToml, UserAgent: "stellar-rpc-fullhistory", }, }, nil From a07c973d676da6d794d68678039bcaa455a50a67 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 01:53:20 -0400 Subject: [PATCH 13/55] Align hot ingestion with always-on hot tier --- .../internal/fullhistory/hotloop.go | 16 ++--- .../internal/fullhistory/hotloop_test.go | 19 +++-- .../internal/fullhistory/hotsource.go | 14 +--- .../internal/fullhistory/ingest/service.go | 43 ++++++----- .../fullhistory/lifecycle/helpers_test.go | 8 +-- .../lifecycle/lifecycle_helpers_test.go | 2 +- .../pkg/stores/hotchunk/hotchunk.go | 71 +++++++------------ .../pkg/stores/hotchunk/hotchunk_test.go | 41 +++++------ .../internal/fullhistory/startup.go | 2 +- 9 files changed, 85 insertions(+), 131 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index f84b32b32..488ad966e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -34,13 +34,6 @@ type LedgerGetter interface { GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseMetaView, error) } -// allHotTypes is the hot tier's ingest selection: the hot DB is the sole copy of -// a chunk's recently ingested ledgers until the cold artifacts freeze, so it -// always ingests all three types in the one atomic batch. -// -//nolint:gochecknoglobals // immutable selection, the production ingest config -var allHotTypes = hotchunk.Ingest{Ledgers: true, Txhash: true, Events: true} - // openHotDBForChunk opens/recovers/creates the chunk's shared hot DB, keyed on // the durable hot:chunk state: // - "ready": open it. A MISSING dir is hot-volume loss (the hot DB is the sole @@ -118,14 +111,13 @@ func openHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlo // HANDOFF FENCE: the DB is CLOSED before the next chunk's hot:chunk key is // created — that key is what makes THIS chunk complete to the lifecycle, which // could then discard a dir a still-live writer holds. notify() fires only after -// the next DB is open. The HotService (nil-sink-safe) is rebuilt each boundary. +// the next DB is open. The HotService is rebuilt each boundary. func runIngestionLoop( ctx context.Context, core LedgerGetter, hotDB *hotchunk.DB, cat *catalog.Catalog, lifecycleCh chan<- chunk.ID, - ingestTypes hotchunk.Ingest, logger *supportlog.Entry, metrics observability.Metrics, sink ingest.MetricSink, @@ -165,7 +157,7 @@ func runIngestionLoop( // hotService binds the metrics sink to THIS hotDB instance; the boundary // handoff rebuilds it for the reopened chunk DB below. - hotService := ingest.NewHotService(hotDB, ingestTypes, sink) + hotService := ingest.NewHotService(hotDB, sink) // Indexed poll from the resume ledger. GetLedger blocks until seq is // available; its error ends the loop for the daemon top level to classify. @@ -175,7 +167,7 @@ func runIngestionLoop( return fmt.Errorf("get ledger %d: %w", seq, gerr) } - // One atomic synced WriteBatch across all enabled CFs (via + // One atomic synced WriteBatch across all hot CFs (via // hotDB.IngestLedger), reporting per-type LedgerCounts to the sink. if ierr := hotService.Ingest(ctx, seq, lcm); ierr != nil { return fmt.Errorf("ingest ledger %d: %w", seq, ierr) @@ -199,7 +191,7 @@ func runIngestionLoop( return fmt.Errorf("open hot DB for chunk %s at boundary: %w", next, oerr) } hotDB = nextDB - hotService = ingest.NewHotService(hotDB, ingestTypes, sink) + hotService = ingest.NewHotService(hotDB, sink) // next's key (created inside openHotDBForChunk) moved the partition; // only now notify the lifecycle of the completed chunk. notify(closed) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index b2cea2f17..2c62557d4 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -102,9 +102,9 @@ func openLiveHotDB(t *testing.T, cat *catalog.Catalog, c chunk.ID) *hotchunk.DB // seedWatermark writes a single ledgers-CF entry at seq into the chunk's hot DB // so the indexed poll resumes at seq+1 — letting a boundary test drive the loop // over only the last ledger or two of a chunk instead of all 10,000. The -// returned DB is the (re-opened, ready) live handle the loop then owns. Used by -// the boundary tests, whose ingestTypes are Ledgers+Txhash (no events -// contiguity requirement, so a sparse ledgers-CF watermark is valid). +// returned DB is the (re-opened, ready) live handle the loop then owns. The +// zero-event fixtures keep the sparse ledgers-CF watermark valid for boundary +// tests without preloading all preceding event offsets. func seedWatermark(t *testing.T, cat *catalog.Catalog, c chunk.ID, seq uint32) *hotchunk.DB { t.Helper() db := openLiveHotDB(t, cat, c) @@ -204,7 +204,7 @@ func TestRunIngestionLoop_LedgerLandsAcrossAllCFs(t *testing.T) { getter.endErr = errors.New("backend crashed") ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) - err := runIngestionLoop(context.Background(), getter, db, cat, ch, allHotTypes, silentLogger(), nil, nil) + err := runIngestionLoop(context.Background(), getter, db, cat, ch, silentLogger(), nil, nil) require.Error(t, err, "poll ran past the prefix and the getter errored") require.NotErrorIs(t, err, backfill.ErrHotVolumeLost) @@ -239,7 +239,6 @@ func TestRunIngestionLoop_BoundaryNotifiesCompletedChunk(t *testing.T) { c1 := c + 1 db := seedWatermark(t, cat, c, c.LastLedger()-1) - ingestTypes := hotchunk.Ingest{Ledgers: true, Txhash: true} getter := &fakeLedgerGetter{frames: map[uint32][]byte{ c.LastLedger(): zeroTxLCMBytes(t, c.LastLedger()), // boundary 0->1 c1.FirstLedger(): zeroTxLCMBytes(t, c1.FirstLedger()), // a ledger in chunk 1 @@ -248,7 +247,7 @@ func TestRunIngestionLoop_BoundaryNotifiesCompletedChunk(t *testing.T) { done := make(chan error, 1) go func() { - done <- runIngestionLoop(context.Background(), getter, db, cat, ch, ingestTypes, silentLogger(), nil, nil) + done <- runIngestionLoop(context.Background(), getter, db, cat, ch, silentLogger(), nil, nil) }() select { @@ -283,7 +282,7 @@ func TestRunIngestionLoop_CtxCancelReturnsCtxErr(t *testing.T) { done := make(chan error, 1) go func() { - done <- runIngestionLoop(ctx, getter, db, cat, ch, allHotTypes, silentLogger(), nil, nil) + done <- runIngestionLoop(ctx, getter, db, cat, ch, silentLogger(), nil, nil) }() require.Eventually(t, func() bool { @@ -314,7 +313,7 @@ func TestRunIngestionLoop_GetLedgerErrorReturnsError(t *testing.T) { getter.errAt = boom ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) - err := runIngestionLoop(context.Background(), getter, db, cat, ch, allHotTypes, silentLogger(), nil, nil) + err := runIngestionLoop(context.Background(), getter, db, cat, ch, silentLogger(), nil, nil) require.Error(t, err) require.ErrorIs(t, err, boom) require.NotErrorIs(t, err, backfill.ErrHotVolumeLost) @@ -339,7 +338,7 @@ func TestRunIngestionLoop_RestartResumesFromWatermark(t *testing.T) { getter1 := getterForSeqs(t, first, first+2) getter1.endErr = errors.New("end") ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) - err := runIngestionLoop(context.Background(), getter1, db1, cat, ch, allHotTypes, silentLogger(), nil, nil) + err := runIngestionLoop(context.Background(), getter1, db1, cat, ch, silentLogger(), nil, nil) require.Error(t, err) assert.Equal(t, first, getter1.firstSeen.Load(), "first run resumed at the chunk's first ledger") @@ -355,7 +354,7 @@ func TestRunIngestionLoop_RestartResumesFromWatermark(t *testing.T) { // two new ones. getter2 := getterForSeqs(t, first+2, first+5) getter2.endErr = errors.New("end") - err = runIngestionLoop(context.Background(), getter2, db2, cat, ch, allHotTypes, silentLogger(), nil, nil) + err = runIngestionLoop(context.Background(), getter2, db2, cat, ch, silentLogger(), nil, nil) require.Error(t, err) assert.Equal(t, first+3, getter2.firstSeen.Load(), "second run resumed at watermark+1") diff --git a/cmd/stellar-rpc/internal/fullhistory/hotsource.go b/cmd/stellar-rpc/internal/fullhistory/hotsource.go index c439c4dc8..3a66176ce 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotsource.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotsource.go @@ -51,23 +51,18 @@ func (p *rocksHotProbe) OpenHotChunk(chunkID chunk.ID) (backfill.HotChunk, bool, if err != nil { return nil, false, fmt.Errorf("open hot chunk DB: %w", err) } - return &rocksHotChunk{chunkID: chunkID, db: db}, true, nil + return &rocksHotChunk{db: db}, true, nil } // rocksHotChunk is one chunk's opened hot tier — the single shared DB. type rocksHotChunk struct { - chunkID chunk.ID - db *hotchunk.DB + db *hotchunk.DB } // MaxCommittedSeq returns the single authoritative last-committed ledger (decision (a)): the // highest ledger seq the shared DB has durably committed. ok=false on an empty DB. func (h *rocksHotChunk) MaxCommittedSeq() (uint32, bool, error) { - seq, ok, err := h.db.MaxCommittedSeq() - if err != nil { - return 0, false, fmt.Errorf("hot DB max committed seq: %w", err) - } - return seq, ok, nil + return h.db.MaxCommittedSeq() } // Source streams the chunk's LCMs from the ledgers CF as a LedgerStream the cold @@ -79,9 +74,6 @@ func (h *rocksHotChunk) Source() ledgerbackend.LedgerStream { // Close releases the shared hot DB. func (h *rocksHotChunk) Close() error { - if h.db == nil { - return nil - } return h.db.Close() } diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go index 0b7b09f07..d2c80820b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go @@ -22,49 +22,46 @@ func errOrFirst(prev, cur error) error { } // HotService commits one ledger to the shared per-chunk hot DB as ONE atomic -// synced WriteBatch across all enabled CFs (decision (a)) and emits per-ledger +// synced WriteBatch across all hot CFs (decision (a)) and emits per-ledger // wall-clock + per-type volume signals. No fan-out — the three types are CFs of // one RocksDB committing in one WriteBatch (hotchunk.DB.IngestLedger). type HotService struct { db *hotchunk.DB - cfg hotchunk.Ingest sink MetricSink } -// NewHotService builds a HotService that writes the data types enabled in cfg -// into the shared per-chunk DB. A nil sink defaults to NopSink. -func NewHotService(db *hotchunk.DB, cfg hotchunk.Ingest, sink MetricSink) *HotService { - return &HotService{db: db, cfg: cfg, sink: orNop(sink)} +// NewHotService builds a HotService that writes ledgers, txhash, and events into +// the shared per-chunk DB. A nil sink defaults to NopSink. +func NewHotService(db *hotchunk.DB, sink MetricSink) *HotService { + return &HotService{db: db, sink: orNop(sink)} } +var errNilHotDB = errors.New("ingest: nil hot DB") + // Ingest commits lcm to the shared hot DB in one atomic synced WriteBatch // (decision (a)). HotLedgerTotal is emitted regardless of success; on success, -// one HotIngest per enabled type reports its item count. A nil DB (no hot tier) -// is a no-op other than the aggregate timing. +// one HotIngest per hot data type reports its item count. func (s *HotService) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { start := time.Now() if s.db == nil { - s.sink.HotLedgerTotal(time.Since(start)) - return nil + d := time.Since(start) + s.emit(hotchunk.LedgerCounts{}, d, errNilHotDB) + s.sink.HotLedgerTotal(d) + return errNilHotDB } - counts, err := s.db.IngestLedger(seq, lcm, s.cfg) - s.emit(counts, time.Since(start), err) - s.sink.HotLedgerTotal(time.Since(start)) + counts, err := s.db.IngestLedger(seq, lcm) + d := time.Since(start) + s.emit(counts, d, err) + s.sink.HotLedgerTotal(d) return err } -// emit reports one HotIngest per enabled type. On error, counts are 0 with the +// emit reports one HotIngest per hot data type. On error, counts are 0 with the // error attached (a failed atomic commit wrote nothing durably). func (s *HotService) emit(counts hotchunk.LedgerCounts, d time.Duration, err error) { - if s.cfg.Ledgers { - s.sink.HotIngest(dataTypeLedgers, d, itemsOnSuccess(counts.Ledgers, err), err) - } - if s.cfg.Txhash { - s.sink.HotIngest(dataTypeTxhash, d, itemsOnSuccess(counts.Txhash, err), err) - } - if s.cfg.Events { - s.sink.HotIngest(dataTypeEvents, d, itemsOnSuccess(counts.Events, err), err) - } + s.sink.HotIngest(dataTypeLedgers, d, itemsOnSuccess(counts.Ledgers, err), err) + s.sink.HotIngest(dataTypeTxhash, d, itemsOnSuccess(counts.Txhash, err), err) + s.sink.HotIngest(dataTypeEvents, d, itemsOnSuccess(counts.Events, err), err) } // itemsOnSuccess returns n on success and 0 on error — a failed atomic batch diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go index c35871b22..1607bceab 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go @@ -29,7 +29,7 @@ import ( // This file provides the shared test scaffolding the lifecycle tests need. The // catalog/fixture helpers are copied verbatim from the root fullhistory package's // helpers_test.go (which still serves the root tests). The hot-tier helpers -// (allHotTypes / openHotDBForChunk / openLiveHotDB / NewRocksHotProbe) are +// (openHotDBForChunk / openLiveHotDB / NewRocksHotProbe) are // test-local equivalents of the production hot-source primitives that live in the // root fullhistory package — the lifecycle package cannot import root (root imports // lifecycle), so the lifecycle tests rebuild them over the same public store APIs. @@ -122,17 +122,13 @@ func zeroTxLCMBytes(t *testing.T, seq uint32) []byte { // --------------------------------------------------------------------------- // Hot-tier test scaffolding: test-local equivalents of the root package's -// production hot-source primitives (ingest.go's openHotDBForChunk/allHotTypes +// production hot-source primitives (ingest.go's openHotDBForChunk // and hotsource.go's rocksHotProbe/NewRocksHotProbe). They use only the public // hotchunk/ledger/catalog/backfill APIs the production code uses, so a lifecycle // test reads and freezes the SAME on-disk hot DB the real daemon would, without // importing the root fullhistory package (which would be an import cycle). // --------------------------------------------------------------------------- -// allHotTypes is the hot tier's ingest selection (all three CFs), mirroring the -// production ingest config. -var allHotTypes = hotchunk.Ingest{Ledgers: true, Txhash: true, Events: true} - // openHotDBForChunk creates a "ready" shared hot DB for chunkID under the // hot:chunk bracket (transient -> create -> ready) and returns an open handle the // caller owns. The test equivalent of the production opener, trimmed to the diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go index 98298e327..a230d849b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go @@ -98,7 +98,7 @@ func ingestFullHotChunk(t *testing.T, cat *catalog.Catalog, c chunk.ID) { } else { raw = zeroTxLCMBytes(t, seq) } - _, err := db.IngestLedger(seq, xdr.LedgerCloseMetaView(raw), allHotTypes) + _, err := db.IngestLedger(seq, xdr.LedgerCloseMetaView(raw)) require.NoError(t, err) } require.NoError(t, db.Close()) // release the write handle (boundary handoff) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index dffe212bf..0660a58f7 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -128,14 +128,6 @@ func (d *DB) MaxCommittedSeq() (seq uint32, ok bool, err error) { return d.ledger.LastSeq() } -// Ingest toggles which data types the single per-ledger batch writes. Mirrors -// ingest.Config but kept local so hotchunk needn't depend on ingest. -type Ingest struct { - Ledgers bool - Txhash bool - Events bool -} - // LedgerCounts reports how many items each data type contributed to one // IngestLedger call, so the caller can emit per-type volume metrics. type LedgerCounts struct { @@ -145,14 +137,14 @@ type LedgerCounts struct { } // IngestLedger commits ONE ledger as a SINGLE atomic synced WriteBatch across all -// enabled CFs (decision (a)): queue each enabled type's rows into one -// BatchWriter, commit once, and only then apply the events in-memory -// mirror/offsets update. +// hot CFs (decision (a)): queue ledgers, txhash, and events rows into one +// BatchWriter, commit once, and only then apply the events in-memory mirror/offsets +// update. // // lcm is a borrowed zero-copy view; every extractor copies what it retains, so // the view need not outlive this call. An idempotent-duplicate events ledger // contributes nothing (nil apply hook) while the upsert-keyed CFs still write. -func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView, cfg Ingest) (LedgerCounts, error) { +func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts, error) { var counts LedgerCounts if d.store.IsClosed() { return counts, stores.ErrStoreClosed @@ -160,56 +152,41 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView, cfg Ingest) ( // Pre-extract anything that can fail BEFORE opening the batch, so a decode // error rejects the ledger without a half-built batch. - var txEntries []txhash.Entry - if cfg.Txhash { - hashes, err := sdkingest.ExtractTxHashes(lcm) - if err != nil { - return counts, fmt.Errorf("extract tx hashes seq %d: %w", seq, err) - } - if len(hashes) > 0 { - txEntries = make([]txhash.Entry, len(hashes)) - for i, h := range hashes { - txEntries[i] = txhash.Entry{Hash: [32]byte(h), LedgerSeq: seq} - } - } - counts.Txhash = len(hashes) + hashes, err := sdkingest.ExtractTxHashes(lcm) + if err != nil { + return counts, fmt.Errorf("extract tx hashes seq %d: %w", seq, err) } - - var payloads []events.Payload - if cfg.Events { - p, err := eventPayloads(seq, lcm) - if err != nil { - return counts, err - } - payloads = p - counts.Events = len(payloads) + txEntries := make([]txhash.Entry, len(hashes)) + for i, h := range hashes { + txEntries[i] = txhash.Entry{Hash: [32]byte(h), LedgerSeq: seq} } - if cfg.Ledgers { - counts.Ledgers = 1 + counts.Txhash = len(hashes) + + payloads, err := eventPayloads(seq, lcm) + if err != nil { + return counts, err } + counts.Events = len(payloads) + counts.Ledgers = 1 // The events facade validates + marshals up front (so a rejected ledger // never touches the batch) and returns the post-commit apply hook (nil for // an idempotent duplicate). var applyEvents func() cerr := d.store.Batch(func(b *rocksdb.BatchWriter) error { - if cfg.Ledgers { - if err := d.ledger.AddLedgerToBatch(b, ledger.Entry{Seq: seq, Bytes: []byte(lcm)}); err != nil { - return fmt.Errorf("queue ledger seq %d: %w", seq, err) - } + if err := d.ledger.AddLedgerToBatch(b, ledger.Entry{Seq: seq, Bytes: []byte(lcm)}); err != nil { + return fmt.Errorf("queue ledger seq %d: %w", seq, err) } - if cfg.Txhash && len(txEntries) > 0 { + if len(txEntries) > 0 { if err := d.txhash.AddEntriesToBatch(b, txEntries); err != nil { return fmt.Errorf("queue tx hashes seq %d: %w", seq, err) } } - if cfg.Events { - apply, err := d.events.IngestLedgerToBatch(b, seq, payloads) - if err != nil { - return fmt.Errorf("queue events seq %d: %w", seq, err) - } - applyEvents = apply + apply, err := d.events.IngestLedgerToBatch(b, seq, payloads) + if err != nil { + return fmt.Errorf("queue events seq %d: %w", seq, err) } + applyEvents = apply return nil }) if cerr != nil { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go index 842b65325..5855b4a7a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go @@ -38,8 +38,6 @@ func openTestDB(t *testing.T, chunkID chunk.ID) *DB { return db } -func allTypes() Ingest { return Ingest{Ledgers: true, Txhash: true, Events: true} } - func TestOpen_ValidatesInputs(t *testing.T) { _, err := Open("", chunk.ID(0), silentLogger()) require.ErrorIs(t, err, stores.ErrInvalidConfig) @@ -83,11 +81,11 @@ func TestIngestLedger_AllCFsAdvanceTogether(t *testing.T) { rawA, hashA, termA := lcmWithEvent(t, first) rawB, hashB, _ := lcmWithEvent(t, first+1) - counts, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(rawA), allTypes()) + counts, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(rawA)) require.NoError(t, err) assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 1, Events: 1}, counts) - counts, err = db.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawB), allTypes()) + counts, err = db.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawB)) require.NoError(t, err) assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 1, Events: 1}, counts) @@ -131,7 +129,7 @@ func TestIngestLedger_RejectedLedgerPersistsNothingAcrossAnyCF(t *testing.T) { badSeq := chunkID.LastLedger() + 1 raw, hash, term := lcmWithEvent(t, badSeq) - _, err := db.IngestLedger(badSeq, xdr.LedgerCloseMetaView(raw), allTypes()) + _, err := db.IngestLedger(badSeq, xdr.LedgerCloseMetaView(raw)) require.Error(t, err) require.ErrorIs(t, err, eventstore.ErrLedgerOutOfRange) @@ -167,7 +165,7 @@ func TestIngestLedger_MidBatchCommitFailurePersistsNothing(t *testing.T) { // Commit one good ledger so there is a known watermark, then close the DB. rawGood, hashGood, _ := lcmWithEvent(t, first) - _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(rawGood), allTypes()) + _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(rawGood)) require.NoError(t, err) require.NoError(t, db.Close()) @@ -185,7 +183,7 @@ func TestIngestLedger_MidBatchCommitFailurePersistsNothing(t *testing.T) { // store: the commit fails, and nothing for that ledger persists anywhere. require.NoError(t, db2.Close()) rawNext, hashNext, _ := lcmWithEvent(t, first+1) - _, err = db2.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawNext), allTypes()) + _, err = db2.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawNext)) require.Error(t, err) // Reopen a third time: the failed ledger left NO trace in any CF, and the @@ -253,26 +251,29 @@ func TestSharedBatch_DirectRocksAbortAcrossCFs(t *testing.T) { // package, so no production accessor is needed). func storeOf(db *DB) *rocksdb.Store { return db.store } -// TestIngestLedger_DisabledTypesUntouched confirms the Ingest toggles select -// which CFs the single batch writes: ledgers-only leaves txhash/events empty. -func TestIngestLedger_DisabledTypesUntouched(t *testing.T) { +// TestIngestLedger_WritesEveryHotType confirms the hot tier always writes all +// three hot data types; per-type disabling is not a supported hot DB mode. +func TestIngestLedger_WritesEveryHotType(t *testing.T) { chunkID := chunk.ID(0) first := chunkID.FirstLedger() db := openTestDB(t, chunkID) raw, hash, term := lcmWithEvent(t, first) - counts, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw), Ingest{Ledgers: true}) + counts, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) require.NoError(t, err) - assert.Equal(t, LedgerCounts{Ledgers: 1}, counts) + assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 1, Events: 1}, counts) got, err := db.Ledgers().GetLedgerRaw(first) require.NoError(t, err) assert.Equal(t, raw, got) - _, gerr := db.Txhash().Get(hash) - require.ErrorIs(t, gerr, stores.ErrNotFound) - _, lerr := db.Events().Lookup(context.Background(), term) - require.ErrorIs(t, lerr, eventstore.ErrTermNotFound) + seq, err := db.Txhash().Get(hash) + require.NoError(t, err) + assert.Equal(t, first, seq) + bm, err := db.Events().Lookup(context.Background(), term) + require.NoError(t, err) + require.NotNil(t, bm) + assert.Equal(t, uint64(1), bm.GetCardinality()) } // TestReopen_RecoversEventsMirror confirms the events facade's warmup runs over @@ -286,7 +287,7 @@ func TestReopen_RecoversEventsMirror(t *testing.T) { db, err := Open(dir, chunkID, silentLogger()) require.NoError(t, err) raw, _, _ := lcmWithEvent(t, first) - _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(raw), allTypes()) + _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) require.NoError(t, err) require.NoError(t, db.Close()) @@ -309,7 +310,7 @@ func TestOpenReadOnly_ReadsCommittedAndRejectsWrites(t *testing.T) { db, err := Open(dir, chunkID, silentLogger()) require.NoError(t, err) for _, seq := range []uint32{first, first + 1} { - _, ierr := db.IngestLedger(seq, xdr.LedgerCloseMetaView(zeroTxLCM(t, seq)), allTypes()) + _, ierr := db.IngestLedger(seq, xdr.LedgerCloseMetaView(zeroTxLCM(t, seq))) require.NoError(t, ierr) } require.NoError(t, db.Close()) @@ -325,7 +326,7 @@ func TestOpenReadOnly_ReadsCommittedAndRejectsWrites(t *testing.T) { assert.Equal(t, first+1, seq, "read-only handle sees the committed data") // A write through the read-only handle must fail — the freeze never mutates. - _, err = ro.IngestLedger(first+2, xdr.LedgerCloseMetaView(zeroTxLCM(t, first+2)), allTypes()) + _, err = ro.IngestLedger(first+2, xdr.LedgerCloseMetaView(zeroTxLCM(t, first+2))) require.Error(t, err, "read-only DB must reject writes") } @@ -337,7 +338,7 @@ func TestIngestLedger_ClosedDBFails(t *testing.T) { require.NoError(t, db.Close()) raw := zeroTxLCM(t, chunkID.FirstLedger()) - _, err = db.IngestLedger(chunkID.FirstLedger(), xdr.LedgerCloseMetaView(raw), allTypes()) + _, err = db.IngestLedger(chunkID.FirstLedger(), xdr.LedgerCloseMetaView(raw)) require.ErrorIs(t, err, stores.ErrStoreClosed) } diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index 04cc1d43c..6624fee9e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -138,7 +138,7 @@ func run(ctx context.Context, cfg StartConfig) error { // The ingestion loop owns hotDB for the rest of its life (closes it on any exit, // reopens at each boundary). Returns the GetLedger/boundary error; the daemon top // level classifies a ctx-cancelled return as a clean shutdown. - return runIngestionLoop(ctx, core, hotDB, cat, lifecycleCh, allHotTypes, logger, metrics, cfg.Exec.Process.Sink) + return runIngestionLoop(ctx, core, hotDB, cat, lifecycleCh, logger, metrics, cfg.Exec.Process.Sink) } // backfillToTip runs the backfill loop, returning lastCommitted as backfill makes From 5cabb45f7e4ecaaa44197241c7fd2e366791b2be Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 02:51:57 -0400 Subject: [PATCH 14/55] Recover hot progress from WAL on startup --- .../internal/fullhistory/daemon.go | 3 ++- .../internal/fullhistory/hotsource.go | 27 +++++++++++++++---- .../fullhistory/lifecycle/progress.go | 7 ++--- .../internal/fullhistory/startup.go | 16 ++++++++--- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index d975db981..c90cd91c3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -205,7 +205,8 @@ func startConfig( }, } return StartConfig{ - Exec: exec, + Exec: exec, + HotProgressProbe: NewRocksHotRecoveryProbe(cat.Layout().HotChunkPath, logger), Lifecycle: lifecycle.LifecycleConfig{ ExecConfig: exec, RetentionChunks: deref(cfg.Retention.RetentionChunks), diff --git a/cmd/stellar-rpc/internal/fullhistory/hotsource.go b/cmd/stellar-rpc/internal/fullhistory/hotsource.go index 3a66176ce..0c415b911 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotsource.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotsource.go @@ -22,6 +22,7 @@ import ( type rocksHotProbe struct { hotRoot func(chunkID chunk.ID) string logger *supportlog.Entry + recover bool } // NewRocksHotProbe returns the production backfill.HotProbe (hotChunkPath maps a @@ -34,6 +35,14 @@ func NewRocksHotProbe(hotChunkPath func(chunk.ID) string, logger *supportlog.Ent return &rocksHotProbe{hotRoot: hotChunkPath, logger: logger} } +// NewRocksHotRecoveryProbe returns the startup progress probe. Unlike the freeze +// probe, it opens the highest ready hot DB read-write so RocksDB replays any +// synced WAL left by an ungraceful crash before MaxCommittedSeq is read. Startup +// uses it before ingestion opens a writer, then closes it immediately. +func NewRocksHotRecoveryProbe(hotChunkPath func(chunk.ID) string, logger *supportlog.Entry) backfill.HotProbe { + return &rocksHotProbe{hotRoot: hotChunkPath, logger: logger, recover: true} +} + func (p *rocksHotProbe) OpenHotChunk(chunkID chunk.ID) (backfill.HotChunk, bool, error) { dir := p.hotRoot(chunkID) if _, err := os.Stat(dir); err != nil { @@ -43,11 +52,19 @@ func (p *rocksHotProbe) OpenHotChunk(chunkID chunk.ID) (backfill.HotChunk, bool, return nil, false, fmt.Errorf("stat hot dir %s: %w", dir, err) } - // Open the chunk's shared multi-CF DB READ-ONLY: the freeze reads its ledgers - // to re-derive the cold artifacts and must never mutate it (the design's - // openRocksDBReadOnly). The probe only ever opens a chunk ingestion already - // released, so its data is fully in SST — no concurrent writer, no WAL replay. - db, err := hotchunk.OpenReadOnly(dir, chunkID, p.logger) + var ( + db *hotchunk.DB + err error + ) + if p.recover { + db, err = hotchunk.Open(dir, chunkID, p.logger) + } else { + // Open the chunk's shared multi-CF DB READ-ONLY: the freeze reads its + // ledgers to re-derive the cold artifacts and must never mutate it. The + // freeze only targets chunks ingestion already released, so its data is in + // SST (no concurrent writer, no WAL replay needed). + db, err = hotchunk.OpenReadOnly(dir, chunkID, p.logger) + } if err != nil { return nil, false, fmt.Errorf("open hot chunk DB: %w", err) } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go index bd66d22a8..22b3bd6fc 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go @@ -75,9 +75,10 @@ func LastCommittedLedger(cat *catalog.Catalog, probe backfill.HotProbe) (uint32, return through, nil } -// refineWithHotDB opens the highest ready hot chunk read-only and returns its -// MaxCommittedSeq, or CompleteThrough(live-1) on an empty DB. A "ready" key whose -// dir/DB is gone surfaces as backfill.ErrHotVolumeLost (lazy loss detection). +// refineWithHotDB opens the highest ready hot chunk through probe and returns +// its MaxCommittedSeq, or CompleteThrough(live-1) on an empty DB. A "ready" key +// whose dir/DB is gone surfaces as backfill.ErrHotVolumeLost (lazy loss +// detection). func refineWithHotDB(cat *catalog.Catalog, probe backfill.HotProbe, live int64) (uint32, error) { id := chunk.ID(live) //nolint:gosec // live > cold >= -1, so live >= 0 hot, ok, openErr := probe.OpenHotChunk(id) diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index 6624fee9e..df00187e9 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -44,9 +44,14 @@ func run(ctx context.Context, cfg StartConfig) error { // Derived, never stored: highest durably-committed ledger (frozen cold artifacts // vs the highest ready hot DB's max committed seq), clamped by earliest-1. The - // probe does ONE read of the highest ready hot DB and detects hot-volume loss - // LAZILY on that open (ErrHotVolumeLost) before ingestion ever opens a writer. - lastCommitted, err := lifecycle.LastCommittedLedger(cat, cfg.Exec.Process.HotProbe) + // startup probe opens the highest ready hot DB before ingestion opens a writer, + // so production uses a recovery probe that replays any synced WAL from an + // ungraceful crash before MaxCommittedSeq is read. + hotProgressProbe := cfg.HotProgressProbe + if hotProgressProbe == nil { + hotProgressProbe = cfg.Exec.Process.HotProbe + } + lastCommitted, err := lifecycle.LastCommittedLedger(cat, hotProgressProbe) if err != nil { return fmt.Errorf("startup derive last-committed: %w", err) } @@ -260,6 +265,11 @@ type StartConfig struct { // Exec drives backfill's RunBackfill; its Catalog/Logger are the shared ones. Exec backfill.ExecConfig + // HotProgressProbe refines startup's last-committed ledger from the highest + // ready hot DB. Production uses a read-write recovery probe so RocksDB replays + // synced WAL after crashes; nil falls back to Exec.Process.HotProbe for tests. + HotProgressProbe backfill.HotProbe + // Lifecycle drives the lifecycle goroutine. Its embedded ExecConfig is the SAME // wiring as Exec (one catalog, one pool); RetentionChunks is the backfill floor's // width too (0 ⇒ the earliest-ledger floor only). From 27aae3091610b5df9d694a2af36f99ecb8dc77d6 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 11:50:43 -0400 Subject: [PATCH 15/55] fullhistory: seed events contiguity in boundary test for always-on hot tier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Always-on events ingestion (the ingestTypes removal) means the events CF now requires strict per-ledger contiguity from the chunk's first ledger, so the boundary test's sparse ledgers-CF watermark no longer produces a valid hot DB — ingesting the boundary ledger failed with ErrLedgerOutOfOrder before the boundary, so the completed-chunk notification never fired. Seed a zero-event offset for every ledger up to the watermark and mark the test t.Parallel() so its per-ledger synced-commit cost overlaps the other heavy full-chunk tests. --- .../internal/fullhistory/hotloop_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index 2c62557d4..211741e6d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -99,15 +99,20 @@ func openLiveHotDB(t *testing.T, cat *catalog.Catalog, c chunk.ID) *hotchunk.DB return db } -// seedWatermark writes a single ledgers-CF entry at seq into the chunk's hot DB -// so the indexed poll resumes at seq+1 — letting a boundary test drive the loop -// over only the last ledger or two of a chunk instead of all 10,000. The -// returned DB is the (re-opened, ready) live handle the loop then owns. The -// zero-event fixtures keep the sparse ledgers-CF watermark valid for boundary -// tests without preloading all preceding event offsets. +// seedWatermark advances a chunk's hot DB to a last-committed ledger of seq so +// the indexed poll resumes at seq+1, letting a boundary test drive the loop over +// only the last ledger or two of a chunk. The always-on events CF requires +// strict ledger contiguity from the chunk's first ledger, so it seeds a +// zero-event offset for every ledger up to seq; the ledgers CF only needs the +// watermark entry, since MaxCommittedSeq is its last key. The returned DB is the +// (re-opened, ready) live handle the loop then owns. Seeding a near-full chunk +// costs one synced commit per ledger, so its callers run t.Parallel(). func seedWatermark(t *testing.T, cat *catalog.Catalog, c chunk.ID, seq uint32) *hotchunk.DB { t.Helper() db := openLiveHotDB(t, cat, c) + for s := c.FirstLedger(); s <= seq; s++ { + require.NoError(t, db.Events().IngestLedgerEvents(s, nil)) + } require.NoError(t, db.Ledgers().AddLedgers(ledgerEntry(t, seq))) require.NoError(t, db.Close()) reopened, err := openHotDBForChunk(cat, c, silentLogger()) @@ -234,6 +239,7 @@ func TestRunIngestionLoop_LedgerLandsAcrossAllCFs(t *testing.T) { // is far above the at-most-one a healthy daemon holds, so it never blocks the // loop. func TestRunIngestionLoop_BoundaryNotifiesCompletedChunk(t *testing.T) { + t.Parallel() // seeds a near-full chunk (one synced commit per ledger) cat, _ := testCatalog(t) c := chunk.ID(0) c1 := c + 1 From 51124303eedc34c5dd7844decfa6b7a18f901a76 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 12:26:16 -0400 Subject: [PATCH 16/55] fullhistory: align hot txhash tier + index catalog key to design #787 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two design-conformance fixes against the streaming/getTransaction design docs (PR #787), both correcting choices inherited from the Phase 1 base: - Hot txhash tier: collapse the 16 nibble-routed CFs (cf-0..cf-f) to the single `txhash` CF the design specifies (gettransaction §5.1, streaming data-model). Removes the nibble routing table, numCFs, cfNames() and cfNameForTxHash(); key/value semantics (full 32B hash -> 4B seq, point lookup) are unchanged. Tuning comments that referenced the 16-CF budget are corrected. - Index catalog key prefix: rename `txhash_index:` -> `index:` to match the design's catalog-key schema (streaming "Index keys", gettransaction §6.2). Only the durable key string changes; the Go identifiers keep their names. Verified: go build + go test -short green across ./fullhistory/... (rocksdb cgo toolchain). --- .../internal/fullhistory/geometry/keys.go | 8 +- .../fullhistory/geometry/keys_test.go | 8 +- .../pkg/stores/hotchunk/hotchunk.go | 6 +- .../pkg/stores/hotchunk/hotchunk_test.go | 2 +- .../pkg/stores/txhash/hot_store.go | 97 +++++++------------ .../pkg/stores/txhash/hot_store_test.go | 68 ++++--------- 6 files changed, 68 insertions(+), 121 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/geometry/keys.go b/cmd/stellar-rpc/internal/fullhistory/geometry/keys.go index e1fc33ff8..f8d054f3a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/geometry/keys.go +++ b/cmd/stellar-rpc/internal/fullhistory/geometry/keys.go @@ -79,7 +79,7 @@ func (i TxHashIndexID) String() string { return fmt.Sprintf("%08d", uint32(i)) } const ( ChunkPrefix = "chunk:" HotChunkPrefix = "hot:chunk:" - TxHashIndexPrefix = "txhash_index:" + TxHashIndexPrefix = "index:" // ConfigEarliestLedger is the sole config pin key. (chunks_per_txhash_index is // the fixed ChunksPerTxhashIndex constant, not a pin.) @@ -97,7 +97,7 @@ func HotChunkKey(c chunk.ID) string { return HotChunkPrefix + c.String() } -// TxHashIndexKey returns the index coverage key txhash_index:{idx:08d}:{lo:08d}:{hi:08d}. +// TxHashIndexKey returns the index coverage key index:{idx:08d}:{lo:08d}:{hi:08d}. // The coverage [lo, hi] lives in the key NAME; the value is pure lifecycle // state. lo > hi is a programmer error, surfaced loudly via panic. func TxHashIndexKey(idx TxHashIndexID, lo, hi chunk.ID) string { @@ -107,7 +107,7 @@ func TxHashIndexKey(idx TxHashIndexID, lo, hi chunk.ID) string { return TxHashIndexPrefix + idx.String() + ":" + lo.String() + ":" + hi.String() } -// TxHashIndexPrefixFor returns the scan prefix txhash_index:{idx:08d}: that enumerates +// TxHashIndexPrefixFor returns the scan prefix index:{idx:08d}: that enumerates // all coverage keys of one index. func TxHashIndexPrefixFor(idx TxHashIndexID) string { return TxHashIndexPrefix + idx.String() + ":" @@ -163,7 +163,7 @@ func ParseHotChunkKey(key string) (chunk.ID, bool) { return chunk.ID(n), true } -// ParseTxHashIndexKey decodes txhash_index:{idx:08d}:{lo:08d}:{hi:08d}. State is not part +// ParseTxHashIndexKey decodes index:{idx:08d}:{lo:08d}:{hi:08d}. State is not part // of the key; callers fill TxHashIndexCoverage.State from the scanned value. func ParseTxHashIndexKey(key string) (TxHashIndexCoverage, bool) { rest, found := strings.CutPrefix(key, TxHashIndexPrefix) diff --git a/cmd/stellar-rpc/internal/fullhistory/geometry/keys_test.go b/cmd/stellar-rpc/internal/fullhistory/geometry/keys_test.go index 971c82030..93b9a64a4 100644 --- a/cmd/stellar-rpc/internal/fullhistory/geometry/keys_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/geometry/keys_test.go @@ -16,7 +16,7 @@ func TestKeyConstructorsMatchSpec(t *testing.T) { require.Equal(t, "chunk:00005350:ledgers", ChunkKey(5350, KindLedgers)) require.Equal(t, "chunk:00005350:events", ChunkKey(5350, KindEvents)) require.Equal(t, "chunk:00005350:txhash", ChunkKey(5350, KindTxHash)) - require.Equal(t, "txhash_index:00000005:00005100:00005349", TxHashIndexKey(5, 5100, 5349)) + require.Equal(t, "index:00000005:00005100:00005349", TxHashIndexKey(5, 5100, 5349)) } func TestChunkKeyBijection(t *testing.T) { @@ -65,8 +65,8 @@ func TestParseRejectsMalformed(t *testing.T) { "chunk:5350:ledgers", // not 8-digit padded "chunk:00005350:bogus", // unknown kind "chunk:00005350", // missing kind - "txhash_index:00000005:00005100", // too few segments - "txhash_index:5:5100:5349", // not padded + "index:00000005:00005100", // too few segments + "index:5:5100:5349", // not padded "unrelated:key", // wrong family } for _, key := range bad { @@ -77,7 +77,7 @@ func TestParseRejectsMalformed(t *testing.T) { // Specific rejections. _, _, ok := ParseChunkKey("chunk:00005350:bogus") require.False(t, ok) - _, ok2 := ParseTxHashIndexKey("txhash_index:00000005:00005349:00005100") // lo > hi + _, ok2 := ParseTxHashIndexKey("index:00000005:00005349:00005100") // lo > hi require.False(t, ok2) } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index 0660a58f7..4933b9e59 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -1,6 +1,6 @@ // Package hotchunk implements decision (a): the per-chunk hot tier is ONE -// RocksDB holding the union of every hot data type's CFs (ledger + 3 events + 16 -// nibble-routed txhash), and each ledger commits as ONE atomic synced WriteBatch +// RocksDB holding the union of every hot data type's CFs (ledger + 3 events + 1 +// txhash), and each ledger commits as ONE atomic synced WriteBatch // across ALL of them — so a ledger is fully present or fully absent, with a // SINGLE per-chunk last-committed ledger (max committed seq, from the ledgers CF's last key) // and no per-store frontiers / min-of-three. The three typed facades @@ -41,7 +41,7 @@ type DB struct { } // columnFamilies is the full CF list for the shared per-chunk DB (ledger + 3 -// events + 16 txhash). Names are already non-colliding across the facades. +// events + 1 txhash). Names are already non-colliding across the facades. func columnFamilies() []string { cfs := []string{ledger.LedgersCF} cfs = append(cfs, eventstore.CFNames()...) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go index 5855b4a7a..c82659a95 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go @@ -231,7 +231,7 @@ func TestSharedBatch_DirectRocksAbortAcrossCFs(t *testing.T) { err := storeOf(db).Batch(func(b *rocksdb.BatchWriter) error { b.Put(ledger.LedgersCF, rocksdb.EncodeUint32(2), []byte("ledger-row")) - b.Put(txhash.CFNames()[0xa], hash[:], rocksdb.EncodeUint32(2)) + b.Put(txhash.CFNames()[0], hash[:], rocksdb.EncodeUint32(2)) b.Put(eventstore.DataCF, []byte{0, 0, 0, 0}, []byte("event-row")) return sentinelErr // abort: nothing should commit }) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go index 6d929eeab..9698c35b2 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go @@ -1,6 +1,6 @@ -// Package txhash holds the hot transaction-hash store (RocksDB-backed, -// 16-CF nibble-routed) and its value types. A future cold reader -// (RecSplit-backed) will live alongside the HotStore in this package. +// Package txhash holds the hot transaction-hash store (RocksDB-backed, a single +// txhash CF) and its value types. A future cold reader (RecSplit-backed) will +// live alongside the HotStore in this package. package txhash import ( @@ -11,19 +11,9 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores" ) -// 16 CFs — one per high-nibble bucket of byte 0 of the txhash. -// Same routing the cold RecSplit index uses. -const numCFs = 16 - -// cfNameByNibble is the precomputed (cf-0..cf-f) table indexed by -// hash[0]>>4. Single source of truth used by both cfNames (open-time -// CF list) and cfNameForTxHash (hot path). -// -//nolint:gochecknoglobals -var cfNameByNibble = [16]string{ - "cf-0", "cf-1", "cf-2", "cf-3", "cf-4", "cf-5", "cf-6", "cf-7", - "cf-8", "cf-9", "cf-a", "cf-b", "cf-c", "cf-d", "cf-e", "cf-f", -} +// txhashCF is the single column family holding every (txhash → ledgerSeq) +// entry for the chunk, per the design's hot-tier spec (one `txhash` CF). +const txhashCF = "txhash" // Entry — one (txhash → ledgerSeq) mapping. type Entry struct { @@ -31,9 +21,9 @@ type Entry struct { LedgerSeq uint32 } -// HotStore — RocksDB-backed hot transaction-hash store. 16 CFs named -// cf-0..cf-f; each hash routes to cf-{txhash[0]>>4}; ledgerSeq -// encoded big-endian. Routing, CF names, and encoding are internal. +// HotStore — RocksDB-backed hot transaction-hash store. A single txhash CF +// holding the full 32-byte hash as key and the big-endian ledgerSeq as value. +// The CF name and encoding are internal. // // Like every hot store, a HotStore instance is chunk-bound: it // accumulates exactly one chunk's (txhash → seq) tuples before being @@ -62,7 +52,7 @@ func NewHotStore(path string, chunkID chunk.ID, logger *supportlog.Entry) (*HotS } store, err := rocksdb.New(rocksdb.Config{ Path: path, - ColumnFamilies: cfNames(), + ColumnFamilies: CFNames(), Logger: logger, Tuning: tuning(), }) @@ -73,43 +63,29 @@ func NewHotStore(path string, chunkID chunk.ID, logger *supportlog.Entry) (*HotS } // NewWithStore wraps an ALREADY-OPEN rocksdb.Store as a txhash HotStore on the -// 16 nibble-routed CFs (CFNames()). The store is NOT owned (Close is a no-op) — +// single txhash CF (CFNames()). The store is NOT owned (Close is a no-op) — // the constructor hotchunk uses to compose this facade over the shared per-chunk // DB. The store must have CFNames() registered. func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) *HotStore { return &HotStore{store: store, chunkID: chunkID} } -// CFNames returns the 16 nibble-routed CF names this facade owns. Exported so -// the hotchunk shared-DB opener can register them alongside the other CFs. -func CFNames() []string { return cfNames() } +// CFNames returns the single txhash CF name this facade owns. Exported so +// the hotchunk shared-DB opener can register it alongside the other CFs. +func CFNames() []string { return []string{txhashCF} } // Tuning returns this facade's RocksDB tuning, applied to the shared per-chunk // DB by the hotchunk opener. func Tuning() rocksdb.Tuning { return tuning() } -func cfNames() []string { - out := make([]string, numCFs) - copy(out, cfNameByNibble[:]) - return out -} - -func cfNameForTxHash(hash [32]byte) string { - return cfNameByNibble[hash[0]>>4] -} - -// tuning — the hot txhash workload is write-once / point-lookup over -// 16 CFs; the cross-knob interactions below are non-obvious enough -// that they get an explicit per-stanza rationale. The other facades -// ride on RocksDB defaults by contrast — only this workload earned -// the calibration. +// tuning — the hot txhash workload is write-once / point-lookup; the +// cross-knob interactions below are non-obvious enough that they get an +// explicit per-stanza rationale. The other facades ride on RocksDB defaults +// by contrast — only this workload earned the calibration. func tuning() rocksdb.Tuning { return rocksdb.Tuning{ - // Per-CF memtable budget × 16 CFs (64 MB × 16 = 1024 MB) - // matches the MaxTotalWalSizeMB cap below. Memtable-fill - // cadence and WAL-cap cadence align under uniform writes; - // either trigger fires at roughly the same time and produces - // ~64 MB SSTs. + // 64 MB memtable so one flush produces one ~64 MB SST under + // uniform writes. WriteBufferMB: 64, MaxWriteBufferNumber: 2, @@ -137,8 +113,7 @@ func tuning() rocksdb.Tuning { TargetFileSizeMB: 64, MaxBytesForLevelBaseMB: 256, - // High background-job budget for the periodic memtable - // flushes across 16 CFs. + // Background-job budget for the periodic memtable flushes. MaxBackgroundJobs: 8, MaxOpenFiles: 10_000, @@ -151,10 +126,9 @@ func tuning() rocksdb.Tuning { BlockCacheMB: 512, BloomFilterBitsPerKey: 12, - // 1 GB WAL cap matches the natural memtable budget above. - // Graceful Close auto-Flushes (see rocksdb.Store.Close), so - // this cap only bounds ungraceful-shutdown recovery (kernel - // panic, power loss, OOM kill). + // 1 GB WAL cap. Graceful Close auto-Flushes (see + // rocksdb.Store.Close), so this cap only bounds ungraceful-shutdown + // recovery (kernel panic, power loss, OOM kill). MaxTotalWalSizeMB: 1024, } } @@ -173,9 +147,8 @@ func (h *HotStore) Close() error { // never reads the store). func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } -// AddEntries writes a batch of (txhash → ledgerSeq) atomically -// across however many CFs the hashes' nibbles cover. One fsync per -// call. +// AddEntries writes a batch of (txhash → ledgerSeq) atomically to the +// txhash CF. One fsync per call. func (h *HotStore) AddEntries(entries []Entry) error { if h.store.IsClosed() { return rocksdb.ErrStoreClosed @@ -185,35 +158,35 @@ func (h *HotStore) AddEntries(entries []Entry) error { return nil case 1: e := entries[0] - return h.store.Put(cfNameForTxHash(e.Hash), e.Hash[:], rocksdb.EncodeUint32(e.LedgerSeq)) + return h.store.Put(txhashCF, e.Hash[:], rocksdb.EncodeUint32(e.LedgerSeq)) default: return h.store.Batch(func(b *rocksdb.BatchWriter) error { for _, e := range entries { - b.Put(cfNameForTxHash(e.Hash), e.Hash[:], rocksdb.EncodeUint32(e.LedgerSeq)) + b.Put(txhashCF, e.Hash[:], rocksdb.EncodeUint32(e.LedgerSeq)) } return nil }) } } -// AddEntriesToBatch queues each (txhash → ledgerSeq) Put into b on its -// nibble-routed CF — the building block hotchunk uses to fold the tx-hash writes -// into the one shared per-ledger WriteBatch (decision (a)). Does not commit -// (caller owns the batch). A closed store returns ErrStoreClosed. +// AddEntriesToBatch queues each (txhash → ledgerSeq) Put into b on the txhash +// CF — the building block hotchunk uses to fold the tx-hash writes into the one +// shared per-ledger WriteBatch (decision (a)). Does not commit (caller owns the +// batch). A closed store returns ErrStoreClosed. func (h *HotStore) AddEntriesToBatch(b *rocksdb.BatchWriter, entries []Entry) error { if h.store.IsClosed() { return rocksdb.ErrStoreClosed } for _, e := range entries { - b.Put(cfNameForTxHash(e.Hash), e.Hash[:], rocksdb.EncodeUint32(e.LedgerSeq)) + b.Put(txhashCF, e.Hash[:], rocksdb.EncodeUint32(e.LedgerSeq)) } return nil } // Get returns the ledger sequence the hash was committed in, or -// (0, stores.ErrNotFound) on miss. Only the routed CF is queried. +// (0, stores.ErrNotFound) on miss. func (h *HotStore) Get(hash [32]byte) (uint32, error) { - v, found, err := h.store.Get(cfNameForTxHash(hash), hash[:]) + v, found, err := h.store.Get(txhashCF, hash[:]) if err != nil { return 0, err } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go index c600d6141..02069ed1d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go @@ -27,6 +27,8 @@ func silentLogger() *supportlog.Entry { return log } +// txhashFor builds a distinct 32-byte hash from a (high-nibble, tag) pair — +// a convenient generator of many distinct keys for the single txhash CF. func txhashFor(nibble, tag byte) [32]byte { var h [32]byte h[0] = nibble << 4 @@ -102,26 +104,27 @@ func TestHotStore_AddGetRoundTrip(t *testing.T) { require.NoError(t, s.AddEntries([]Entry{})) } -func TestHotStore_NibbleRoutingAcrossAllCFs(t *testing.T) { +func TestHotStore_ManyDistinctKeys(t *testing.T) { s := openTestHotStore(t) - entries := make([]Entry, numCFs) - for n := range numCFs { - entries[n] = Entry{ - Hash: txhashFor(byte(n), 1), - LedgerSeq: uint32(n) * 100, + const n = 16 + entries := make([]Entry, n) + for i := range n { + entries[i] = Entry{ + Hash: txhashFor(byte(i), 1), + LedgerSeq: uint32(i) * 100, } } require.NoError(t, s.AddEntries(entries)) - for n := range numCFs { - got, err := s.Get(entries[n].Hash) - require.NoError(t, err, "nibble %x", n) - assert.Equal(t, uint32(n)*100, got, "nibble %x", n) + for i := range n { + got, err := s.Get(entries[i].Hash) + require.NoError(t, err, "key %d", i) + assert.Equal(t, uint32(i)*100, got, "key %d", i) } } -func TestHotStore_AddEntriesMultipleSpansCFs(t *testing.T) { +func TestHotStore_AddEntriesMultiple(t *testing.T) { s := openTestHotStore(t) entries := []Entry{ @@ -171,7 +174,7 @@ func TestHotStore_GracefulCloseAndReopenRoundTrips(t *testing.T) { first, err := NewHotStore(path, chunk.ID(0), silentLogger()) require.NoError(t, err) - for n := range numCFs { + for n := range 16 { require.NoError(t, first.AddEntries([]Entry{ {Hash: txhashFor(byte(n), 1), LedgerSeq: uint32(n) + 1}, })) @@ -182,7 +185,7 @@ func TestHotStore_GracefulCloseAndReopenRoundTrips(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = second.Close() }) - for n := range numCFs { + for n := range 16 { got, err := second.Get(txhashFor(byte(n), 1)) require.NoError(t, err) assert.Equal(t, uint32(n)+1, got) @@ -191,9 +194,9 @@ func TestHotStore_GracefulCloseAndReopenRoundTrips(t *testing.T) { func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { s := openTestHotStore(t) - // Pre-populate one entry per nibble. - pre := make([]Entry, numCFs) - for n := range numCFs { + // Pre-populate a spread of distinct keys. + pre := make([]Entry, 16) + for n := range 16 { pre[n] = Entry{Hash: txhashFor(byte(n), 1), LedgerSeq: uint32(n)} } require.NoError(t, s.AddEntries(pre)) @@ -205,13 +208,13 @@ func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { wg.Go(func() { for i := byte(0); !stop.Load(); i++ { _ = s.AddEntries([]Entry{ - {Hash: txhashFor(i%numCFs, byte(w+5)), LedgerSeq: uint32(i)}, + {Hash: txhashFor(i%16, byte(w+5)), LedgerSeq: uint32(i)}, }) } }) wg.Go(func() { for i := byte(0); !stop.Load(); i++ { - _, _ = s.Get(txhashFor(i%numCFs, 1)) + _, _ = s.Get(txhashFor(i%16, 1)) } }) } @@ -224,32 +227,3 @@ func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { postClose := []Entry{{Hash: txhashFor(0x1, 1), LedgerSeq: 1}} require.ErrorIs(t, s.AddEntries(postClose), rocksdb.ErrStoreClosed) } - -func TestCFNameForTxHash_AllHighNibbles(t *testing.T) { - cases := []struct { - topByte byte - want string - }{ - {0x00, "cf-0"}, - {0x10, "cf-1"}, - {0x20, "cf-2"}, - {0x30, "cf-3"}, - {0x40, "cf-4"}, - {0x50, "cf-5"}, - {0x60, "cf-6"}, - {0x70, "cf-7"}, - {0x80, "cf-8"}, - {0x90, "cf-9"}, - {0xa0, "cf-a"}, - {0xb0, "cf-b"}, - {0xc0, "cf-c"}, - {0xd0, "cf-d"}, - {0xe0, "cf-e"}, - {0xf0, "cf-f"}, - } - for _, c := range cases { - var h [32]byte - h[0] = c.topByte - assert.Equal(t, c.want, cfNameForTxHash(h)) - } -} From 9ea3b4c9f97d6465effd5f6238274fd765ee94ba Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 12:51:56 -0400 Subject: [PATCH 17/55] fullhistory: fix golangci-lint failures Clears every golangci-lint finding on the PR's changed code (verified with golangci-lint v2.11.3 via --new-from-rev against the merge base): - lifecycle.Config: rename the stuttering LifecycleConfig -> Config; extract an abortTick helper that centralizes the tick's fatal-unless-cancelled error policy (kills the gocognit finding + the per-stage duplication). - backfill: extract resolveHotSource so backfillSource's hot branch is flat (nestif). - eventstore/hotchunk: annotate the intentional (nil, nil) idempotent-duplicate returns (nilnil), drop a stale //nolint:cyclop, unname MaxCommittedSeq's returns, preallocate the CF slice. - drop unused params (pendingArtifacts cfg, refineWithHotDB cat) and the always-0 openTestDB chunkID; delete the dead smallTxHashIndexCatalog and the unused metastore return of the lifecycle test catalog helper. - mechanical: cancelled -> canceled, integer-range loops, gci import order, line wraps, WaitGroup.Go, unnecessary conversion, embedded-field spacing. - suppress cyclop/funlen/maintidx on the single linear E2E lifecycle scenario. go build + go test -short + golangci-lint all green across ./fullhistory/... --- .../internal/fullhistory/backfill/process.go | 39 ++++++--- .../catalog/catalog_protocol_test.go | 2 +- .../fullhistory/catalog/catalog_sweep.go | 2 +- .../fullhistory/catalog/helpers_test.go | 2 +- .../catalog/keys_roundtrip_test.go | 2 +- .../internal/fullhistory/daemon.go | 5 +- .../internal/fullhistory/daemon_test.go | 6 +- .../internal/fullhistory/e2e_test.go | 9 +- .../fullhistory/geometry/keys_test.go | 8 +- .../internal/fullhistory/helpers_test.go | 9 -- .../internal/fullhistory/hotloop.go | 4 +- .../internal/fullhistory/hotloop_test.go | 8 +- .../fullhistory/lifecycle/eligibility.go | 8 +- .../fullhistory/lifecycle/helpers_test.go | 11 +-- .../fullhistory/lifecycle/hot_fakes_test.go | 3 +- .../fullhistory/lifecycle/lifecycle.go | 86 ++++++++----------- .../lifecycle/lifecycle_helpers_test.go | 12 +-- .../lifecycle/lifecycle_loop_test.go | 4 +- .../fullhistory/lifecycle/lifecycle_test.go | 16 ++-- .../fullhistory/lifecycle/progress.go | 4 +- .../fullhistory/pkg/rocksdb/rocksdb.go | 3 +- .../pkg/stores/eventstore/hot_store.go | 11 ++- .../pkg/stores/hotchunk/hotchunk.go | 5 +- .../pkg/stores/hotchunk/hotchunk_test.go | 15 ++-- .../internal/fullhistory/startup.go | 17 ++-- .../internal/fullhistory/startup_test.go | 8 +- 26 files changed, 150 insertions(+), 149 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go index 2ae3a7b25..e8817549a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go @@ -181,21 +181,13 @@ func backfillSource( // (1) Hot branch: only when a HotProbe is wired and the hot key is "ready". A // "transient" key (mid-op or recovery-demoted) is not a read source. if cfg.HotProbe != nil { - hotState, err := cat.HotState(chunkID) - if err != nil { - return nil, noClose, fmt.Errorf("read hot state chunk %s: %w", chunkID, err) + src, closer, used, herr := resolveHotSource(chunkID, cfg) + if herr != nil { + return nil, noClose, herr // case-4 loss is fatal } - if hotState == geometry.HotReady { - src, closer, used, herr := tryHotSource(chunkID, cfg) - if herr != nil { - return nil, noClose, herr // case-4 loss is fatal - } - if used { - cfg.Logger.Debugf("backfillSource: chunk %s from complete hot tier", chunkID) - return src, closer, nil - } - // Present but incomplete: legitimate staleness — fall through. - cfg.Logger.Debugf("backfillSource: chunk %s hot tier present but incomplete; falling through", chunkID) + if used { + cfg.Logger.Debugf("backfillSource: chunk %s from complete hot tier", chunkID) + return src, closer, nil } } @@ -234,6 +226,23 @@ func backfillSource( return cfg.Backend, noClose, nil } +// resolveHotSource applies the hot branch end to end: it reads the hot key and, +// only when "ready", tries the hot tier. used=true → src/closer are the hot +// source; used=false → no "ready" key or present-but-incomplete (caller falls +// through); err → case-4 loss (fatal). Keeps backfillSource's hot branch flat. +func resolveHotSource( + chunkID chunk.ID, cfg ProcessConfig, +) (ledgerbackend.LedgerStream, func() error, bool, error) { + hotState, err := cfg.Catalog.HotState(chunkID) + if err != nil { + return nil, nil, false, fmt.Errorf("read hot state chunk %s: %w", chunkID, err) + } + if hotState != geometry.HotReady { + return nil, nil, false, nil // "transient"/absent: not a read source + } + return tryHotSource(chunkID, cfg) +} + // tryHotSource handles the hot branch under a "ready" key: used=true when present // AND complete; used=false when present-but-incomplete (staleness, caller falls // through); err only for case-4 LOSS (ErrHotVolumeLost), detected lazily on the open. @@ -259,6 +268,8 @@ func tryHotSource(chunkID chunk.ID, cfg ProcessConfig) (ledgerbackend.LedgerStre if present && maxSeq >= chunkID.LastLedger() { return hot.Source(), hot.Close, true, nil } + // Present but incomplete: legitimate staleness — caller falls through. + cfg.Logger.Debugf("backfillSource: chunk %s hot tier present but incomplete; falling through", chunkID) _ = hot.Close() return nil, nil, false, nil } diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol_test.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol_test.go index 2bb384a34..b83590657 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/require" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) // --------------------------------------------------------------------------- diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_sweep.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_sweep.go index 9eb1825e0..d9dcb0e44 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_sweep.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_sweep.go @@ -1,8 +1,8 @@ package catalog import ( - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/metastore" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/metastore" ) // Key-driven sweeps — the ONLY two deletion bodies in the system, one per key diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/catalog/helpers_test.go index 29906238b..3f34f3291 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/helpers_test.go @@ -13,9 +13,9 @@ import ( supportlog "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/metastore" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" ) func silentLogger() *supportlog.Entry { diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/keys_roundtrip_test.go b/cmd/stellar-rpc/internal/fullhistory/catalog/keys_roundtrip_test.go index 03c16009f..3669aac91 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/keys_roundtrip_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/keys_roundtrip_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/require" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) // --------------------------------------------------------------------------- diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index c90cd91c3..47fefe5a4 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -175,7 +175,8 @@ func runDaemonWith(ctx context.Context, configPath string, opts daemonOptions) e } // --- Assemble the StartConfig and run the supervised run loop. --- - start := startConfig(cfg, cat, logger, backend, networkTip, core, serveReads, metrics, sink, tipBackoff, tipMaxAttempts) + start := startConfig( + cfg, cat, logger, backend, networkTip, core, serveReads, metrics, sink, tipBackoff, tipMaxAttempts) backoff := opts.RestartBackoff if backoff <= 0 { @@ -207,7 +208,7 @@ func startConfig( return StartConfig{ Exec: exec, HotProgressProbe: NewRocksHotRecoveryProbe(cat.Layout().HotChunkPath, logger), - Lifecycle: lifecycle.LifecycleConfig{ + Lifecycle: lifecycle.Config{ ExecConfig: exec, RetentionChunks: deref(cfg.Retention.RetentionChunks), }, diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon_test.go b/cmd/stellar-rpc/internal/fullhistory/daemon_test.go index b263147a9..7d814410d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon_test.go @@ -84,7 +84,7 @@ func TestRunDaemon_LoadValidateWireStartCleanShutdown(t *testing.T) { select { case err := <-errCh: - require.NoError(t, err, "a ctx-cancelled ingestion loop is a clean shutdown") + require.NoError(t, err, "a ctx-canceled ingestion loop is a clean shutdown") case <-time.After(3 * time.Second): t.Fatal("runDaemonWith did not return after ctx cancel") } @@ -195,7 +195,7 @@ func TestRunDaemon_BackfillMaterializesAllColdTypesAndIndex(t *testing.T) { defer cancel() // ServeReads runs after backfill completes, just before the blocking ingestion // loop — so it is the "backfill done" signal. The injected core then blocks until - // the ctx cancel below, and a ctx-cancelled ingestion loop is a clean shutdown. + // the ctx cancel below, and a ctx-canceled ingestion loop is a clean shutdown. servedCh := make(chan struct{}, 1) errCh := make(chan error, 1) go func() { @@ -219,7 +219,7 @@ func TestRunDaemon_BackfillMaterializesAllColdTypesAndIndex(t *testing.T) { cancel() // request a clean shutdown of the parked ingestion loop select { case err := <-errCh: - require.NoError(t, err, "a ctx-cancelled ingestion loop is a clean shutdown") + require.NoError(t, err, "a ctx-canceled ingestion loop is a clean shutdown") case <-time.After(10 * time.Second): t.Fatal("runDaemonWith did not return after ctx cancel") } diff --git a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go index 6d272117c..69292c361 100644 --- a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go @@ -77,7 +77,7 @@ func (c *e2eCore) OpenCore(_ context.Context, resume uint32) (LedgerGetter, func // e2eGetter is the FAKE captive-core ledger getter: a resumable LedgerGetter the // ingestion loop polls by sequence. It returns the frame for the requested seq // when its core has one, and once the poll runs past the synthetic backlog it -// blocks until ctx is cancelled (a live tip stream ends only on shutdown). It +// blocks until ctx is canceled (a live tip stream ends only on shutdown). It // records (into its core) the FIRST seq it was asked for, so the restart step can // assert the daemon re-derived the watermark and resumed with no gap. type e2eGetter struct { @@ -108,6 +108,7 @@ func (s *e2eGetter) GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseM // boundaries and freezes the daemon emits (the rest discarded via NopMetrics). type e2eMetrics struct { observability.NopMetrics + mu sync.Mutex boundaries int freezes int @@ -236,7 +237,7 @@ func hotKeyExists(cat *catalog.Catalog, c chunk.ID) (bool, error) { // hashAt builds a deterministic 32-byte hash from n (for the never-committed miss). func hashAt(n uint64) [32]byte { var h [32]byte - for i := 0; i < 8; i++ { + for i := range 8 { h[i] = byte(n >> (8 * i)) } return h @@ -257,6 +258,8 @@ func hashAt(n uint64) [32]byte { // drive retention far enough to prune chunk 0, confirm a pruned read is not-found. // // Correctness is asserted at every step. +// +//nolint:cyclop,funlen,maintidx // one linear end-to-end scenario asserted step by step func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing.T) { if testing.Short() { t.Skip("e2e ingests a full 10k-ledger chunk; skipped in -short") @@ -348,7 +351,7 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing }, 60*time.Second, 50*time.Millisecond, "the boundary ticks must freeze+fold+discard chunks 0 and 1") require.GreaterOrEqual(t, served.Load(), int32(1), "reads were served") - require.Equal(t, uint32(c0First), core.resumeSeen.Load(), + require.Equal(t, c0First, core.resumeSeen.Load(), "first start resumes captive core at genesis (watermark+1)") // --- Correctness: chunks 0 and 1 per-chunk cold artifacts (ledgers + events) froze. --- diff --git a/cmd/stellar-rpc/internal/fullhistory/geometry/keys_test.go b/cmd/stellar-rpc/internal/fullhistory/geometry/keys_test.go index 93b9a64a4..424ca0dff 100644 --- a/cmd/stellar-rpc/internal/fullhistory/geometry/keys_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/geometry/keys_test.go @@ -62,12 +62,12 @@ func TestKeyToPathBijection(t *testing.T) { func TestParseRejectsMalformed(t *testing.T) { bad := []string{ - "chunk:5350:ledgers", // not 8-digit padded - "chunk:00005350:bogus", // unknown kind - "chunk:00005350", // missing kind + "chunk:5350:ledgers", // not 8-digit padded + "chunk:00005350:bogus", // unknown kind + "chunk:00005350", // missing kind "index:00000005:00005100", // too few segments "index:5:5100:5349", // not padded - "unrelated:key", // wrong family + "unrelated:key", // wrong family } for _, key := range bad { _, _, okChunk := ParseChunkKey(key) diff --git a/cmd/stellar-rpc/internal/fullhistory/helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/helpers_test.go index 44fa9c29e..d93cdf705 100644 --- a/cmd/stellar-rpc/internal/fullhistory/helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/helpers_test.go @@ -66,15 +66,6 @@ func testCatalog(t *testing.T) (*catalog.Catalog, string) { return cat, root } -// smallTxHashIndexCatalog builds a test catalog whose indexes are cpi chunks -// wide, so a "terminal" (full-index) build needs only a few chunks. Returns the -// catalog and the artifact root. -func smallTxHashIndexCatalog(t *testing.T, cpi uint32) (*catalog.Catalog, string) { - t.Helper() - cat, _, root := newTestCatalog(t, cpi) - return cat, root -} - // freezeKinds flips the given per-chunk kinds to "frozen" via the one-write protocol. func freezeKinds(t *testing.T, cat *catalog.Catalog, chunkID chunk.ID, kinds ...geometry.Kind) { t.Helper() diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index 488ad966e..8cd33e54a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -25,7 +25,7 @@ import ( // the last synced batch IS the last-committed ledger, re-derived at startup. Its only // coupling to the lifecycle is the channel: at each boundary it sends the // just-completed chunk id (the two goroutines share no memory). Clean-shutdown vs -// crash is decided at the daemon top level (a ctx-cancelled return is clean). +// crash is decided at the daemon top level (a ctx-canceled return is clean). // LedgerGetter is the indexed-poll source the ingestion loop drives: it returns // one ledger's view, blocking until that ledger is available (the design's @@ -105,7 +105,7 @@ func openHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlo // runIngestionLoop polls core for LCMs by seq into hotDB (one atomic synced // WriteBatch each), and at each chunk boundary hands the frontier forward by // closing the just-filled DB and opening the next. It never returns nil; the -// daemon classifies a ctx-cancelled return as clean shutdown, any other as +// daemon classifies a ctx-canceled return as clean shutdown, any other as // RESTARTABLE (startup re-derives the last-committed ledger, losing nothing). // // HANDOFF FENCE: the DB is CLOSED before the next chunk's hot:chunk key is diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index 211741e6d..5be69db56 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -34,7 +34,7 @@ func ledgerEntry(t *testing.T, seq uint32) ledger.Entry { // fakeLedgerGetter — an injectable LedgerGetter the ingestion loop polls by // sequence (the design's indexed core.GetLedger(ctx, seq)). For seqs it has a // programmed frame it returns those bytes; once the poll runs past the last -// programmed seq it either blocks until ctx is cancelled (a live tip stream that +// programmed seq it either blocks until ctx is canceled (a live tip stream that // only ends on shutdown) or returns endErr (a crashed backend). It records the // FIRST seq it was asked for (the restart resume point) and the GetLedger call // count. @@ -269,12 +269,12 @@ func TestRunIngestionLoop_BoundaryNotifiesCompletedChunk(t *testing.T) { // --------------------------------------------------------------------------- // runIngestionLoop — clean shutdown vs crash (classified at the daemon top -// level: ctx-cancelled return is clean, any other error is restartable). +// level: ctx-canceled return is clean, any other error is restartable). // --------------------------------------------------------------------------- // TestRunIngestionLoop_CtxCancelReturnsCtxErr: a ctx cancellation while the poll // is blocking on the tip makes GetLedger return ctx.Err(); the loop returns that -// (the daemon top level classifies a ctx-cancelled return as a clean shutdown). +// (the daemon top level classifies a ctx-canceled return as a clean shutdown). func TestRunIngestionLoop_CtxCancelReturnsCtxErr(t *testing.T) { cat, _ := testCatalog(t) c := chunk.ID(0) @@ -299,7 +299,7 @@ func TestRunIngestionLoop_CtxCancelReturnsCtxErr(t *testing.T) { select { case err := <-done: require.Error(t, err) - require.ErrorIs(t, err, context.Canceled, "the loop surfaces the ctx-cancelled GetLedger error") + require.ErrorIs(t, err, context.Canceled, "the loop surfaces the ctx-canceled GetLedger error") case <-time.After(10 * time.Second): t.Fatal("ingestion loop did not stop on ctx cancellation") } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go index 240e9974f..737e587ab 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go @@ -16,7 +16,7 @@ import ( // otherwise (live, or frozen awaiting coverage) → leave alone. // discardHotDBForChunk is idempotent, so a crash between freeze and discard // self-heals next tick. -func eligibleDiscardOps(cfg LifecycleConfig, cat *catalog.Catalog, through uint32) ([]func() error, error) { +func eligibleDiscardOps(cfg Config, cat *catalog.Catalog, through uint32) ([]func() error, error) { earliest, _, err := cat.EarliestLedger() if err != nil { return nil, err @@ -38,7 +38,7 @@ func eligibleDiscardOps(cfg LifecycleConfig, cat *catalog.Catalog, through uint3 case gate.Excludes(c): ops = append(ops, func() error { return discardHotDBForChunk(cat, c) }) case last <= through: - pending, perr := pendingArtifacts(c, cfg, cat) + pending, perr := pendingArtifacts(c, cat) if perr != nil { return nil, perr } @@ -60,7 +60,7 @@ func eligibleDiscardOps(cfg LifecycleConfig, cat *catalog.Catalog, through uint3 // be frozen; txhash/.bin is exempt when the window's index already covers the // chunk (after finalization the chunk:c:txhash key is demoted/swept, so // regenerating the .bin would orphan it). -func pendingArtifacts(c chunk.ID, cfg LifecycleConfig, cat *catalog.Catalog) (catalog.ArtifactSet, error) { +func pendingArtifacts(c chunk.ID, cat *catalog.Catalog) (catalog.ArtifactSet, error) { var need catalog.ArtifactSet for _, kind := range []geometry.Kind{geometry.KindLedgers, geometry.KindEvents} { state, err := cat.State(c, kind) @@ -102,7 +102,7 @@ func indexCovers(c chunk.ID, cat *catalog.Catalog) (bool, error) { // batched SweepChunkArtifacts for the chunk family). "Below the floor" is the // gate predicate shared with the discard scan and read path, so prune deletes // exactly what the reader has stopped admitting. -func eligiblePruneOps(cfg LifecycleConfig, cat *catalog.Catalog, through uint32) ([]func() error, error) { +func eligiblePruneOps(cfg Config, cat *catalog.Catalog, through uint32) ([]func() error, error) { earliest, _, err := cat.EarliestLedger() if err != nil { return nil, err diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go index 1607bceab..d0be6d36e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go @@ -47,8 +47,9 @@ func silentLogger() *supportlog.Entry { } // newTestCatalog builds a Catalog over a real metastore on temp dirs with -// cpi-wide tx-hash indexes; returns the catalog, open store, and artifact root. -func newTestCatalog(t *testing.T, cpi uint32) (*catalog.Catalog, *metastore.Store, string) { +// cpi-wide tx-hash indexes; returns the catalog and artifact root (the store is +// closed via t.Cleanup). +func newTestCatalog(t *testing.T, cpi uint32) (*catalog.Catalog, string) { t.Helper() metaDir := t.TempDir() artifactRoot := t.TempDir() @@ -60,14 +61,14 @@ func newTestCatalog(t *testing.T, cpi uint32) (*catalog.Catalog, *metastore.Stor idxLayout, err := geometry.NewTxHashIndexLayout(cpi) require.NoError(t, err) - return catalog.NewCatalog(store, geometry.NewLayout(artifactRoot), idxLayout), store, artifactRoot + return catalog.NewCatalog(store, geometry.NewLayout(artifactRoot), idxLayout), artifactRoot } // testCatalog builds a catalog with the default (wide) tx-hash index, returning it // and the artifact root. func testCatalog(t *testing.T) (*catalog.Catalog, string) { t.Helper() - cat, _, root := newTestCatalog(t, testCPI) + cat, root := newTestCatalog(t, testCPI) return cat, root } @@ -76,7 +77,7 @@ func testCatalog(t *testing.T) (*catalog.Catalog, string) { // catalog and the artifact root. func smallTxHashIndexCatalog(t *testing.T, cpi uint32) (*catalog.Catalog, string) { t.Helper() - cat, _, root := newTestCatalog(t, cpi) + cat, root := newTestCatalog(t, cpi) return cat, root } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/hot_fakes_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/hot_fakes_test.go index a8fb1f332..0b7296aa9 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/hot_fakes_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/hot_fakes_test.go @@ -6,9 +6,10 @@ import ( "sync/atomic" "testing" - "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" "github.com/stretchr/testify/require" + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index 9948b9e6e..6f5e44d4b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -2,6 +2,7 @@ package lifecycle import ( "context" + "fmt" "log" "time" @@ -25,10 +26,10 @@ import ( // below storage — start is RAISED to lowestMaterializedChunk; extending the // bottom is backfill's job, producibility enforced lazily per chunk. -// LifecycleConfig bundles the tick/loop dependencies. It composes the scheduler's +// Config bundles the tick/loop dependencies. It composes the scheduler's // ExecConfig (shared postconditions + worker pool with backfill) plus the // retention knob and an injectable fatal sink. -type LifecycleConfig struct { +type Config struct { backfill.ExecConfig // RetentionChunks bounds the sliding retention floor's width. 0 disables the @@ -42,14 +43,28 @@ type LifecycleConfig struct { // WithLifecycleDefaults returns a copy with ExecConfig and Fatalf defaults // applied. Called once at startup before launching the loop. -func (cfg LifecycleConfig) WithLifecycleDefaults() LifecycleConfig { - cfg.ExecConfig = cfg.ExecConfig.WithDefaults() +func (cfg Config) WithLifecycleDefaults() Config { + cfg.ExecConfig = cfg.WithDefaults() if cfg.Fatalf == nil { cfg.Fatalf = log.Fatalf } return cfg } +// abortTick centralizes the tick's error policy so each stage is one line. +// nil err → false (continue). A non-nil err aborts the tick (returns true); it +// calls Fatalf only when ctx is still live — a canceled ctx is a clean +// shutdown, not a failure. "what" names the failing step. +func (cfg Config) abortTick(ctx context.Context, err error, what string) bool { + if err == nil { + return false + } + if ctx.Err() == nil { + cfg.Fatalf("streaming: lifecycle tick: %s: %v", what, err) + } + return true +} + // lastCompleteChunkAtID maps geometry.LastCompleteChunkAt to a chunk.ID; // ok=false when no complete chunk exists (negative result). func lastCompleteChunkAtID(ledger uint32) (chunk.ID, bool) { @@ -96,10 +111,12 @@ func lowestMaterializedChunk(cat *catalog.Catalog) (chunk.ID, bool, error) { // next tick's work). Plan range is [floor, lastChunk] (start raised to storage); // discard/prune key off through. // -// CLEAN-SHUTDOWN (binding): on an op error with ctx cancelled, return WITHOUT +// CLEAN-SHUTDOWN (binding): on an op error with ctx canceled, return WITHOUT // Fatalf — cancellation is a shutdown, not a failure. Only a genuine failure // (ctx still live) aborts via Fatalf. -func runLifecycle(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog, lastChunk chunk.ID) { +// +//nolint:cyclop // linear 3-stage pipeline; the branch count is uniform abortTick guards, not real complexity +func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChunk chunk.ID) { metrics := observability.MetricsOrNop(cfg.Metrics) logger := cfg.Logger @@ -107,11 +124,7 @@ func runLifecycle(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog through := lastChunk.LastLedger() earliest, _, err := cat.EarliestLedger() - if err != nil { - if ctx.Err() != nil { - return - } - cfg.Fatalf("streaming: lifecycle tick: read earliest ledger: %v", err) + if cfg.abortTick(ctx, err, "read earliest ledger") { return } floor := EffectiveRetentionFloor(through, cfg.RetentionChunks, earliest) @@ -128,11 +141,7 @@ func runLifecycle(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog // — the production-boundary rule (never plan below existing storage). start := ChunkIDOfLedger(floor) low, hasLow, err := lowestMaterializedChunk(cat) - if err != nil { - if ctx.Err() != nil { - return - } - cfg.Fatalf("streaming: lifecycle tick: lowest materialized chunk: %v", err) + if cfg.abortTick(ctx, err, "lowest materialized chunk") { return } if hasLow && int64(low) > start { @@ -148,11 +157,7 @@ func runLifecycle(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog // No complete chunk ⇒ empty range, production skipped, scans below still run. freezeStart := time.Now() durableThrough, derr := LastCommittedLedger(cat, nil) // chunk-granularity, no hot DB read - if derr != nil { - if ctx.Err() != nil { - return - } - cfg.Fatalf("streaming: lifecycle tick: derive durable through: %v", derr) + if cfg.abortTick(ctx, derr, "derive durable through") { return } highestComplete, haveComplete := lastCompleteChunkAtID(durableThrough) @@ -162,14 +167,11 @@ func runLifecycle(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog } if haveComplete && start >= 0 && start <= int64(rangeEnd) { // Plan-and-execute over [start, rangeEnd] via the same entry point backfill - // uses (resolve → executePlan → Freeze metric, recorded internally). - if eerr := backfill.RunBackfill(ctx, cfg.ExecConfig, chunk.ID(start), rangeEnd); eerr != nil { //nolint:gosec // start >= 0 - // CLEAN-SHUTDOWN: a cancelled ctx makes RunBackfill return ctx.Err() — - // a shutdown, not an op failure. Return before any Fatalf. - if ctx.Err() != nil { - return - } - cfg.Fatalf("streaming: lifecycle tick: run backfill [%d,%s]: %v", start, rangeEnd, eerr) + // uses (resolve → executePlan → Freeze metric, recorded internally). A + // canceled ctx makes RunBackfill return ctx.Err(), which abortTick treats + // as a clean shutdown (no Fatalf). + eerr := backfill.RunBackfill(ctx, cfg.ExecConfig, chunk.ID(start), rangeEnd) //nolint:gosec // start >= 0 + if cfg.abortTick(ctx, eerr, fmt.Sprintf("run backfill [%d,%s]", start, rangeEnd)) { return } } else { @@ -181,19 +183,11 @@ func runLifecycle(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog // Stage 2 — discard scan. discardStart := time.Now() discardOps, err := eligibleDiscardOps(cfg, cat, through) - if err != nil { - if ctx.Err() != nil { - return - } - cfg.Fatalf("streaming: lifecycle tick: eligible discard ops: %v", err) + if cfg.abortTick(ctx, err, "eligible discard ops") { return } for _, op := range discardOps { - if oerr := op(); oerr != nil { - if ctx.Err() != nil { - return - } - cfg.Fatalf("streaming: lifecycle tick: discard op: %v", oerr) + if cfg.abortTick(ctx, op(), "discard op") { return } } @@ -210,19 +204,11 @@ func runLifecycle(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog // Stage 3 — prune scan. pruneStart := time.Now() pruneOps, err := eligiblePruneOps(cfg, cat, through) - if err != nil { - if ctx.Err() != nil { - return - } - cfg.Fatalf("streaming: lifecycle tick: eligible prune ops: %v", err) + if cfg.abortTick(ctx, err, "eligible prune ops") { return } for _, op := range pruneOps { - if oerr := op(); oerr != nil { - if ctx.Err() != nil { - return - } - cfg.Fatalf("streaming: lifecycle tick: prune op: %v", oerr) + if cfg.abortTick(ctx, op(), "prune op") { return } } @@ -242,7 +228,7 @@ const LifecycleQueueDepth = 8 // (one tick over [floor, lastChunk] subsumes the rest) and runs one tick. It // selects on both ctx.Done() and the channel, so it never blocks or fatals on // shutdown. -func Loop(ctx context.Context, cfg LifecycleConfig, cat *catalog.Catalog, ch <-chan chunk.ID) { +func Loop(ctx context.Context, cfg Config, cat *catalog.Catalog, ch <-chan chunk.ID) { for { select { case <-ctx.Done(): diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go index a230d849b..81bd06268 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go @@ -104,13 +104,13 @@ func ingestFullHotChunk(t *testing.T, cat *catalog.Catalog, c chunk.ID) { require.NoError(t, db.Close()) // release the write handle (boundary handoff) } -// lifecycleTestConfig wires a LifecycleConfig over the real production primitives +// lifecycleTestConfig wires a Config over the real production primitives // (a real RocksHotProbe over the catalog's hot layout) plus a fatal recorder so a // tick abort is observable instead of killing the test process. -func lifecycleTestConfig(t *testing.T, cat *catalog.Catalog, retentionChunks uint32) (LifecycleConfig, *fatalRecorder) { +func lifecycleTestConfig(t *testing.T, cat *catalog.Catalog, retentionChunks uint32) (Config, *fatalRecorder) { t.Helper() rec := &fatalRecorder{} - cfg := LifecycleConfig{ + cfg := Config{ ExecConfig: backfill.ExecConfig{ Catalog: cat, Logger: silentLogger(), @@ -144,7 +144,7 @@ func (r *fatalRecorder) fired() bool { return r.count.Load() > 0 } // hands over at a boundary) and passes it as lastChunk. A negative result (young // network, no complete chunk) is passed as chunk 0 — the resolve range guard // then makes the plan empty, matching the design's young-network no-op. -func runTickForCatalog(ctx context.Context, t *testing.T, cfg LifecycleConfig, cat *catalog.Catalog) { +func runTickForCatalog(ctx context.Context, t *testing.T, cfg Config, cat *catalog.Catalog) { t.Helper() through, err := deriveCompleteThrough(cat) require.NoError(t, err) @@ -167,7 +167,7 @@ func makeReadyHotDirNoData(t *testing.T, cat *catalog.Catalog, c chunk.ID) { // assertQuiescent re-runs the tick's three derivations against the SAME through // snapshot and asserts none schedule work — the quiescence postcondition. -func assertQuiescent(t *testing.T, cfg LifecycleConfig, cat *catalog.Catalog, through uint32) { +func assertQuiescent(t *testing.T, cfg Config, cat *catalog.Catalog, through uint32) { t.Helper() earliest, _, err := cat.EarliestLedger() require.NoError(t, err) @@ -182,7 +182,7 @@ func assertQuiescent(t *testing.T, cfg LifecycleConfig, cat *catalog.Catalog, th // At quiescence resolve finds an empty plan, so RunBackfill (resolve + // executePlan) is a no-op that returns nil — even with no Backend wired, // since an empty plan never reaches backfillSource. - perr := backfill.RunBackfill(context.Background(), cfg.ExecConfig, chunk.ID(start), rangeEnd) //nolint:gosec // start >= 0 + perr := backfill.RunBackfill(context.Background(), cfg.ExecConfig, chunk.ID(start), rangeEnd) assert.NoError(t, perr, "re-running backfill schedules no work at quiescence") } dops, err := eligibleDiscardOps(cfg, cat, through) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go index b26a770fb..e71bb6e67 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go @@ -98,7 +98,7 @@ func TestLifecycleLoop_DrainsToMostRecent(t *testing.T) { } } -// TestLifecycleLoop_ReturnsImmediatelyOnAlreadyCancelledCtx: an already-cancelled +// TestLifecycleLoop_ReturnsImmediatelyOnAlreadyCancelledCtx: an already-canceled // ctx makes the loop return without running any tick (never blocks on the // channel forever). func TestLifecycleLoop_ReturnsImmediatelyOnAlreadyCancelledCtx(t *testing.T) { @@ -117,6 +117,6 @@ func TestLifecycleLoop_ReturnsImmediatelyOnAlreadyCancelledCtx(t *testing.T) { select { case <-done: case <-time.After(5 * time.Second): - t.Fatal("the loop blocked instead of observing the cancelled ctx") + t.Fatal("the loop blocked instead of observing the canceled ctx") } } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go index d715aa327..0e6a6ebfc 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go @@ -27,7 +27,9 @@ import ( // // Then re-running the tick is a no-op (quiescence). func TestRunLifecycleTick_BoundaryFreezesFoldsDiscards(t *testing.T) { - t.Parallel() // full-chunk ingest; isolated TempDir/catalog — overlap with the other heavy tests to fit the gate's go-test timeout + // full-chunk ingest on an isolated TempDir/catalog; overlaps the other heavy + // tests to fit the gate's go-test timeout. + t.Parallel() cat, _ := smallTxHashIndexCatalog(t, 1) // window w == chunk w; a one-chunk window finalizes immediately cfg, rec := lifecycleTestConfig(t, cat, 0) @@ -186,7 +188,7 @@ func TestRunLifecycleTick_PrunesTransientIndexDebris(t *testing.T) { } // --------------------------------------------------------------------------- -// CLEAN SHUTDOWN: a ctx cancelled mid-tick returns WITHOUT fatal. +// CLEAN SHUTDOWN: a ctx canceled mid-tick returns WITHOUT fatal. // --------------------------------------------------------------------------- // genuineFailureTickSetup wires a catalog whose chunk-0 build is GENUINELY @@ -196,7 +198,7 @@ func TestRunLifecycleTick_PrunesTransientIndexDebris(t *testing.T) { // lifecycleTestConfig default), backfillSource has no source for chunk 0 and // RunBackfill fails with a non-cancellation error. MaxRetries defaults to 0, so it // fails fast. Returns the config and the fatal recorder. -func genuineFailureTickSetup(t *testing.T) (LifecycleConfig, *fatalRecorder, *catalog.Catalog) { +func genuineFailureTickSetup(t *testing.T) (Config, *fatalRecorder, *catalog.Catalog) { t.Helper() cat, _ := smallTxHashIndexCatalog(t, 1) cfg, rec := lifecycleTestConfig(t, cat, 0) // HotProbe wired, no Backend @@ -206,17 +208,17 @@ func genuineFailureTickSetup(t *testing.T) (LifecycleConfig, *fatalRecorder, *ca } // TestRunLifecycleTick_CleanShutdownNoFatal: when RunBackfill returns an error AND -// ctx was cancelled, the tick must NOT call Fatalf — cancellation is a shutdown, +// ctx was canceled, the tick must NOT call Fatalf — cancellation is a shutdown, // never an op failure. The chunk-0 build is genuinely unproducible (no source), but -// the cancelled ctx takes precedence per the clean-shutdown policy. +// the canceled ctx takes precedence per the clean-shutdown policy. func TestRunLifecycleTick_CleanShutdownNoFatal(t *testing.T) { cfg, rec, cat := genuineFailureTickSetup(t) ctx, cancel := context.WithCancel(context.Background()) cancel() // shutdown requested before the tick runs - runLifecycle(ctx, cfg, cat, 0) // lastChunk 0: plan range [0,0], build fails under a cancelled ctx - require.False(t, rec.fired(), "a cancelled ctx is a clean shutdown, NOT an op failure — no Fatalf") + runLifecycle(ctx, cfg, cat, 0) // lastChunk 0: plan range [0,0], build fails under a canceled ctx + require.False(t, rec.fired(), "a canceled ctx is a clean shutdown, NOT an op failure — no Fatalf") } // TestRunLifecycleTick_GenuineFailureAborts: when a plan op fails for a real diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go index 22b3bd6fc..abe538c0f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go @@ -54,7 +54,7 @@ func LastCommittedLedger(cat *catalog.Catalog, probe backfill.HotProbe) (uint32, } else { // One refinement read of the highest ready hot DB; loss detected lazily // on this open (no eager scan over every ready key). - refined, rerr := refineWithHotDB(cat, probe, hot) + refined, rerr := refineWithHotDB(probe, hot) if rerr != nil { return 0, rerr } @@ -79,7 +79,7 @@ func LastCommittedLedger(cat *catalog.Catalog, probe backfill.HotProbe) (uint32, // its MaxCommittedSeq, or CompleteThrough(live-1) on an empty DB. A "ready" key // whose dir/DB is gone surfaces as backfill.ErrHotVolumeLost (lazy loss // detection). -func refineWithHotDB(cat *catalog.Catalog, probe backfill.HotProbe, live int64) (uint32, error) { +func refineWithHotDB(probe backfill.HotProbe, live int64) (uint32, error) { id := chunk.ID(live) //nolint:gosec // live > cold >= -1, so live >= 0 hot, ok, openErr := probe.OpenHotChunk(id) if openErr != nil { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go index 5465dc0b9..b6af40547 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go @@ -434,7 +434,8 @@ func (s *Store) Close() error { // would reject it); only a writable store flushes its memtable on close. if !s.cfg.ReadOnly { if err := s.doFlush(); err != nil { - s.cfg.Logger.WithError(err).Warnf("rocksdb: graceful close Flush failed at %s; next Open will replay WAL", s.cfg.Path) + s.cfg.Logger.WithError(err).Warnf( + "rocksdb: graceful close Flush failed at %s; next Open will replay WAL", s.cfg.Path) } } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index 9327bb4fe..e771f1217 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -544,7 +544,8 @@ func (h *HotStore) IngestLedgerToBatchCommit(ledgerSeq uint32, payloads []events return nil, err } if prep == nil { - return nil, nil // idempotent duplicate no-op + //nolint:nilnil // (nil, nil) is the idempotent-duplicate signal; the caller runs the hook only when non-nil + return nil, nil } if cerr := h.chunkStore.Batch(func(b *rocksdb.BatchWriter) error { return prep.queue(b) @@ -559,7 +560,9 @@ func (h *HotStore) IngestLedgerToBatchCommit(ledgerSeq uint32, payloads []events // runs AFTER b commits (decision (a)). Returns (nil, nil) for an idempotent // duplicate. All validation + term derivation happen up front, so a rejected // ledger leaves b untouched. -func (h *HotStore) IngestLedgerToBatch(b *rocksdb.BatchWriter, ledgerSeq uint32, payloads []events.Payload) (func(), error) { +func (h *HotStore) IngestLedgerToBatch( + b *rocksdb.BatchWriter, ledgerSeq uint32, payloads []events.Payload, +) (func(), error) { if h.chunkStore.IsClosed() { return nil, ErrClosed } @@ -568,6 +571,7 @@ func (h *HotStore) IngestLedgerToBatch(b *rocksdb.BatchWriter, ledgerSeq uint32, return nil, err } if prep == nil { + //nolint:nilnil // (nil, nil) is the idempotent-duplicate signal; the caller runs the hook only when non-nil return nil, nil } if qerr := prep.queue(b); qerr != nil { @@ -609,8 +613,6 @@ func (p *preparedLedger) queue(b *rocksdb.BatchWriter) error { // ready to queue + apply, or (nil, nil) for an idempotent duplicate. It does NO // disk write and NO mirror mutation, so it is safe to call before touching a // shared batch. -// -//nolint:cyclop // sequential pipeline: validate -> derive terms -> marshal -> build apply hook func (h *HotStore) prepareLedger(ledgerSeq uint32, payloads []events.Payload) (*preparedLedger, error) { // Validate BEFORE marshaling: failing after a shared batch holds this // ledger's rows would orphan them. @@ -623,6 +625,7 @@ func (h *HotStore) prepareLedger(ledgerSeq uint32, payloads []events.Payload) (* if ledgerSeq < expected { // Already ingested: idempotent no-op (a restarted ingester may // re-deliver). Re-delivered events are not re-verified. + //nolint:nilnil // (nil, nil) is the idempotent-duplicate signal; callers branch on a nil *preparedLedger return nil, nil } if ledgerSeq > expected { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index 4933b9e59..1607c2586 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -43,7 +43,8 @@ type DB struct { // columnFamilies is the full CF list for the shared per-chunk DB (ledger + 3 // events + 1 txhash). Names are already non-colliding across the facades. func columnFamilies() []string { - cfs := []string{ledger.LedgersCF} + cfs := make([]string, 0, 1+len(eventstore.CFNames())+len(txhash.CFNames())) + cfs = append(cfs, ledger.LedgersCF) cfs = append(cfs, eventstore.CFNames()...) cfs = append(cfs, txhash.CFNames()...) return cfs @@ -124,7 +125,7 @@ func (d *DB) Close() error { return d.store.Close() } // MaxCommittedSeq returns the single authoritative per-chunk last-committed ledger: the // highest seq durably committed, from the ledgers CF's last key. Under decision // (a) this one value pins EVERY CF's frontier. ok=false on an empty DB. -func (d *DB) MaxCommittedSeq() (seq uint32, ok bool, err error) { +func (d *DB) MaxCommittedSeq() (uint32, bool, error) { return d.ledger.LastSeq() } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go index c82659a95..ce1177f2a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go @@ -30,9 +30,10 @@ func silentLogger() *supportlog.Entry { return log } -func openTestDB(t *testing.T, chunkID chunk.ID) *DB { +// openTestDB opens a fresh hot DB bound to chunk 0 (every test uses chunk 0). +func openTestDB(t *testing.T) *DB { t.Helper() - db, err := Open(t.TempDir(), chunkID, silentLogger()) + db, err := Open(t.TempDir(), chunk.ID(0), silentLogger()) require.NoError(t, err) t.Cleanup(func() { _ = db.Close() }) return db @@ -48,7 +49,7 @@ func TestOpen_ValidatesInputs(t *testing.T) { func TestColumnFamilies_UnionIsNonColliding(t *testing.T) { cfs := columnFamilies() - // 1 ledger CF + 3 events CFs + 16 txhash CFs = 20. + // 1 ledger CF + 3 events CFs + 1 txhash CF = 5. require.Len(t, cfs, 1+len(eventstore.CFNames())+len(txhash.CFNames())) seen := map[string]bool{} for _, cf := range cfs { @@ -71,7 +72,7 @@ func TestColumnFamilies_UnionIsNonColliding(t *testing.T) { func TestIngestLedger_AllCFsAdvanceTogether(t *testing.T) { chunkID := chunk.ID(0) first := chunkID.FirstLedger() - db := openTestDB(t, chunkID) + db := openTestDB(t) // Empty DB: no watermark. _, ok, err := db.MaxCommittedSeq() @@ -122,7 +123,7 @@ func TestIngestLedger_AllCFsAdvanceTogether(t *testing.T) { // advance. func TestIngestLedger_RejectedLedgerPersistsNothingAcrossAnyCF(t *testing.T) { chunkID := chunk.ID(0) - db := openTestDB(t, chunkID) + db := openTestDB(t) // A ledger seq ABOVE the chunk's range: the events facade rejects it // (ErrLedgerOutOfRange) from inside the batch callback, aborting the write. @@ -223,7 +224,7 @@ func TestIngestLedger_MidBatchCommitFailurePersistsNothing(t *testing.T) { // IngestLedger path relies on (intra-store cross-CF atomicity of one // WriteBatch). func TestSharedBatch_DirectRocksAbortAcrossCFs(t *testing.T) { - db := openTestDB(t, chunk.ID(0)) + db := openTestDB(t) var hash [32]byte hash[0] = 0xa0 @@ -256,7 +257,7 @@ func storeOf(db *DB) *rocksdb.Store { return db.store } func TestIngestLedger_WritesEveryHotType(t *testing.T) { chunkID := chunk.ID(0) first := chunkID.FirstLedger() - db := openTestDB(t, chunkID) + db := openTestDB(t) raw, hash, term := lcmWithEvent(t, first) counts, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index df00187e9..a375a9aed 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -20,7 +20,7 @@ import ( // tip, then (2) SERVE + INGEST — open the resume chunk's hot DB, start captive // core (injected), launch the lifecycle goroutine on a doorbell, begin serving // reads (injected), and run the live ingestion loop. Returns nil only on a clean -// shutdown (ctx cancelled mid-run, or the ingestion loop's clean stop); any other +// shutdown (ctx canceled mid-run, or the ingestion loop's clean stop); any other // return is a restartable error the supervisor surfaces (ErrFirstStartNoTip on a // first start with no reachable backend; a backfill/ingest failure; ErrHotVolumeLost). func run(ctx context.Context, cfg StartConfig) error { @@ -57,7 +57,8 @@ func run(ctx context.Context, cfg StartConfig) error { } metrics := observability.MetricsOrNop(cfg.Exec.Metrics) - metrics.LastCommitted(lastCommitted, lifecycle.EffectiveRetentionFloor(lastCommitted, cfg.Lifecycle.RetentionChunks, earliest)) + metrics.LastCommitted(lastCommitted, + lifecycle.EffectiveRetentionFloor(lastCommitted, cfg.Lifecycle.RetentionChunks, earliest)) logger.WithField("last_committed", lastCommitted). WithField("earliest", earliest). WithField("pinned", pinned). @@ -112,7 +113,7 @@ func run(ctx context.Context, cfg StartConfig) error { } // The lifecycle goroutine is tied to a PER-ITERATION child ctx (not the daemon - // ctx) and is cancelled + JOINED before run returns for ANY reason — restoring + // ctx) and is canceled + JOINED before run returns for ANY reason — restoring // the single-lifecycle-goroutine invariant across supervisor restarts (a // daemon-ctx-tied loop would survive a restartable return and run a tick // concurrently with the next iteration's lifecycle + ingestion: two backfill @@ -120,11 +121,9 @@ func run(ctx context.Context, cfg StartConfig) error { // the join cannot block past the current step. lifecycleCtx, cancelLifecycle := context.WithCancel(ctx) var lifecycleWG sync.WaitGroup - lifecycleWG.Add(1) - go func() { - defer lifecycleWG.Done() + lifecycleWG.Go(func() { lifecycle.Loop(lifecycleCtx, cfg.Lifecycle, cat, lifecycleCh) - }() + }) // The two return paths registered after this defer (the ingestion-loop return // and the ServeReads error path) have no live sender on lifecycleCh — ingestion // is a same-goroutine call whose inline notify has stopped, and the serve path @@ -142,7 +141,7 @@ func run(ctx context.Context, cfg StartConfig) error { // The ingestion loop owns hotDB for the rest of its life (closes it on any exit, // reopens at each boundary). Returns the GetLedger/boundary error; the daemon top - // level classifies a ctx-cancelled return as a clean shutdown. + // level classifies a ctx-canceled return as a clean shutdown. return runIngestionLoop(ctx, core, hotDB, cat, lifecycleCh, logger, metrics, cfg.Exec.Process.Sink) } @@ -273,7 +272,7 @@ type StartConfig struct { // Lifecycle drives the lifecycle goroutine. Its embedded ExecConfig is the SAME // wiring as Exec (one catalog, one pool); RetentionChunks is the backfill floor's // width too (0 ⇒ the earliest-ledger floor only). - Lifecycle lifecycle.LifecycleConfig + Lifecycle lifecycle.Config // NetworkTip samples the bulk backend's tip during backfill. Required. NetworkTip NetworkTipBackend diff --git a/cmd/stellar-rpc/internal/fullhistory/startup_test.go b/cmd/stellar-rpc/internal/fullhistory/startup_test.go index 684f0e400..4d9142c48 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup_test.go @@ -95,7 +95,7 @@ func startTestConfig( } cfg := StartConfig{ Exec: exec, - Lifecycle: lifecycle.LifecycleConfig{ + Lifecycle: lifecycle.Config{ ExecConfig: exec, RetentionChunks: 0, // A tick op failure should fail the test loudly, not kill the process; the @@ -134,7 +134,7 @@ func (c *fakeCore) OpenCore(_ context.Context, resumeLedger uint32) (LedgerGette } getter := c.getter if getter == nil { - // Default: a live getter that blocks until ctx is cancelled (the daemon's + // Default: a live getter that blocks until ctx is canceled (the daemon's // steady state). Tests that need a finite poll set c.getter. getter = &fakeLedgerGetter{frames: map[uint32][]byte{}, blockOnCtx: true} } @@ -356,7 +356,7 @@ func TestBackfill_LaggingBulkTipFoldsLastCommittedChunk(t *testing.T) { // A young-network first start does no backfill, opens the resume hot DB, starts // the (blocking) fake core, serves reads, and runs the ingestion loop — which -// surfaces the ctx-cancelled GetLedger error on a clean shutdown (the daemon top +// surfaces the ctx-canceled GetLedger error on a clean shutdown (the daemon top // level classifies it as clean). The resume ledger is genesis (watermark+1). func TestRun_FirstStartServeIngestCleanShutdown(t *testing.T) { cat, _ := testCatalog(t) @@ -379,7 +379,7 @@ func TestRun_FirstStartServeIngestCleanShutdown(t *testing.T) { select { case err := <-errCh: - require.ErrorIs(t, err, context.Canceled, "clean shutdown surfaces the ctx-cancelled error") + require.ErrorIs(t, err, context.Canceled, "clean shutdown surfaces the ctx-canceled error") case <-time.After(3 * time.Second): t.Fatal("run did not return after ctx cancel") } From a5bf892d7fdcb6b0790fc727df590743bad681f5 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 12:59:54 -0400 Subject: [PATCH 18/55] fullhistory: drop redundant "streaming:" prefix from PR-introduced fatal messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review comment: the package/component prefix is noise on fatal messages — the wrapped error and the fatal's own call site already identify where it came from. Scoped to the two Fatalf messages this PR introduced (hotloop lifecycle-backpressure fatal, lifecycle-tick fatal); pre-existing "streaming:" errors and the log messages are left as-is. --- cmd/stellar-rpc/internal/fullhistory/hotloop.go | 2 +- cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index 8cd33e54a..a87e89dcd 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -131,7 +131,7 @@ func runIngestionLoop( select { case lifecycleCh <- complete: default: - logger.Fatalf("streaming: lifecycle fell %d boundaries behind ingestion; investigate", + logger.Fatalf("lifecycle fell %d boundaries behind ingestion; investigate", lifecycle.LifecycleQueueDepth) } } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index 6f5e44d4b..99075a328 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -60,7 +60,7 @@ func (cfg Config) abortTick(ctx context.Context, err error, what string) bool { return false } if ctx.Err() == nil { - cfg.Fatalf("streaming: lifecycle tick: %s: %v", what, err) + cfg.Fatalf("lifecycle tick: %s: %v", what, err) } return true } From 0c993a846b42a6a1120686885ae5554b4071dd39 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 13:08:20 -0400 Subject: [PATCH 19/55] fullhistory: make hot DB a required HotService dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review comment: drop the dead s.db == nil guard and the errNilHotDB sentinel from HotService.Ingest. db is a required dependency — both call sites pass it straight from openHotDBForChunk, which returns a non-nil DB or an error, so nil never reaches Ingest on any wired path. The guard was an unreachable branch that read as if nil-db were a supported runtime state. A wiring bug now surfaces as a nil-deref panic with a stack instead of a swallowed error. --- .../internal/fullhistory/ingest/service.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go index d2c80820b..a1eb0ed8c 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go @@ -31,24 +31,19 @@ type HotService struct { } // NewHotService builds a HotService that writes ledgers, txhash, and events into -// the shared per-chunk DB. A nil sink defaults to NopSink. +// the shared per-chunk DB. db is REQUIRED (the hot DB is the sole copy of a +// chunk's un-frozen ledgers) — the caller opens it via openHotDBForChunk, which +// returns a non-nil DB or an error, so it is never nil on any wired path. A nil +// sink defaults to NopSink. func NewHotService(db *hotchunk.DB, sink MetricSink) *HotService { return &HotService{db: db, sink: orNop(sink)} } -var errNilHotDB = errors.New("ingest: nil hot DB") - // Ingest commits lcm to the shared hot DB in one atomic synced WriteBatch // (decision (a)). HotLedgerTotal is emitted regardless of success; on success, // one HotIngest per hot data type reports its item count. func (s *HotService) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { start := time.Now() - if s.db == nil { - d := time.Since(start) - s.emit(hotchunk.LedgerCounts{}, d, errNilHotDB) - s.sink.HotLedgerTotal(d) - return errNilHotDB - } counts, err := s.db.IngestLedger(seq, lcm) d := time.Since(start) s.emit(counts, d, err) From cb991385364d686f0985870f23336a13b4e9a521 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 13:28:51 -0400 Subject: [PATCH 20/55] fullhistory: /simplify cleanups (reuse + dead-param + boundary dedup) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Low-risk quality cleanups surfaced by a parallel /simplify pass; no behavior change: - drop the parentDir wrapper for filepath.Dir; hoist parent once in openHotDBForChunk (matches discard.go's inline filepath.Dir usage). - columnFamilies: slices.Concat instead of make+3×append with manual capacity. - observability.Metrics.ChunkBoundary: drop the closedChunk arg every impl ignored (the id is logged at the call site); update the 5 impls + caller. - hotloop boundary: compute chunk.IDFromLedger(seq) once instead of twice. Skipped as too broad / immaterial for a simplify pass (flagged for follow-up): single-snapshot catalog-scan threading through the lifecycle tick, 3-site hot-DB open/loss consolidation, dead standalone eventstore facades, and moving CompleteThrough/ChunkIDOfLedger into geometry. go build + vet + test -short + golangci-lint all green. --- .../fullhistory/backfill/recorder_test.go | 2 +- cmd/stellar-rpc/internal/fullhistory/e2e_test.go | 2 +- .../internal/fullhistory/helpers_test.go | 2 +- cmd/stellar-rpc/internal/fullhistory/hotloop.go | 15 ++++++--------- .../fullhistory/observability/observability.go | 9 +++++---- .../fullhistory/pkg/stores/hotchunk/hotchunk.go | 7 ++----- 6 files changed, 16 insertions(+), 21 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go b/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go index 8b168cf0e..bb849fbfd 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go @@ -42,7 +42,7 @@ func (r *recordingMetrics) Prune(count int, d time.Duration) { } func (*recordingMetrics) LastCommitted(uint32, uint32) {} -func (*recordingMetrics) ChunkBoundary(uint32) {} +func (*recordingMetrics) ChunkBoundary() {} func (*recordingMetrics) BackfillPass(time.Duration) {} func (*recordingMetrics) LiveHotChunks(int) {} func (*recordingMetrics) Discard(int, time.Duration) {} diff --git a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go index 69292c361..b1d6a10c6 100644 --- a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go @@ -114,7 +114,7 @@ type e2eMetrics struct { freezes int } -func (m *e2eMetrics) ChunkBoundary(uint32) { +func (m *e2eMetrics) ChunkBoundary() { m.mu.Lock() defer m.mu.Unlock() m.boundaries++ diff --git a/cmd/stellar-rpc/internal/fullhistory/helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/helpers_test.go index d93cdf705..3627a9e6a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/helpers_test.go @@ -105,7 +105,7 @@ func (r *recordingMetrics) BackfillPass(time.Duration) { r.backfillPasses++ } -func (*recordingMetrics) ChunkBoundary(uint32) {} +func (*recordingMetrics) ChunkBoundary() {} func (*recordingMetrics) Freeze(time.Duration) {} func (*recordingMetrics) Rebuild(time.Duration) {} func (*recordingMetrics) Prune(int, time.Duration) {} diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index a87e89dcd..6daee752d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -91,9 +91,10 @@ func openHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlo _ = db.Close() return nil, fmt.Errorf("fsync hot dir %s: %w", dir, syncErr) } - if syncErr := geometry.FsyncDir(parentDir(dir)); syncErr != nil { + parent := filepath.Dir(dir) + if syncErr := geometry.FsyncDir(parent); syncErr != nil { _ = db.Close() - return nil, fmt.Errorf("fsync hot parent dir %s: %w", parentDir(dir), syncErr) + return nil, fmt.Errorf("fsync hot parent dir %s: %w", parent, syncErr) } if flipErr := cat.FlipHotReady(chunkID); flipErr != nil { _ = db.Close() @@ -174,8 +175,8 @@ func runIngestionLoop( } // Chunk boundary: this seq is the chunk's last ledger. - if seq == chunk.IDFromLedger(seq).LastLedger() { - closed := chunk.IDFromLedger(seq) + closed := chunk.IDFromLedger(seq) + if seq == closed.LastLedger() { next := closed + 1 // Handoff fence: close the write handle BEFORE the next chunk's key is // created (that key is what makes THIS chunk complete to a tick, which @@ -197,7 +198,7 @@ func runIngestionLoop( notify(closed) // Boundary observability (the woken tick reports the freeze/discard/prune). - metrics.ChunkBoundary(uint32(closed)) + metrics.ChunkBoundary() logger.WithField("closed_chunk", closed.String()). WithField("next_chunk", next.String()). WithField("last_ledger", seq). @@ -218,7 +219,3 @@ func nextIngestLedger(db *hotchunk.DB) (uint32, error) { } return maxSeq + 1, nil } - -// parentDir is the dirent the hot-tier create barrier fsyncs so the chunk dir's -// creation is itself durable. -func parentDir(dir string) string { return filepath.Dir(dir) } diff --git a/cmd/stellar-rpc/internal/fullhistory/observability/observability.go b/cmd/stellar-rpc/internal/fullhistory/observability/observability.go index 186a463be..dc454f5f2 100644 --- a/cmd/stellar-rpc/internal/fullhistory/observability/observability.go +++ b/cmd/stellar-rpc/internal/fullhistory/observability/observability.go @@ -14,8 +14,9 @@ type Metrics interface { // retention floor (the two advance together each backfill pass / lifecycle tick). LastCommitted(lastCommitted, retentionFloor uint32) - // ChunkBoundary counts one ingestion chunk-boundary handoff (closedChunk = just-filled chunk id). - ChunkBoundary(closedChunk uint32) + // ChunkBoundary counts one ingestion chunk-boundary handoff. The closed chunk + // id is logged at the call site; this metric is a plain counter. + ChunkBoundary() // LiveHotChunks sets the count of hot-chunk DBs currently on disk (the // hot:chunk key count). Reported by every lifecycle tick after the discard @@ -38,7 +39,7 @@ type Metrics interface { type NopMetrics struct{} func (NopMetrics) LastCommitted(uint32, uint32) {} -func (NopMetrics) ChunkBoundary(uint32) {} +func (NopMetrics) ChunkBoundary() {} func (NopMetrics) LiveHotChunks(int) {} func (NopMetrics) BackfillPass(time.Duration) {} func (NopMetrics) Freeze(time.Duration) {} @@ -129,7 +130,7 @@ func (m *PrometheusMetrics) LastCommitted(lastCommitted, retentionFloor uint32) m.retentionFloor.Set(float64(retentionFloor)) } -func (m *PrometheusMetrics) ChunkBoundary(uint32) { m.chunkBoundaries.Inc() } +func (m *PrometheusMetrics) ChunkBoundary() { m.chunkBoundaries.Inc() } func (m *PrometheusMetrics) LiveHotChunks(count int) { m.liveHotChunks.Set(float64(count)) } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index 1607c2586..739354402 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -10,6 +10,7 @@ package hotchunk import ( "fmt" + "slices" sdkingest "github.com/stellar/go-stellar-sdk/ingest" supportlog "github.com/stellar/go-stellar-sdk/support/log" @@ -43,11 +44,7 @@ type DB struct { // columnFamilies is the full CF list for the shared per-chunk DB (ledger + 3 // events + 1 txhash). Names are already non-colliding across the facades. func columnFamilies() []string { - cfs := make([]string, 0, 1+len(eventstore.CFNames())+len(txhash.CFNames())) - cfs = append(cfs, ledger.LedgersCF) - cfs = append(cfs, eventstore.CFNames()...) - cfs = append(cfs, txhash.CFNames()...) - return cfs + return slices.Concat([]string{ledger.LedgersCF}, eventstore.CFNames(), txhash.CFNames()) } // config builds the shared store's rocksdb.Config: events' per-CF options (ZSTD From 4c164060cf01db23d8c97ec1cd7dfba157a3901f Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 16:48:00 -0400 Subject: [PATCH 21/55] stores: drop standalone hot-store open paths; wrap-only over shared per-chunk DB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three hot stores (events/ledger/txhash) always wrap the shared per-chunk multi-CF DB via NewWithStore in production; hotchunk.DB owns and closes that DB once. Remove the dead standalone OpenHotStore/NewHotStore constructors, the ownsStore flag, and the per-store Close, and narrow eventstore.Reader to drop the now-unused Close() — nothing consumes it polymorphically (Query is read-only; lifecycle closes hotchunk.DB at the concrete level). --- .../pkg/stores/eventstore/hot_store.go | 94 ++++--------------- .../pkg/stores/eventstore/hot_store_test.go | 65 ++++++++----- .../pkg/stores/eventstore/hot_warmup_test.go | 85 +++++++---------- .../pkg/stores/eventstore/reader.go | 6 -- .../pkg/stores/ledger/hot_store.go | 71 ++------------ .../pkg/stores/ledger/hot_store_test.go | 63 +++++-------- .../pkg/stores/txhash/hot_store.go | 43 +-------- .../pkg/stores/txhash/hot_store_test.go | 64 +++++-------- 8 files changed, 146 insertions(+), 345 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index e771f1217..b2471ed7a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -13,8 +13,6 @@ import ( "github.com/RoaringBitmap/roaring/v2" "github.com/linxGnu/grocksdb" - supportlog "github.com/stellar/go-stellar-sdk/support/log" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/events" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" @@ -43,9 +41,9 @@ func HotChunkDir(dataDir string, chunkID chunk.ID) string { // RemoveHotChunkDir deletes chunkID's hot DB directory. Idempotent — // returns nil when the directory is already absent. // -// The caller MUST close chunkID's HotStore before calling this; -// otherwise RocksDB's LOCK file is still held and the on-disk state -// will be inconsistent. +// The caller MUST close chunkID's caller-owned RocksDB handle before calling +// this; otherwise RocksDB's LOCK file is still held and the on-disk state will be +// inconsistent. func RemoveHotChunkDir(dataDir string, chunkID chunk.ID) error { return os.RemoveAll(HotChunkDir(dataDir, chunkID)) } @@ -87,22 +85,6 @@ func CFNames() []string { return []string{DataCF, IndexCF, OffsetsCF} } // opener merges them into the shared per-chunk DB's PerCFOptions. func CFOptions() map[string]rocksdb.CFOptions { return hotStoreCFOptions() } -// openHotChunk opens (or creates) chunkID's per-Chunk hot DB. The three CFs -// auto-create on a fresh DB and rediscover on reopen. Unexported — OpenHotStore -// is the only caller (warmup is mandatory before the store is usable). -func openHotChunk(dataDir string, chunkID chunk.ID, logger *supportlog.Entry) (*rocksdb.Store, error) { - store, err := rocksdb.New(rocksdb.Config{ - Path: HotChunkDir(dataDir, chunkID), - ColumnFamilies: []string{DataCF, IndexCF, OffsetsCF}, - Logger: logger, - PerCFOptions: hotStoreCFOptions(), - }) - if err != nil { - return nil, fmt.Errorf("events: open hot chunk %s: %w", chunkID, err) - } - return store, nil -} - const ( dataKeyLen = 4 // event_id (chunk encoded by per-Chunk DB directory) indexKeyLen = 16 + 4 // term hash || event_id @@ -135,57 +117,24 @@ var ErrLedgerOutOfOrder = errors.New("events: ledger out of order") // - Reads (Lookup, FetchEvents, All) take NO HotStore-level lock — they guard // via chunkStore.IsClosed() and rely on the mirror's internal locks and // RocksDB's thread-safety. -// - Metadata split by Close: ChunkID, NextEventID, Index are infallible -// (cached, usable post-Close); EventCount, Offsets return ErrClosed after -// Close (Reader-interface contract). +// - Metadata split after the caller-owned store is closed: ChunkID, +// NextEventID, Index are infallible (cached, usable post-close); EventCount, +// Offsets return ErrClosed after close (Reader-interface contract). type HotStore struct { chunkStore *rocksdb.Store chunkID chunk.ID mirror *events.ConcurrentBitmaps offsets *events.ConcurrentLedgerOffsets - // ownsStore is true on the standalone OpenHotStore path; false when wrapping - // the SHARED per-chunk DB via NewWithStore (decision (a)), which hotchunk.DB - // owns and closes once. - ownsStore bool } // Compile-time guard: *HotStore satisfies Reader. var _ Reader = (*HotStore)(nil) -// OpenHotStore opens (or creates) chunkID's hot DB at -// HotChunkDir(dataDir, chunkID), warms up the in-memory mirror and -// offsets from disk, and returns a ready-to-use HotStore. The -// returned store owns its chunkStore; Close releases it. -func OpenHotStore( - dataDir string, - chunkID chunk.ID, - logger *supportlog.Entry, -) (*HotStore, error) { - if dataDir == "" { - return nil, errors.New("events: OpenHotStore requires a data dir") - } - if logger == nil { - return nil, errors.New("events: OpenHotStore requires a logger") - } - - chunkStore, err := openHotChunk(dataDir, chunkID, logger) - if err != nil { - return nil, err - } - h, err := NewWithStore(chunkStore, chunkID) - if err != nil { - _ = chunkStore.Close() - return nil, err - } - h.ownsStore = true - return h, nil -} - // NewWithStore wraps an ALREADY-OPEN rocksdb.Store as an events HotStore on the // three events CFs (CFNames()), running the mandatory warmup to rebuild the -// in-memory mirror + offsets. The store is NOT owned (Close is a no-op) — the -// constructor hotchunk uses to compose this facade over the shared per-chunk DB -// (decision (a)). The store must have CFNames() registered + CFOptions() applied. +// in-memory mirror + offsets. The store is owned by the caller — in production, +// hotchunk.DB composes this facade over the shared per-chunk DB and closes that +// DB once. The store must have CFNames() registered + CFOptions() applied. // A warmup failure returns the error WITHOUT closing the caller-owned store. func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) (*HotStore, error) { mirror, offsets, err := warmup(store, chunkID) @@ -200,23 +149,12 @@ func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) (*HotStore, error) { }, nil } -// Close releases the chunk store IF this HotStore owns it (standalone -// OpenHotStore); a no-op when wrapping the shared per-chunk DB (NewWithStore), -// which hotchunk.DB closes once. Idempotent; not safe to call alongside in-flight -// reads/writes on this HotStore. -func (h *HotStore) Close() error { - if !h.ownsStore { - return nil - } - return h.chunkStore.Close() -} - // ChunkID returns the chunk this store serves. func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } // EventCount is the total number of events committed to this Chunk // so far. Equal to the next event-id IngestLedgerEvents would assign. -// Returns (0, ErrClosed) after Close. The Reader interface signature +// Returns (0, ErrClosed) after the caller-owned store is closed. The Reader interface signature // is fallible to accommodate ColdReader's lazy metadata load; on the // hot side the value is always live and the error is only ErrClosed. func (h *HotStore) EventCount() (uint32, error) { @@ -249,7 +187,7 @@ func (h *HotStore) NextEventID() uint32 { return h.offsets.TotalEvents() } // with the live backing array. Calling Append on the view would // silently fork it from the live data; the contract is read-only. // -// Returns (nil, ErrClosed) after Close. +// Returns (nil, ErrClosed) after the caller-owned store is closed. func (h *HotStore) Offsets() (*events.LedgerOffsets, error) { if h.chunkStore.IsClosed() { return nil, ErrClosed @@ -271,7 +209,7 @@ func (h *HotStore) Index() *events.ConcurrentBitmaps { return h.mirror } // bitmap. Callers MUST NOT mutate it themselves. See Reader.Lookup // and ConcurrentBitmaps.Get for the full contract. Returns // (nil, ErrTermNotFound) when the term has no matching events. -// Returns (nil, ErrClosed) after Close. +// Returns (nil, ErrClosed) after the caller-owned store is closed. // // ctx is checked as a fast guard but the hot path does no blocking // I/O — the bitmap comes from the in-memory mirror. @@ -341,7 +279,7 @@ func (h *HotStore) LookupKeys(ctx context.Context, keys []events.TermKey) ([]*ro // RocksDB also has them. A miss indicates corruption or a // writer/reader mismatch, not a normal not-found case. // -// After Close, returns ErrClosed. +// After the caller-owned store is closed, returns ErrClosed. func (h *HotStore) FetchEvents(ctx context.Context, eventIDs []uint32) ([]events.Payload, error) { if h.chunkStore.IsClosed() { return nil, ErrClosed @@ -397,7 +335,7 @@ func (h *HotStore) FetchEvents(ctx context.Context, eventIDs []uint32) ([]events // Yielded Payloads are borrowed: ContractEventBytes aliases the iteration // buffer and is valid only until the next step — clone to retain. // -// After Close, yields (zero Payload, ErrClosed) and stops. +// After the caller-owned store is closed, yields (zero Payload, ErrClosed) and stops. // ctx is checked at entry and between iterator steps — // rocksdb.Store.IterateRange does not itself accept a ctx, so a // very slow Next() can block past a cancellation until the next @@ -472,7 +410,7 @@ func (h *HotStore) FetchRange(ctx context.Context, start, count uint32) iter.Seq // concurrent ingest between r.All(ctx) returning the Seq2 and the // consumer's first range step is included in the snapshot. // -// After Close, yields (zero Payload, ErrClosed) and stops. +// After the caller-owned store is closed, yields (zero Payload, ErrClosed) and stops. func (h *HotStore) All(ctx context.Context) iter.Seq2[events.Payload, error] { return func(yield func(events.Payload, error) bool) { // FetchRange stops iterating after yielding an error; we @@ -704,7 +642,7 @@ func (h *HotStore) applyLedger(p *preparedLedger) { // ────────────────────────────────────────────────────────────────── // Warmup — reconstructs the in-memory mirror + offsets from the -// per-Chunk DB's on-disk CFs. Called only by OpenHotStore. +// per-Chunk DB's on-disk CFs. Called by NewWithStore. // ────────────────────────────────────────────────────────────────── // warmup rebuilds the in-memory mirrors for chunkID by prefix-scanning diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go index ea5d3ce7d..821af210f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go @@ -19,6 +19,7 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/events" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" ) // silentLogger returns a logger whose output is buffered into an @@ -37,6 +38,7 @@ func silentLogger() *supportlog.Entry { type hotStoreHarness struct { dataDir string store *HotStore + raw *rocksdb.Store } // openHotStoreForTest opens a fresh per-Chunk hot DB for chunkID @@ -48,11 +50,39 @@ func openHotStoreForTest(t *testing.T, chunkID chunk.ID) *hotStoreHarness { t.Helper() dir := t.TempDir() - hot, err := OpenHotStore(dir, chunkID, silentLogger()) + hot, raw := openHotStoreForTestAt(t, dir, chunkID) + return &hotStoreHarness{dataDir: dir, store: hot, raw: raw} +} + +func openHotStoreForTestAt(t *testing.T, dir string, chunkID chunk.ID) (*HotStore, *rocksdb.Store) { + t.Helper() + hot, raw, err := tryOpenHotStoreForTest(t, dir, chunkID) require.NoError(t, err) - t.Cleanup(func() { _ = hot.Close() }) + return hot, raw +} + +func tryOpenHotStoreForTest(t *testing.T, dir string, chunkID chunk.ID) (*HotStore, *rocksdb.Store, error) { + t.Helper() + raw := openRawHotChunkForTest(t, dir, chunkID) + hot, err := NewWithStore(raw, chunkID) + if err != nil { + _ = raw.Close() + return nil, nil, err + } + t.Cleanup(func() { _ = raw.Close() }) + return hot, raw, nil +} - return &hotStoreHarness{dataDir: dir, store: hot} +func openRawHotChunkForTest(t *testing.T, dir string, chunkID chunk.ID) *rocksdb.Store { + t.Helper() + raw, err := rocksdb.New(rocksdb.Config{ + Path: HotChunkDir(dir, chunkID), + ColumnFamilies: CFNames(), + Logger: silentLogger(), + PerCFOptions: CFOptions(), + }) + require.NoError(t, err) + return raw } func makePayload(symbol string) (events.Payload, []events.TermKey) { @@ -105,16 +135,6 @@ func dataSym(t *testing.T, p events.Payload) string { return string(*eventOf(p).Body.V0.Data.Sym) } -func TestOpenHotStore_RequiresDataDirAndLogger(t *testing.T) { - dir := t.TempDir() - - _, err := OpenHotStore("", 0, silentLogger()) - require.Error(t, err, "missing dataDir") - - _, err = OpenHotStore(dir, 0, nil) - require.Error(t, err, "missing logger") -} - func TestHotStore_FreshChunkHasEmptyState(t *testing.T) { const chunkID = chunk.ID(0) h := openHotStoreForTest(t, chunkID) @@ -364,7 +384,7 @@ func TestHotStore_AllEmptyChunkYieldsNothing(t *testing.T) { func TestHotStore_CloseRejectsWrites(t *testing.T) { h := openHotStoreForTest(t, 0) - require.NoError(t, h.store.Close()) + require.NoError(t, h.raw.Close()) err := h.store.IngestLedgerEvents(2, nil) assert.ErrorIs(t, err, ErrClosed) } @@ -380,7 +400,7 @@ func TestHotStore_PostCloseReadsError(t *testing.T) { p, keys := makePayload("seed") require.NoError(t, h.store.IngestLedgerEvents(chunkID.FirstLedger(), []events.Payload{p})) - require.NoError(t, h.store.Close()) + require.NoError(t, h.raw.Close()) // Lookup must error rather than silently returning the cached bitmap. bm, err := h.store.Lookup(context.Background(), keys[0]) @@ -490,8 +510,8 @@ func TestHotStore_IngestLedgerEvents_RejectsOutOfRangeLedger(t *testing.T) { func TestHotStore_CloseIsIdempotent(t *testing.T) { h := openHotStoreForTest(t, 0) - require.NoError(t, h.store.Close()) - assert.NoError(t, h.store.Close()) + require.NoError(t, h.raw.Close()) + assert.NoError(t, h.raw.Close()) } func TestHotStore_ReopenRecoversState(t *testing.T) { @@ -501,15 +521,12 @@ func TestHotStore_ReopenRecoversState(t *testing.T) { const chunkID = chunk.ID(0) dir := t.TempDir() - hot1, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) + hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("before") require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1})) - require.NoError(t, hot1.Close()) + require.NoError(t, raw1.Close()) - hot2, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) - t.Cleanup(func() { _ = hot2.Close() }) + hot2, _ := openHotStoreForTestAt(t, dir, chunkID) assert.Equal(t, uint32(1), hot2.NextEventID(), "warmup recovered offsets") @@ -641,7 +658,7 @@ func TestHotStore_FetchRangeOutOfBoundsErrors(t *testing.T) { func TestHotStore_FetchRangePostCloseYieldsErrClosed(t *testing.T) { const chunkID = chunk.ID(0) h := openHotStoreForTest(t, chunkID) - require.NoError(t, h.store.Close()) + require.NoError(t, h.raw.Close()) require.ErrorIs(t, firstIterError(h.store.FetchRange(context.Background(), 0, 1)), ErrClosed) } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_warmup_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_warmup_test.go index bae6da25a..43d5bff41 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_warmup_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_warmup_test.go @@ -13,11 +13,11 @@ import ( ) // These tests exercise the (unexported) warmup() function indirectly -// through OpenHotStore, which is the only production caller. They +// through NewWithStore over an explicitly opened RocksDB store. They // document the "fresh chunk → empty caches", "ingested chunk → // reconstructed caches" contract. -func TestWarmup_FreshChunkProducesEmptyMirrorsViaOpenHotStore(t *testing.T) { +func TestWarmup_FreshChunkProducesEmptyMirrorsViaNewWithStore(t *testing.T) { const chunkID = chunk.ID(0) h := openHotStoreForTest(t, chunkID) @@ -37,8 +37,7 @@ func TestWarmup_RebuildsMirrorFromIngestedRows(t *testing.T) { const chunkID = chunk.ID(0) dir := t.TempDir() - hot1, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) + hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("alpha") p2, _ := makePayload("beta") require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1, p2})) @@ -49,12 +48,10 @@ func TestWarmup_RebuildsMirrorFromIngestedRows(t *testing.T) { for term, bm := range hot1.mirror.Snapshot() { expected[term] = bm.GetCardinality() } - require.NoError(t, hot1.Close()) + require.NoError(t, raw1.Close()) // Reopen — warmup replays events_index into a fresh mirror. - hot2, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) - t.Cleanup(func() { _ = hot2.Close() }) + hot2, _ := openHotStoreForTestAt(t, dir, chunkID) got := make(map[events.TermKey]uint64) for term, bm := range hot2.mirror.Snapshot() { @@ -67,17 +64,14 @@ func TestWarmup_RestoresEventIDsForRepeatedTerm(t *testing.T) { const chunkID = chunk.ID(0) dir := t.TempDir() - hot1, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) + hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("shared") p2, _ := makePayload("shared") p3, _ := makePayload("shared") require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1, p2, p3})) - require.NoError(t, hot1.Close()) + require.NoError(t, raw1.Close()) - hot2, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) - t.Cleanup(func() { _ = hot2.Close() }) + hot2, _ := openHotStoreForTestAt(t, dir, chunkID) contractTermKey := events.ComputeTermKey(eventOf(p1).ContractId[:], events.FieldContractID) bm, err := hot2.Lookup(context.Background(), contractTermKey) @@ -93,18 +87,15 @@ func TestWarmup_OffsetsReconstructedAcrossLedgers(t *testing.T) { const chunkID = chunk.ID(0) dir := t.TempDir() - hot1, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) + hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("a") p2, _ := makePayload("b") require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1, p2})) p3, _ := makePayload("c") require.NoError(t, hot1.IngestLedgerEvents(3, []events.Payload{p3})) - require.NoError(t, hot1.Close()) + require.NoError(t, raw1.Close()) - hot2, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) - t.Cleanup(func() { _ = hot2.Close() }) + hot2, _ := openHotStoreForTestAt(t, dir, chunkID) assert.Equal(t, uint32(3), mustEventCount(t, hot2)) @@ -127,8 +118,7 @@ func TestWarmup_OffsetsReconstructedAcrossLedgers(t *testing.T) { //nolint:unparam // chunkID kept as a param for call-site clarity; today every caller uses 0 func corruptHotChunk(t *testing.T, dir string, chunkID chunk.ID, mutate func(raw *rocksdb.Store)) { t.Helper() - raw, err := openHotChunk(dir, chunkID, silentLogger()) - require.NoError(t, err) + raw := openRawHotChunkForTest(t, dir, chunkID) defer func() { require.NoError(t, raw.Close()) }() // release LOCK even if mutate fails mutate(raw) } @@ -137,12 +127,11 @@ func TestWarmup_RejectsDataEventBeyondOffsets(t *testing.T) { const chunkID = chunk.ID(0) dir := t.TempDir() - hot1, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) + hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("a") p2, _ := makePayload("b") require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1, p2})) // total = 2 - require.NoError(t, hot1.Close()) + require.NoError(t, raw1.Close()) // An orphan data row well beyond total (id 7, total = 2): proves the // check catches any id >= total, not just one past the boundary. @@ -150,7 +139,7 @@ func TestWarmup_RejectsDataEventBeyondOffsets(t *testing.T) { require.NoError(t, raw.Put(DataCF, encodeDataKey(7), []byte("orphan"))) }) - _, err = OpenHotStore(dir, chunkID, silentLogger()) + _, _, err := tryOpenHotStoreForTest(t, dir, chunkID) // Branch-specific substring: every corruption shares "corrupt chunk", // so assert the data-orphan message to prove this branch fired. require.ErrorContains(t, err, "data present at id >= committed count") @@ -160,13 +149,12 @@ func TestWarmup_RejectsOffsetsGap(t *testing.T) { const chunkID = chunk.ID(0) dir := t.TempDir() - hot1, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) + hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) for _, seq := range []uint32{2, 3, 4} { p, _ := makePayload("x") require.NoError(t, hot1.IngestLedgerEvents(seq, []events.Payload{p})) } - require.NoError(t, hot1.Close()) + require.NoError(t, raw1.Close()) // Drop ledger 3's offset row: warmup then iterates 2, 4 and must // reject the gap. This is the sequence check that moved out of @@ -175,7 +163,7 @@ func TestWarmup_RejectsOffsetsGap(t *testing.T) { require.NoError(t, raw.Delete(OffsetsCF, encodeOffsetKey(3))) }) - _, err = OpenHotStore(dir, chunkID, silentLogger()) + _, _, err := tryOpenHotStoreForTest(t, dir, chunkID) require.ErrorContains(t, err, "expected ledger 3, got 4") } @@ -183,13 +171,12 @@ func TestWarmup_RejectsOffsetsOverflow(t *testing.T) { const chunkID = chunk.ID(0) dir := t.TempDir() - hot1, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) + hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) for _, seq := range []uint32{2, 3} { p, _ := makePayload("x") require.NoError(t, hot1.IngestLedgerEvents(seq, []events.Payload{p})) } - require.NoError(t, hot1.Close()) + require.NoError(t, raw1.Close()) // Overwrite the offset rows with counts that sum past uint32: warmup // must reject the cumulative overflow rather than silently wrapping. @@ -198,7 +185,7 @@ func TestWarmup_RejectsOffsetsOverflow(t *testing.T) { require.NoError(t, raw.Put(OffsetsCF, encodeOffsetKey(3), encodeLedgerEventCount(2_000_000_000))) }) - _, err = OpenHotStore(dir, chunkID, silentLogger()) + _, _, err := tryOpenHotStoreForTest(t, dir, chunkID) require.ErrorContains(t, err, "cumulative event count overflow") } @@ -206,9 +193,8 @@ func TestWarmup_RejectsOrphanInEmptyChunk(t *testing.T) { const chunkID = chunk.ID(0) dir := t.TempDir() - hot1, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) - require.NoError(t, hot1.Close()) // total = 0, nothing committed + _, raw1 := openHotStoreForTestAt(t, dir, chunkID) + require.NoError(t, raw1.Close()) // total = 0, nothing committed // A data row in a chunk that committed nothing: total == 0, so the // tail Get is skipped and the orphan scan must fire from id 0. @@ -216,7 +202,7 @@ func TestWarmup_RejectsOrphanInEmptyChunk(t *testing.T) { require.NoError(t, raw.Put(DataCF, encodeDataKey(0), []byte("orphan"))) }) - _, err = OpenHotStore(dir, chunkID, silentLogger()) + _, _, err := tryOpenHotStoreForTest(t, dir, chunkID) require.ErrorContains(t, err, "data present at id >= committed count 0") } @@ -224,12 +210,11 @@ func TestWarmup_RejectsMissingTailDataEvent(t *testing.T) { const chunkID = chunk.ID(0) dir := t.TempDir() - hot1, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) + hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("a") p2, _ := makePayload("b") require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1, p2})) // total = 2 - require.NoError(t, hot1.Close()) + require.NoError(t, raw1.Close()) // Drop the last data row (event id total-1 == 1) while offsets still // count 2. @@ -237,7 +222,7 @@ func TestWarmup_RejectsMissingTailDataEvent(t *testing.T) { require.NoError(t, raw.Delete(DataCF, encodeDataKey(1))) }) - _, err = OpenHotStore(dir, chunkID, silentLogger()) + _, _, err := tryOpenHotStoreForTest(t, dir, chunkID) require.ErrorContains(t, err, "missing from data") } @@ -245,12 +230,11 @@ func TestWarmup_RejectsIndexBeyondCommitted(t *testing.T) { const chunkID = chunk.ID(0) dir := t.TempDir() - hot1, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) + hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("a") p2, _ := makePayload("b") require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1, p2})) // total = 2 - require.NoError(t, hot1.Close()) + require.NoError(t, raw1.Close()) // An index row at exactly total (id 2): the tightest "beyond // committed" case, pinning the > (not >=) bound — valid ids are 0..1. @@ -260,7 +244,7 @@ func TestWarmup_RejectsIndexBeyondCommitted(t *testing.T) { require.NoError(t, raw.Put(IndexCF, encodeIndexKey(term, 2), nil)) }) - _, err = OpenHotStore(dir, chunkID, silentLogger()) + _, _, err := tryOpenHotStoreForTest(t, dir, chunkID) require.ErrorContains(t, err, "index references event 2 but only 2 committed") } @@ -268,16 +252,13 @@ func TestWarmup_OffsetsHandleEmptyTrailingLedger(t *testing.T) { const chunkID = chunk.ID(0) dir := t.TempDir() - hot1, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) + hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p, _ := makePayload("only") require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p})) require.NoError(t, hot1.IngestLedgerEvents(3, nil)) - require.NoError(t, hot1.Close()) + require.NoError(t, raw1.Close()) - hot2, err := OpenHotStore(dir, chunkID, silentLogger()) - require.NoError(t, err) - t.Cleanup(func() { _ = hot2.Close() }) + hot2, _ := openHotStoreForTestAt(t, dir, chunkID) assert.Equal(t, uint32(1), mustEventCount(t, hot2)) assert.Equal(t, 2, mustOffsets(t, hot2).LedgerCount()) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/reader.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/reader.go index 77f306d4c..aa46ebeec 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/reader.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/reader.go @@ -183,12 +183,6 @@ type Reader interface { // Each events.Payload carries its LedgerSequence, so consumers can // track ledger boundaries without separate signaling. All(ctx context.Context) iter.Seq2[events.Payload, error] - - // Close releases any resources the Reader holds. Idempotent. - // After Close, Lookup / FetchEvents / FetchRange / All return - // ErrClosed. Metadata accessors (ChunkID, EventCount, Offsets) - // survive Close — see each impl's docstring for details. - Close() error } // validateSortedEventIDs returns a wrapped ErrUnsortedEventIDs if diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go index ce246d0d2..8846759bb 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go @@ -9,8 +9,6 @@ import ( "iter" "sync" - supportlog "github.com/stellar/go-stellar-sdk/support/log" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores" @@ -18,9 +16,7 @@ import ( ) // LedgersCF is the column family the hot ledger data lives in. Registered the -// same whether the DB is the shared per-chunk multi-CF DB (decision (a)) or a -// standalone single-purpose DB (OpenHotStore), so the on-disk layout is -// identical either way. +// shared per-chunk multi-CF DB (decision (a)). const LedgersCF = "ledgers" // Entry — one (sequence, uncompressed ledger bytes) pair. Compression is @@ -36,62 +32,23 @@ type Entry struct { // the ingest driver can reject a mismatched store. The store does not itself // range-check writes (the driver's drain loop already validates every sequence). // -// Concurrency: all methods, including Close, are safe for concurrent -// use. rocksdb.Store.Close CAS-marks the store closed and then drains -// in-flight ops (each holds an RLock for its duration) before releasing -// resources; a read/write racing Close either completes first or -// observes the closed store and returns stores.ErrStoreClosed. Close is -// idempotent. HotStore adds no unguarded state of its own — the -// compressor pool and decompressor are both concurrent-safe. +// Concurrency: all methods are safe for concurrent use, including use alongside +// the caller-owned rocksdb.Store.Close. A read/write racing Close either completes +// first or observes the closed store and returns stores.ErrStoreClosed. HotStore +// adds no unguarded state of its own — the compressor pool and decompressor are +// both concurrent-safe. type HotStore struct { store *rocksdb.Store chunkID chunk.ID - // ownsStore is true on the standalone OpenHotStore path (Close closes the - // store); false when wrapping the SHARED per-chunk DB via NewWithStore, - // which hotchunk.DB owns and closes once. - ownsStore bool - dec *zstd.Decompressor + dec *zstd.Decompressor // compPool — per-store pool of zstd.Compressors; each concurrent AddLedgers // borrows one for its Encode call. compPool sync.Pool } -// OpenHotStore validates inputs and returns an open HotStore bound -// to chunkID (see the HotStore doc on chunk binding). path and -// logger are both required; logger is forwarded to the -// pkg/rocksdb wrapper (rocksdb writes the on-open state line and -// the close-time Flush warning through it). HotStore itself does -// not emit any logs — the cold store, by contrast, takes no -// logger because packfile is silent. Rides on RocksDB defaults — -// no explicit block cache (RocksDB's per-CF default plus OS page -// cache cover range scans), no bloom filter (callers know in -// advance which sequences this store holds, so it is never asked -// for a key it doesn't have), no WAL cap (graceful Close flushes -// the memtable; ungraceful WAL replay at this scale is sub-second). -// Re-tune only with a workload measurement. -func OpenHotStore(path string, chunkID chunk.ID, logger *supportlog.Entry) (*HotStore, error) { - if path == "" { - return nil, stores.ErrInvalidConfig - } - if logger == nil { - return nil, stores.ErrInvalidConfig - } - store, err := rocksdb.New(rocksdb.Config{ - Path: path, - ColumnFamilies: []string{LedgersCF}, - Logger: logger, - }) - if err != nil { - return nil, err - } - h := NewWithStore(store, chunkID) - h.ownsStore = true - return h, nil -} - // NewWithStore wraps an ALREADY-OPEN rocksdb.Store as a ledger HotStore on -// LedgersCF. The store is NOT owned (Close is a no-op) — the constructor hotchunk -// uses to compose this facade over the shared multi-CF DB (decision (a)). The +// LedgersCF. The store is owned by the caller — in production, hotchunk.DB +// composes this facade over the shared multi-CF DB and closes that DB once. The // store must have LedgersCF registered. func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) *HotStore { return &HotStore{ @@ -104,16 +61,6 @@ func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) *HotStore { } } -// Close releases the store IF this HotStore owns it (standalone OpenHotStore); -// a no-op when wrapping the shared per-chunk DB (NewWithStore), which hotchunk.DB -// closes once. Idempotent; not safe to call alongside in-flight reads/writes. -func (h *HotStore) Close() error { - if !h.ownsStore { - return nil - } - return h.store.Close() -} - // ChunkID returns the chunk this store is bound to (constructor-supplied; // never reads the store). func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go index 4a7f89ecd..52353b939 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go @@ -18,6 +18,7 @@ import ( "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores" ) @@ -31,41 +32,25 @@ func silentLogger() *supportlog.Entry { func openTestHotStore(t *testing.T) *HotStore { t.Helper() - h, err := OpenHotStore(t.TempDir(), chunk.ID(0), silentLogger()) - require.NoError(t, err) - t.Cleanup(func() { _ = h.Close() }) + h, _ := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) return h } -func TestOpenHotStore_ValidatesInputs(t *testing.T) { - _, err := OpenHotStore("", chunk.ID(0), silentLogger()) - require.ErrorIs(t, err, stores.ErrInvalidConfig) - - _, err = OpenHotStore(t.TempDir(), chunk.ID(0), nil) - require.ErrorIs(t, err, stores.ErrInvalidConfig) -} - -func TestOpenHotStore_RecordsChunkBinding(t *testing.T) { - h, err := OpenHotStore(t.TempDir(), chunk.ID(7), silentLogger()) - require.NoError(t, err) - t.Cleanup(func() { _ = h.Close() }) - require.Equal(t, chunk.ID(7), h.ChunkID()) -} - -func TestOpenHotStore_CreatesMissingDirectory(t *testing.T) { - path := filepath.Join(t.TempDir(), "subdir-never-created") - h, err := OpenHotStore(path, chunk.ID(0), silentLogger()) +func openTestHotStoreAt(t *testing.T, path string, chunkID chunk.ID) (*HotStore, *rocksdb.Store) { + t.Helper() + store, err := rocksdb.New(rocksdb.Config{ + Path: path, + ColumnFamilies: []string{LedgersCF}, + Logger: silentLogger(), + }) require.NoError(t, err) - require.NotNil(t, h) - t.Cleanup(func() { _ = h.Close() }) + t.Cleanup(func() { _ = store.Close() }) + return NewWithStore(store, chunkID), store } -func TestHotStore_CloseIsIdempotent(t *testing.T) { - h, err := OpenHotStore(t.TempDir(), chunk.ID(0), silentLogger()) - require.NoError(t, err) - - require.NoError(t, h.Close()) - require.NoError(t, h.Close()) +func TestNewWithStore_RecordsChunkBinding(t *testing.T) { + h, _ := openTestHotStoreAt(t, t.TempDir(), chunk.ID(7)) + require.Equal(t, chunk.ID(7), h.ChunkID()) } func TestHotStore_AddGetRoundTripVerbatim(t *testing.T) { @@ -239,14 +224,11 @@ func TestHotStore_GracefulCloseAndReopen(t *testing.T) { {Seq: 15, Bytes: []byte("payload-15")}, } - first, err := OpenHotStore(path, chunk.ID(0), silentLogger()) - require.NoError(t, err) + first, firstStore := openTestHotStoreAt(t, path, chunk.ID(0)) require.NoError(t, first.AddLedgers(seeded...)) - require.NoError(t, first.Close()) + require.NoError(t, firstStore.Close()) - second, err := OpenHotStore(path, chunk.ID(0), silentLogger()) - require.NoError(t, err) - t.Cleanup(func() { _ = second.Close() }) + second, _ := openTestHotStoreAt(t, path, chunk.ID(0)) for _, want := range seeded { got, err := second.GetLedgerRaw(want.Seq) @@ -256,12 +238,11 @@ func TestHotStore_GracefulCloseAndReopen(t *testing.T) { } func TestHotStore_PostCloseOps(t *testing.T) { - h, err := OpenHotStore(t.TempDir(), chunk.ID(0), silentLogger()) - require.NoError(t, err) - require.NoError(t, h.Close()) + h, store := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) + require.NoError(t, store.Close()) require.ErrorIs(t, h.AddLedgers(Entry{Seq: 1, Bytes: []byte("v")}), stores.ErrStoreClosed) - _, err = h.GetLedgerRaw(1) + _, err := h.GetLedgerRaw(1) require.ErrorIs(t, err, stores.ErrStoreClosed) var iterErr error for _, e := range h.IterateLedgers(0, 100) { @@ -279,7 +260,7 @@ func TestHotStore_PostCloseOps(t *testing.T) { } func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { - h := openTestHotStore(t) + h, store := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) for i := range uint32(50) { require.NoError(t, h.AddLedgers(Entry{Seq: i, Bytes: []byte("v")})) } @@ -310,7 +291,7 @@ func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { } time.Sleep(50 * time.Millisecond) - require.NoError(t, h.Close()) + require.NoError(t, store.Close()) stop.Store(true) wg.Wait() diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go index 9698c35b2..907e62e62 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go @@ -4,8 +4,6 @@ package txhash import ( - supportlog "github.com/stellar/go-stellar-sdk/support/log" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores" @@ -35,37 +33,12 @@ type Entry struct { type HotStore struct { store *rocksdb.Store chunkID chunk.ID - // ownsStore is true on the standalone NewHotStore path; false when wrapping - // the SHARED per-chunk DB via NewWithStore (decision (a)), which - // hotchunk.DB owns and closes once. - ownsStore bool -} - -// NewHotStore validates inputs and returns an open HotStore bound to -// chunkID (see the HotStore doc on chunk binding). -func NewHotStore(path string, chunkID chunk.ID, logger *supportlog.Entry) (*HotStore, error) { - if path == "" { - return nil, rocksdb.ErrInvalidConfig - } - if logger == nil { - return nil, rocksdb.ErrInvalidConfig - } - store, err := rocksdb.New(rocksdb.Config{ - Path: path, - ColumnFamilies: CFNames(), - Logger: logger, - Tuning: tuning(), - }) - if err != nil { - return nil, err - } - return &HotStore{store: store, chunkID: chunkID, ownsStore: true}, nil } // NewWithStore wraps an ALREADY-OPEN rocksdb.Store as a txhash HotStore on the -// single txhash CF (CFNames()). The store is NOT owned (Close is a no-op) — -// the constructor hotchunk uses to compose this facade over the shared per-chunk -// DB. The store must have CFNames() registered. +// single txhash CF (CFNames()). The store is owned by the caller — in production, +// hotchunk.DB composes this facade over the shared per-chunk DB and closes that DB +// once. The store must have CFNames() registered. func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) *HotStore { return &HotStore{store: store, chunkID: chunkID} } @@ -133,16 +106,6 @@ func tuning() rocksdb.Tuning { } } -// Close releases the store IF this HotStore owns it (standalone NewHotStore); -// a no-op when wrapping the shared per-chunk DB (NewWithStore), which hotchunk.DB -// closes once. Idempotent. -func (h *HotStore) Close() error { - if !h.ownsStore { - return nil - } - return h.store.Close() -} - // ChunkID returns the chunk this store is bound to (constructor-supplied; // never reads the store). func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go index 02069ed1d..b0c6ead16 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go @@ -2,7 +2,6 @@ package txhash import ( "bytes" - "path/filepath" "sync" "sync/atomic" "testing" @@ -41,41 +40,26 @@ func txhashFor(nibble, tag byte) [32]byte { func openTestHotStore(t *testing.T) *HotStore { t.Helper() - s, err := NewHotStore(t.TempDir(), chunk.ID(0), silentLogger()) - require.NoError(t, err) - t.Cleanup(func() { _ = s.Close() }) + s, _ := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) return s } -func TestNewHotStore_ValidatesInputs(t *testing.T) { - _, err := NewHotStore("", chunk.ID(0), silentLogger()) - require.ErrorIs(t, err, rocksdb.ErrInvalidConfig) - - _, err = NewHotStore(t.TempDir(), chunk.ID(0), nil) - require.ErrorIs(t, err, rocksdb.ErrInvalidConfig) -} - -func TestNewHotStore_RecordsChunkBinding(t *testing.T) { - s, err := NewHotStore(t.TempDir(), chunk.ID(7), silentLogger()) - require.NoError(t, err) - t.Cleanup(func() { _ = s.Close() }) - require.Equal(t, chunk.ID(7), s.ChunkID()) -} - -func TestNewHotStore_CreatesMissingDirectory(t *testing.T) { - path := filepath.Join(t.TempDir(), "subdir-never-created") - s, err := NewHotStore(path, chunk.ID(0), silentLogger()) +func openTestHotStoreAt(t *testing.T, path string, chunkID chunk.ID) (*HotStore, *rocksdb.Store) { + t.Helper() + store, err := rocksdb.New(rocksdb.Config{ + Path: path, + ColumnFamilies: CFNames(), + Logger: silentLogger(), + Tuning: Tuning(), + }) require.NoError(t, err) - require.NotNil(t, s) - t.Cleanup(func() { _ = s.Close() }) + t.Cleanup(func() { _ = store.Close() }) + return NewWithStore(store, chunkID), store } -func TestHotStore_CloseIsIdempotent(t *testing.T) { - s, err := NewHotStore(t.TempDir(), chunk.ID(0), silentLogger()) - require.NoError(t, err) - - require.NoError(t, s.Close()) - require.NoError(t, s.Close()) +func TestNewWithStore_RecordsChunkBinding(t *testing.T) { + s, _ := openTestHotStoreAt(t, t.TempDir(), chunk.ID(7)) + require.Equal(t, chunk.ID(7), s.ChunkID()) } func TestHotStore_AddGetRoundTrip(t *testing.T) { @@ -156,13 +140,12 @@ func TestHotStore_AddEntriesMultiple(t *testing.T) { } func TestHotStore_PostCloseOps(t *testing.T) { - s, err := NewHotStore(t.TempDir(), chunk.ID(0), silentLogger()) - require.NoError(t, err) - require.NoError(t, s.Close()) + s, store := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) + require.NoError(t, store.Close()) h := txhashFor(0x5, 1) require.ErrorIs(t, s.AddEntries([]Entry{{Hash: h, LedgerSeq: 1}}), rocksdb.ErrStoreClosed) - _, err = s.Get(h) + _, err := s.Get(h) require.ErrorIs(t, err, rocksdb.ErrStoreClosed) require.ErrorIs(t, s.AddEntries(nil), rocksdb.ErrStoreClosed) @@ -172,18 +155,15 @@ func TestHotStore_PostCloseOps(t *testing.T) { func TestHotStore_GracefulCloseAndReopenRoundTrips(t *testing.T) { path := t.TempDir() - first, err := NewHotStore(path, chunk.ID(0), silentLogger()) - require.NoError(t, err) + first, firstStore := openTestHotStoreAt(t, path, chunk.ID(0)) for n := range 16 { require.NoError(t, first.AddEntries([]Entry{ {Hash: txhashFor(byte(n), 1), LedgerSeq: uint32(n) + 1}, })) } - require.NoError(t, first.Close()) + require.NoError(t, firstStore.Close()) - second, err := NewHotStore(path, chunk.ID(0), silentLogger()) - require.NoError(t, err) - t.Cleanup(func() { _ = second.Close() }) + second, _ := openTestHotStoreAt(t, path, chunk.ID(0)) for n := range 16 { got, err := second.Get(txhashFor(byte(n), 1)) @@ -193,7 +173,7 @@ func TestHotStore_GracefulCloseAndReopenRoundTrips(t *testing.T) { } func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { - s := openTestHotStore(t) + s, store := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) // Pre-populate a spread of distinct keys. pre := make([]Entry, 16) for n := range 16 { @@ -220,7 +200,7 @@ func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { } time.Sleep(50 * time.Millisecond) - require.NoError(t, s.Close()) + require.NoError(t, store.Close()) stop.Store(true) wg.Wait() From 4481062d9d054a680815fcadf4fcb25cec81cc40 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 16:51:30 -0400 Subject: [PATCH 22/55] txhash: collapse vestigial Tuning()/tuning() split into one func The private tuning() only existed because the now-deleted NewHotStore called it internally; its sole remaining caller was the exported Tuning() wrapper. Merge the two (values unchanged). --- .../fullhistory/pkg/stores/txhash/hot_store.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go index 907e62e62..f7b71313f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go @@ -48,14 +48,11 @@ func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) *HotStore { func CFNames() []string { return []string{txhashCF} } // Tuning returns this facade's RocksDB tuning, applied to the shared per-chunk -// DB by the hotchunk opener. -func Tuning() rocksdb.Tuning { return tuning() } - -// tuning — the hot txhash workload is write-once / point-lookup; the -// cross-knob interactions below are non-obvious enough that they get an -// explicit per-stanza rationale. The other facades ride on RocksDB defaults -// by contrast — only this workload earned the calibration. -func tuning() rocksdb.Tuning { +// DB by the hotchunk opener. The hot txhash workload is write-once / +// point-lookup; the cross-knob interactions below are non-obvious enough that +// they get an explicit per-stanza rationale. The other facades ride on RocksDB +// defaults by contrast — only this workload earned the calibration. +func Tuning() rocksdb.Tuning { return rocksdb.Tuning{ // 64 MB memtable so one flush produces one ~64 MB SST under // uniform writes. From 270bf500a206c6eff63a0aacf91efdba11cff058 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 17:26:01 -0400 Subject: [PATCH 23/55] Remove live catalog test seam --- .../internal/fullhistory/daemon.go | 9 - .../internal/fullhistory/e2e_test.go | 160 ++++++++---------- 2 files changed, 75 insertions(+), 94 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index 47fefe5a4..d1af58d90 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -67,12 +67,6 @@ type daemonOptions struct { // fixed geometry.ChunksPerTxhashIndex. Tests set it to 1 so a single chunk's // freeze is a terminal index (exercising the fold+prune path cheaply). chunksPerTxhashIndex uint32 - - // onCatalog, when set, receives the daemon's bound Catalog (test-only). The - // metastore is opened RocksDB-primary (exclusive LOCK), so a test cannot open a - // second handle while the daemon runs; this lets it inspect durable state live - // through the daemon's own catalog (safe for concurrent reads). - onCatalog func(*catalog.Catalog) } const defaultRestartBackoff = 5 * time.Second @@ -121,9 +115,6 @@ func runDaemonWith(ctx context.Context, configPath string, opts daemonOptions) e return err } cat := catalog.NewCatalog(store, NewLayoutFromPaths(paths), txLayout) - if opts.onCatalog != nil { - opts.onCatalog(cat) - } // --- Resolve the backfill backend: injected (tests) or built from // [backfill.datastore] (production; nil ⇒ frontfill-only). Its Tip drives both diff --git a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go index b1d6a10c6..952ec7621 100644 --- a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go @@ -104,14 +104,16 @@ func (s *e2eGetter) GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseM return nil, ctx.Err() } -// e2eMetrics is a concurrency-safe observability.Metrics that counts the chunk -// boundaries and freezes the daemon emits (the rest discarded via NopMetrics). +// e2eMetrics is a concurrency-safe observability.Metrics that records the +// lifecycle signals this test waits on. type e2eMetrics struct { observability.NopMetrics mu sync.Mutex boundaries int freezes int + discarded int + pruned int } func (m *e2eMetrics) ChunkBoundary() { @@ -126,6 +128,18 @@ func (m *e2eMetrics) Freeze(time.Duration) { m.freezes++ } +func (m *e2eMetrics) Discard(count int, _ time.Duration) { + m.mu.Lock() + defer m.mu.Unlock() + m.discarded += count +} + +func (m *e2eMetrics) Prune(count int, _ time.Duration) { + m.mu.Lock() + defer m.mu.Unlock() + m.pruned += count +} + func (m *e2eMetrics) boundaryCount() int { m.mu.Lock() defer m.mu.Unlock() @@ -138,6 +152,18 @@ func (m *e2eMetrics) snapshotFreezeCount() int { return m.freezes } +func (m *e2eMetrics) discardedCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.discarded +} + +func (m *e2eMetrics) prunedCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.pruned +} + // e2eConfigPath writes a daemon TOML for an in-process E2E: genesis floor (no // tip needed to validate/start) and the given retention width. captive_core_config // is a stub path the test's injected CoreOpener replaces, never opening a real core. @@ -165,20 +191,15 @@ format = "text" } // runDaemonInBackground starts runDaemonWith on a cancellable ctx and returns a -// cancel func, a channel carrying its (clean-shutdown) return, and a channel -// delivering the daemon's OWN bound *catalog.Catalog (captured via the onCatalog -// seam). The metastore is opened RocksDB-primary (exclusive LOCK), so a test -// cannot open a second handle while the daemon runs — instead it reads durable -// state through the daemon's own catalog (safe for concurrent reads). A young- -// network tip (inside chunk 0) means backfill is a no-op and first-start ingests -// directly from genesis via the fake core. +// cancel func plus a channel carrying its (clean-shutdown) return. A young-network +// tip (inside chunk 0) means backfill is a no-op and first-start ingests directly +// from genesis via the fake core. func runDaemonInBackground( t *testing.T, cfgPath string, core *e2eCore, served *atomic.Int32, metrics observability.Metrics, -) (context.CancelFunc, <-chan error, <-chan *catalog.Catalog) { +) (context.CancelFunc, <-chan error) { t.Helper() ctx, cancelFn := context.WithCancel(context.Background()) errCh := make(chan error, 1) - catChan := make(chan *catalog.Catalog, 1) opts := daemonOptions{ Backend: &fakeBackend{tip: chunk.FirstLedgerSeq + 5}, // young: no backfill Core: core, @@ -187,27 +208,9 @@ func runDaemonInBackground( Metrics: metrics, RestartBackoff: 10 * time.Millisecond, chunksPerTxhashIndex: 1, - onCatalog: func(cat *catalog.Catalog) { - select { - case catChan <- cat: - default: - } - }, } go func() { errCh <- runDaemonWith(ctx, cfgPath, opts) }() - return cancelFn, errCh, catChan -} - -// awaitCatalog waits for the daemon to hand back its bound catalog. -func awaitCatalog(t *testing.T, catCh <-chan *catalog.Catalog) *catalog.Catalog { - t.Helper() - select { - case cat := <-catCh: - return cat - case <-time.After(10 * time.Second): - t.Fatal("daemon did not bind a catalog") - return nil - } + return cancelFn, errCh } // waitClean cancels the daemon and requires a clean (nil) shutdown. @@ -316,11 +319,7 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing // freezing, folding, and discarding each just-closed chunk off the doorbell. // ===================================================================== cfgPath := e2eConfigPath(t, dataDir, 0) // retention 0 (full history) for now - cancel, done, catCh := runDaemonInBackground(t, cfgPath, core, &served, metrics) - - // Inspect durable state through the daemon's OWN bound catalog (metastore is - // RocksDB-primary, so a second handle would fail the LOCK). - cat := awaitCatalog(t, catCh) + cancel, done := runDaemonInBackground(t, cfgPath, core, &served, metrics) // Wait until ingestion crosses BOTH boundaries and commits into chunk 2. // Delivering c2First proves both boundary handoffs fired (chunks 0 and 1 @@ -331,50 +330,48 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing return core.delivered.Load() >= c2First }, 600*time.Second, 200*time.Millisecond, "ingestion must cross both boundaries into chunk 2") - // The boundary doorbells have rung. Per chunk, the durable completion signal is: - // the window has a FROZEN txhash coverage (the .idx) AND the chunk's hot key is - // gone (discarded). - w0 := cat.TxHashIndexLayout().TxHashIndexID(c0) - w1 := cat.TxHashIndexLayout().TxHashIndexID(c1) require.Eventually(t, func() bool { - for w, c := range map[geometry.TxHashIndexID]chunk.ID{w0: c0, w1: c1} { - _, hasCov, err := cat.FrozenTxHashIndex(w) - if err != nil || !hasCov { - return false - } - has, err := hotKeyExists(cat, c) - if err != nil || has { - return false - } - } - return true + return metrics.discardedCount() >= 2 }, 60*time.Second, 50*time.Millisecond, "the boundary ticks must freeze+fold+discard chunks 0 and 1") require.GreaterOrEqual(t, served.Load(), int32(1), "reads were served") require.Equal(t, c0First, core.resumeSeen.Load(), "first start resumes captive core at genesis (watermark+1)") + // ===================================================================== + // STEP 2 — clean shutdown. The supervised loop returns nil on ctx cancel. + // ===================================================================== + waitClean(t, cancel, done) + + // Bind a fresh inspection catalog on the (now lock-free) data dir for the + // post-shutdown reads. It MUST be closed before the restart reopens the metastore. + postCat, closePost := e2eReadCatalog(t, dataDir) + w0 := postCat.TxHashIndexLayout().TxHashIndexID(c0) + // --- Correctness: chunks 0 and 1 per-chunk cold artifacts (ledgers + events) froze. --- for _, c := range []chunk.ID{c0, c1} { for _, kind := range []geometry.Kind{geometry.KindLedgers, geometry.KindEvents} { - st, err := cat.State(c, kind) + st, err := postCat.State(c, kind) require.NoError(t, err) assert.Equal(t, geometry.StateFrozen, st, "chunk %s %s is frozen", c, kind) } + has, err := hotKeyExists(postCat, c) + require.NoError(t, err) + assert.False(t, has, "chunk %s hot key is discarded", c) } // The window's txhash index is a frozen, terminal coverage (the .idx the cold // getTransaction read resolves against). - frozenCov, ok, err := cat.FrozenTxHashIndex(w0) + frozenCov, ok, err := postCat.FrozenTxHashIndex(w0) require.NoError(t, err) require.True(t, ok, "chunk 0's window has a frozen txhash coverage") - require.True(t, cat.TxHashIndexLayout().IsTerminalCoverage(frozenCov), "a one-chunk (cpi=1) window is terminal") + require.True(t, postCat.TxHashIndexLayout().IsTerminalCoverage(frozenCov), "a one-chunk (cpi=1) window is terminal") // ===================================================================== - // STEP 2 — getTransaction-style hash→seq lookup, cold tier. + // STEP 3 — getTransaction-style hash→seq lookup, cold tier. // ===================================================================== // Cold .idx — the exact reader getTransaction will sit on for frozen history. - coldReader, err := txhash.OpenColdReader(cat.Layout().TxHashIndexFilePath(frozenCov)) + coldReader, err := txhash.OpenColdReader(postCat.Layout().TxHashIndexFilePath(frozenCov)) require.NoError(t, err) gotSeq, err := coldReader.Get(coldHash) require.NoError(t, err, "the chunk-0 tx hash must resolve from the frozen cold index") @@ -389,16 +386,8 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing assert.GreaterOrEqual(t, metrics.snapshotFreezeCount(), 1, "at least one freeze stage ran") // ===================================================================== - // STEP 3 — clean shutdown. The supervised loop returns nil on ctx cancel. + // STEP 4 — hot lookup and restart watermark. // ===================================================================== - waitClean(t, cancel, done) - - // Bind a fresh inspection catalog on the (now lock-free) data dir for the - // post-shutdown reads. It MUST be closed before the restart reopens the metastore. - postCat, closePost := e2eReadCatalog(t, dataDir) - - // The durable watermark, re-derived from post-shutdown state (the basis for the - // restart's resume-with-no-gap assertion). wmBeforeRestart := mustDeriveWatermark(t, postCat) require.GreaterOrEqual(t, wmBeforeRestart, c2First, "watermark advanced into chunk 2") @@ -416,7 +405,7 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing // writer closed (the same transient a production reader retries through). var liveDB *hotchunk.DB require.Eventually(t, func() bool { - db, oerr := hotchunk.Open(cat.Layout().HotChunkPath(c2), c2, silentLogger()) + db, oerr := hotchunk.Open(postCat.Layout().HotChunkPath(c2), c2, silentLogger()) if oerr != nil { return false } @@ -427,16 +416,17 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing require.NoError(t, err, "the chunk-2 tx hash must resolve from the live hot CF") assert.Equal(t, c2First, hotSeq, "hot lookup returns the live tx's ledger") require.NoError(t, liveDB.Close()) // release before the restart reopens it as the live writer + prunedIdxPath := postCat.Layout().TxHashIndexFilePath(frozenCov) // ===================================================================== - // STEP 4 — RESTART. A fresh runDaemonWith re-opens everything, re-derives the + // STEP 5 — RESTART. A fresh runDaemonWith re-opens everything, re-derives the // watermark from durable state, and resumes captive core at watermark+1 with no gap. // ===================================================================== closePost() // release the inspection metastore handle before the daemon reopens it core.opens.Store(0) core.resumeSeen.Store(0) core.fromSeen.Store(0) - cancel2, done2, _ := runDaemonInBackground(t, cfgPath, core, &served, &e2eMetrics{}) + cancel2, done2 := runDaemonInBackground(t, cfgPath, core, &served, &e2eMetrics{}) require.Eventually(t, func() bool { return core.opens.Load() >= 1 }, 30*time.Second, 20*time.Millisecond, "the restarted daemon re-opened captive core") @@ -452,31 +442,33 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing waitClean(t, cancel2, done2) // ===================================================================== - // STEP 5 — retention prune. Re-run with retention_chunks = 1: the floor anchors + // STEP 6 — retention prune. Re-run with retention_chunks = 1: the floor anchors // at chunk 1, so chunk 0 (frozen + folded) falls WHOLLY below it and the prune // scan sweeps its files + keys, while chunk 1 (the floor chunk) survives. A read // of a pruned chunk-0 hash is then not-found (no coverage to resolve it). // ===================================================================== prunedCfg := e2eConfigPath(t, dataDir, 1) // retain ~1 chunk - prunedIdxPath := cat.Layout().TxHashIndexFilePath(frozenCov) require.FileExists(t, prunedIdxPath, "chunk 0's cold index exists before the prune") - cancel3, done3, catCh3 := runDaemonInBackground(t, prunedCfg, core, &served, &e2eMetrics{}) - pruneCat := awaitCatalog(t, catCh3) // the pruning daemon's own catalog + pruneMetrics := &e2eMetrics{} + cancel3, done3 := runDaemonInBackground(t, prunedCfg, core, &served, pruneMetrics) // The prune scan runs on the first lifecycle tick (the at-start doorbell ring). - // Poll for chunk 0's per-chunk artifact keys (ledgers + events) to vanish. require.Eventually(t, func() bool { - ledgers, err := pruneCat.State(c0, geometry.KindLedgers) - if err != nil { - return false - } - ev, err := pruneCat.State(c0, geometry.KindEvents) - if err != nil { - return false - } - return ledgers == geometry.State("") && ev == geometry.State("") - }, 60*time.Second, 50*time.Millisecond, "retention must prune chunk 0's artifact keys") + return pruneMetrics.prunedCount() > 0 + }, 60*time.Second, 50*time.Millisecond, "retention prune scan must sweep chunk 0") + + waitClean(t, cancel3, done3) + pruneCat, closePrune := e2eReadCatalog(t, dataDir) + defer closePrune() + + // Chunk 0's per-chunk artifact keys (ledgers + events) vanished. + ledgers, err := pruneCat.State(c0, geometry.KindLedgers) + require.NoError(t, err) + ev, err := pruneCat.State(c0, geometry.KindEvents) + require.NoError(t, err) + assert.Equal(t, geometry.State(""), ledgers, "chunk 0 ledgers key is pruned") + assert.Equal(t, geometry.State(""), ev, "chunk 0 events key is pruned") // Chunk 1 (the floor chunk) is WITHIN retention and survives the prune. c1lfs, err := pruneCat.State(c1, geometry.KindLedgers) @@ -494,8 +486,6 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing _, covOK, err := pruneCat.FrozenTxHashIndex(w0) require.NoError(t, err) assert.False(t, covOK, "chunk 0's window coverage is pruned ⇒ a chunk-0 hash read is not-found") - - waitClean(t, cancel3, done3) } // e2eReadCatalog binds a Catalog over a SEPARATE metastore handle on the daemon's From ce675b7267173310cef4900dce6e817975d540ff Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 18:34:49 -0400 Subject: [PATCH 24/55] e2e_test: drop unused cyclop nolint directive The 'Remove live catalog test seam' rework simplified the E2E function below the cyclop threshold, so the cyclop suppression is now unused (nolintlint). --- cmd/stellar-rpc/internal/fullhistory/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go index 952ec7621..947f34d18 100644 --- a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go @@ -262,7 +262,7 @@ func hashAt(n uint64) [32]byte { // // Correctness is asserted at every step. // -//nolint:cyclop,funlen,maintidx // one linear end-to-end scenario asserted step by step +//nolint:funlen,maintidx // one linear end-to-end scenario asserted step by step func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing.T) { if testing.Short() { t.Skip("e2e ingests a full 10k-ledger chunk; skipped in -short") From 5a96d9ca9d8e4ab7bf7a0466fd837fd929e89fa3 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 21:45:35 -0400 Subject: [PATCH 25/55] fullhistory: drop orphaned self-committing hot writes + eventPayloads dup; fix ReadOnly WAL doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The RunHot deletion orphaned the four self-committing hot-store write methods (ledger.AddLedgers, txhash.AddEntries, eventstore.IngestLedgerEvents + IngestLedgerToBatchCommit): production writes exclusively through the *ToBatch variants inside hotchunk.DB.IngestLedger's one shared batch. Delete them; tests seed through small in-package batch+commit helpers, seedWatermark drives the production IngestLedger path, and the realdb watermark fixtures seed the ledgers CF via a raw reopen. IngestLedgerEvents' contract doc moves to IngestLedgerToBatch, the production entry point it always described. hotchunk.eventPayloads was a byte-for-byte copy of ingest.eventPayloads; the shared helper events.LCMViewToPayloads imports into hotchunk without the cycle the comment feared — inline it at the single call site. rocksdb.Config.ReadOnly's doc claimed an un-flushed WAL is NOT replayed on a read-only open. RocksDB's OpenForReadOnly recovers the WAL into in-memory memtables (verified: sync-write, exit without Close, read-only reopen sees the data), so freeze views observe every synced write. Correct the comment — the wrong claim mis-states what the freeze view can miss. --- .../internal/fullhistory/hotloop_test.go | 25 ++--- .../internal/fullhistory/ingest/events.go | 3 +- .../lifecycle/progress_realdb_test.go | 47 ++++++++-- .../fullhistory/pkg/rocksdb/rocksdb.go | 8 +- .../pkg/stores/eventstore/hot_store.go | 94 +++++-------------- .../pkg/stores/eventstore/hot_store_test.go | 77 +++++++++------ .../pkg/stores/eventstore/hot_warmup_test.go | 22 ++--- .../pkg/stores/eventstore/query_test.go | 14 +-- .../pkg/stores/eventstore/reader.go | 4 +- .../pkg/stores/hotchunk/hotchunk.go | 16 +--- .../pkg/stores/ledger/hot_store.go | 50 +--------- .../pkg/stores/ledger/hot_store_test.go | 49 ++++++---- .../pkg/stores/txhash/hot_store.go | 22 ----- .../pkg/stores/txhash/hot_store_test.go | 36 ++++--- .../pkg/stores/txhash/read_assembly_test.go | 2 +- 15 files changed, 207 insertions(+), 262 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index 5be69db56..37896e116 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -19,17 +19,8 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" ) -// ledgerEntry builds a ledgers-CF entry carrying a real zero-tx LCM for seq — -// the bytes the cold pipeline can later re-read if the chunk freezes from the -// hot DB. -func ledgerEntry(t *testing.T, seq uint32) ledger.Entry { - t.Helper() - return ledger.Entry{Seq: seq, Bytes: zeroTxLCMBytes(t, seq)} -} - // --------------------------------------------------------------------------- // fakeLedgerGetter — an injectable LedgerGetter the ingestion loop polls by // sequence (the design's indexed core.GetLedger(ctx, seq)). For seqs it has a @@ -101,19 +92,19 @@ func openLiveHotDB(t *testing.T, cat *catalog.Catalog, c chunk.ID) *hotchunk.DB // seedWatermark advances a chunk's hot DB to a last-committed ledger of seq so // the indexed poll resumes at seq+1, letting a boundary test drive the loop over -// only the last ledger or two of a chunk. The always-on events CF requires -// strict ledger contiguity from the chunk's first ledger, so it seeds a -// zero-event offset for every ledger up to seq; the ledgers CF only needs the -// watermark entry, since MaxCommittedSeq is its last key. The returned DB is the -// (re-opened, ready) live handle the loop then owns. Seeding a near-full chunk -// costs one synced commit per ledger, so its callers run t.Parallel(). +// only the last ledger or two of a chunk. It ingests a real zero-tx LCM for +// every ledger up to seq through the production IngestLedger path (the events +// CF requires strict ledger contiguity from the chunk's first ledger). The +// returned DB is the (re-opened, ready) live handle the loop then owns. Seeding +// a near-full chunk costs one synced commit per ledger, so its callers run +// t.Parallel(). func seedWatermark(t *testing.T, cat *catalog.Catalog, c chunk.ID, seq uint32) *hotchunk.DB { t.Helper() db := openLiveHotDB(t, cat, c) for s := c.FirstLedger(); s <= seq; s++ { - require.NoError(t, db.Events().IngestLedgerEvents(s, nil)) + _, err := db.IngestLedger(s, zeroTxLCMBytes(t, s)) + require.NoError(t, err) } - require.NoError(t, db.Ledgers().AddLedgers(ledgerEntry(t, seq))) require.NoError(t, db.Close()) reopened, err := openHotDBForChunk(cat, c, silentLogger()) require.NoError(t, err) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/events.go b/cmd/stellar-rpc/internal/fullhistory/ingest/events.go index e5df3ba17..f072e90fe 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/events.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/events.go @@ -124,7 +124,8 @@ func (e *eventsCold) Close() error { } // ingestSeq writes one ledger's events and returns the count written. The -// pre-Soroban (V0) policy lives in eventPayloads, shared with the hot tier. +// pre-Soroban (V0) policy lives in events.LCMViewToPayloads, shared with the +// hot tier. func (e *eventsCold) ingestSeq(seq uint32, lcm xdr.LedgerCloseMetaView) (int, error) { estart := time.Now() payloads, err := eventPayloads(seq, lcm) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go index 3056967bd..3fe7dce2b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go @@ -1,14 +1,44 @@ package lifecycle import ( + "slices" "testing" "github.com/stretchr/testify/require" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash" ) +// seedLedgersCF reopens a CLOSED chunk hot DB raw and commits sparse ledgers-CF +// entries in one batch via the production AddLedgerToBatch. These fixtures need +// arbitrary frontier heights without the events CF's contiguity requirement, so +// they write the one CF the watermark refinement reads (MaxCommittedSeq only +// looks at the ledgers CF's last key; the payload bytes are never decoded). +func seedLedgersCF(t *testing.T, cat *catalog.Catalog, c chunk.ID, entries ...ledger.Entry) { + t.Helper() + store, err := rocksdb.New(rocksdb.Config{ + Path: cat.Layout().HotChunkPath(c), + ColumnFamilies: slices.Concat([]string{ledger.LedgersCF}, eventstore.CFNames(), txhash.CFNames()), + Logger: silentLogger(), + }) + require.NoError(t, err) + h := ledger.NewWithStore(store, c) + require.NoError(t, store.Batch(func(b *rocksdb.BatchWriter) error { + for _, e := range entries { + if berr := h.AddLedgerToBatch(b, e); berr != nil { + return berr + } + } + return nil + })) + require.NoError(t, store.Close()) +} + // TestDeriveWatermark_RealHotDB_RefinementIsNotStale exercises the watermark // refinement against a REAL per-chunk hotchunk DB read through the production // rocksHotProbe — the path the fakeHotProbe table tests stub out. It proves the @@ -22,16 +52,17 @@ func TestDeriveWatermark_RealHotDB_RefinementIsNotStale(t *testing.T) { // Production bracket: creates the hot dir, opens the SINGLE shared multi-CF // DB, flips the hot key "ready". This is exactly what ingestion does. db := openLiveHotDB(t, cat, live) + // Close the live writer before seeding + the probe's read-only reopen + // (RocksDB LOCK). + require.NoError(t, db.Close()) // Commit two real ledgers into the ledgers CF (the CF MaxCommittedSeq reads). first := live.FirstLedger() committedTop := first + 200 - require.NoError(t, db.Ledgers().AddLedgers( + seedLedgersCF(t, cat, live, ledger.Entry{Seq: first, Bytes: []byte("ledger-A")}, ledger.Entry{Seq: committedTop, Bytes: []byte("ledger-B")}, - )) - // Close the live writer before the probe re-opens read-only (RocksDB LOCK). - require.NoError(t, db.Close()) + ) // Sanity: positional baseline (live chunk 5 ⇒ everything below 5) is chunk 4's // last ledger, strictly below the committed top — so the assertion below can @@ -62,15 +93,15 @@ func TestDeriveWatermark_RealHotDB_OpensHighestReady(t *testing.T) { // Lower ready chunk: a real DB committed near the TOP of chunk 4. If the // refinement wrongly opened the lower chunk, the bound would land here. lowDB := openLiveHotDB(t, cat, lower) - lowTop := lower.FirstLedger() + 9000 - require.NoError(t, lowDB.Ledgers().AddLedgers(ledger.Entry{Seq: lowTop, Bytes: []byte("low")})) require.NoError(t, lowDB.Close()) + lowTop := lower.FirstLedger() + 9000 + seedLedgersCF(t, cat, lower, ledger.Entry{Seq: lowTop, Bytes: []byte("low")}) // Higher ready chunk (the live chunk): committed mid-chunk 7. highDB := openLiveHotDB(t, cat, higher) - highMid := higher.FirstLedger() + 1234 - require.NoError(t, highDB.Ledgers().AddLedgers(ledger.Entry{Seq: highMid, Bytes: []byte("high")})) require.NoError(t, highDB.Close()) + highMid := higher.FirstLedger() + 1234 + seedLedgersCF(t, cat, higher, ledger.Entry{Seq: highMid, Bytes: []byte("high")}) // The two frontiers must be unambiguous: chunk 7 mid-seq is far above chunk 4's // top, so reading the wrong chunk yields a strictly different (lower) answer. diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go index b6af40547..107ad9bb1 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go @@ -60,8 +60,10 @@ type Config struct { PerCFOptions map[string]CFOptions // ReadOnly opens the store read-only (dir never created, no writes, no - // flush-on-close). Reads see durable SST/MANIFEST only — an un-flushed WAL is - // NOT replayed (a cleanly-closed DB has none). Used by the freeze source. + // flush-on-close). An un-flushed WAL IS recovered into in-memory memtables + // on open (RocksDB OpenForReadOnly semantics; nothing is persisted), so + // reads see every synced write, not just SST/MANIFEST state. Used by the + // freeze source. ReadOnly bool } @@ -574,7 +576,7 @@ func (s *Store) constructAndOpen() error { // WAL on + per-write Sync on — non-negotiable across every // fullhistory store, so pinned here on the shared wo rather // than exposed via Tuning. The streaming ingestion contract - // requires "AddEntries returned nil" to mean "durable on disk"; + // requires "the ledger batch committed" to mean "durable on disk"; // one fsync per Put/Batch regardless of size. s.wo.DisableWAL(false) s.wo.SetSync(true) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index b2471ed7a..721875745 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -92,12 +92,12 @@ const ( offsetValLen = 4 // per-ledger event count (uint32 BE) ) -// ErrLedgerOutOfRange is returned by IngestLedgerEvents when the +// ErrLedgerOutOfRange is returned by IngestLedgerToBatch when the // supplied ledger sequence falls outside the chunk's [FirstLedger, // LastLedger] window. var ErrLedgerOutOfRange = errors.New("events: ledger outside chunk range") -// ErrLedgerOutOfOrder is returned by IngestLedgerEvents when the +// ErrLedgerOutOfOrder is returned by IngestLedgerToBatch when the // supplied ledger sequence is not the next-expected one. Catches // duplicate ingest of an already-committed ledger as well as gaps // (skipping ahead). Both would silently corrupt the per-ledger @@ -107,13 +107,14 @@ var ErrLedgerOutOfOrder = errors.New("events: ledger out of order") // HotStore wraps one chunk's hot RocksDB DB plus the in-memory term mirror and // ledger-offset cache that feed the query path. // -// Atomicity: the per-Chunk DB is the source of truth. IngestLedgerEvents commits -// data + index + offsets in one atomic batch, then updates the in-memory -// mirrors; warmup reconstructs them from the on-disk CFs on next startup. +// Atomicity: the per-Chunk DB is the source of truth. IngestLedgerToBatch queues +// data + index + offsets into one atomic batch, then (post-commit) the apply +// hook updates the in-memory mirrors; warmup reconstructs them from the on-disk +// CFs on next startup. // // Concurrency: // -// - Writes (IngestLedgerEvents) are single-writer (one goroutine per chunk). +// - Writes (IngestLedgerToBatch) are single-writer (one goroutine per chunk). // - Reads (Lookup, FetchEvents, All) take NO HotStore-level lock — they guard // via chunkStore.IsClosed() and rely on the mirror's internal locks and // RocksDB's thread-safety. @@ -153,7 +154,7 @@ func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) (*HotStore, error) { func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } // EventCount is the total number of events committed to this Chunk -// so far. Equal to the next event-id IngestLedgerEvents would assign. +// so far. Equal to the next event-id IngestLedgerToBatch would assign. // Returns (0, ErrClosed) after the caller-owned store is closed. The Reader interface signature // is fallible to accommodate ColdReader's lazy metadata load; on the // hot side the value is always live and the error is only ErrClosed. @@ -164,7 +165,7 @@ func (h *HotStore) EventCount() (uint32, error) { return h.offsets.TotalEvents(), nil } -// NextEventID is the next chunk-relative event ID IngestLedgerEvents +// NextEventID is the next chunk-relative event ID IngestLedgerToBatch // will assign. Returns the same value as EventCount on the hot side // and is exposed under both names for the ingest-side and reader-side // mental models. Infallible at the type level (hot-only API, not on @@ -177,7 +178,7 @@ func (h *HotStore) NextEventID() uint32 { return h.offsets.TotalEvents() } // // Implementation: returns a *LedgerOffsets sharing the live // backing array, capped at the count visible at call time -// (~24-byte allocation per Query). Concurrent IngestLedgerEvents +// (~24-byte allocation per Query). A concurrent IngestLedgerToBatch // may extend the backing past the cap, but the returned view's // slice stays bounded to what was visible when Offsets returned. // Callers (Query) take the view once at entry and pass it through @@ -423,8 +424,11 @@ func (h *HotStore) All(ctx context.Context) iter.Seq2[events.Payload, error] { } } -// IngestLedgerEvents commits one ledger's events to the chunk store -// atomically and then updates the in-memory mirrors. +// IngestLedgerToBatch validates+marshals one ledger's events and queues their CF +// Puts into the SHARED batch b, returning the post-commit apply hook the caller +// runs AFTER b commits (decision (a)). Returns (nil, nil) for an idempotent +// duplicate. All validation + term derivation happen up front, so a rejected +// ledger leaves b untouched. // // payloads is produced by events.LCMViewToPayloads, which emits each ledger's // events in ascending getEvents cursor order — write order here IS the @@ -432,72 +436,24 @@ func (h *HotStore) All(ctx context.Context) iter.Seq2[events.Payload, error] { // derived internally via events.TermsForBytes on each payload's // ContractEventBytes. // -// Sequence validation is performed up front, before any RocksDB -// write or mirror mutation: +// Sequence validation is performed up front, before any queue or mirror +// mutation: // // - ledgerSeq must lie within [chunkID.FirstLedger(), // chunkID.LastLedger()] — out-of-range returns ErrLedgerOutOfRange. // - ledgerSeq == the next expected ledger (StartLedger + LedgerCount) // is appended normally. // - ledgerSeq < expected (an already-ingested ledger) is an idempotent -// no-op returning nil, so a restarted ingester can blindly re-deliver -// the in-flight ledger; the re-delivered events are not re-verified. +// no-op returning (nil, nil), so a restarted ingester can blindly +// re-deliver the in-flight ledger; the re-delivered events are not +// re-verified. // - ledgerSeq > expected (a gap) returns ErrLedgerOutOfOrder. // -// A rejected call (out-of-range or gap) completes its checks before -// marshaling, leaving the chunk store and in-memory mirrors untouched. -// -// Post-batch atomicity: once the RocksDB batch commits, the in-memory -// mirror + offsets updates are infallible by construction. Any -// failure there panics rather than returning an error, because a -// returned error would leave on-disk state ahead of in-memory state -// with no clean recovery short of close + reopen. -// -//nolint:cyclop // sequential pipeline: validate -> marshal -> batch -> mirror updates -func (h *HotStore) IngestLedgerEvents(ledgerSeq uint32, payloads []events.Payload) error { - if h.chunkStore.IsClosed() { - return ErrClosed - } - - // Same prepare → queue → commit → apply pipeline hotchunk drives across the - // shared DB; here the batch holds only the events CFs. - apply, err := h.IngestLedgerToBatchCommit(ledgerSeq, payloads) - if err != nil { - return err - } - if apply != nil { - apply() - } - return nil -} - -// IngestLedgerToBatchCommit is IngestLedgerEvents over a batch this facade owns -// end-to-end (validate → marshal → one synced batch). Returns the post-commit -// apply hook (mirror+offsets) to run after the batch is durable, or (nil, nil) -// for an idempotent duplicate. Split out so IngestLedgerToBatch can share the -// prepare step while committing into a SHARED cross-CF batch instead. -func (h *HotStore) IngestLedgerToBatchCommit(ledgerSeq uint32, payloads []events.Payload) (func(), error) { - prep, err := h.prepareLedger(ledgerSeq, payloads) - if err != nil { - return nil, err - } - if prep == nil { - //nolint:nilnil // (nil, nil) is the idempotent-duplicate signal; the caller runs the hook only when non-nil - return nil, nil - } - if cerr := h.chunkStore.Batch(func(b *rocksdb.BatchWriter) error { - return prep.queue(b) - }); cerr != nil { - return nil, fmt.Errorf("commit ledger %d to chunk %s: %w", ledgerSeq, h.chunkID, cerr) - } - return prep.apply, nil -} - -// IngestLedgerToBatch validates+marshals one ledger's events and queues their CF -// Puts into the SHARED batch b, returning the post-commit apply hook the caller -// runs AFTER b commits (decision (a)). Returns (nil, nil) for an idempotent -// duplicate. All validation + term derivation happen up front, so a rejected -// ledger leaves b untouched. +// Post-batch atomicity: once the batch commits, the apply hook's in-memory +// mirror + offsets updates are infallible by construction. Any failure there +// panics rather than returning an error, because a returned error would leave +// on-disk state ahead of in-memory state with no clean recovery short of +// close + reopen. func (h *HotStore) IngestLedgerToBatch( b *rocksdb.BatchWriter, ledgerSeq uint32, payloads []events.Payload, ) (func(), error) { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go index 821af210f..92954f160 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go @@ -150,7 +150,7 @@ func TestHotStore_IngestLedgerWritesAllCFs(t *testing.T) { h := openHotStoreForTest(t, chunkID) p, keys := makePayload("transfer") - require.NoError(t, h.store.IngestLedgerEvents(2, []events.Payload{p})) + require.NoError(t, ingestLedgerEvents(h.store, 2, []events.Payload{p})) // events_data row exists. got, found, err := h.store.chunkStore.Get(DataCF, encodeDataKey(0)) @@ -189,10 +189,10 @@ func TestHotStore_EventIDsAreMonotonic(t *testing.T) { p1, _ := makePayload("a") p2, _ := makePayload("b") - require.NoError(t, h.store.IngestLedgerEvents(first, []events.Payload{p1, p2})) + require.NoError(t, ingestLedgerEvents(h.store, first, []events.Payload{p1, p2})) p3, _ := makePayload("c") - require.NoError(t, h.store.IngestLedgerEvents(first+1, []events.Payload{p3})) + require.NoError(t, ingestLedgerEvents(h.store, first+1, []events.Payload{p3})) for id := range uint32(3) { _, found, err := h.store.chunkStore.Get(DataCF, encodeDataKey(id)) @@ -206,7 +206,7 @@ func TestHotStore_EmptyLedgerStillWritesOffsetsAndState(t *testing.T) { const chunkID = chunk.ID(0) h := openHotStoreForTest(t, chunkID) - require.NoError(t, h.store.IngestLedgerEvents(2, nil)) + require.NoError(t, ingestLedgerEvents(h.store, 2, nil)) val, found, err := h.store.chunkStore.Get(OffsetsCF, encodeOffsetKey(2)) require.NoError(t, err) @@ -229,7 +229,7 @@ func TestHotStore_LookupReturnsImmutableSnapshot(t *testing.T) { // Promote to dense mode so we exercise the bm.Load path (sparse // mode allocates a fresh bitmap per Get). for i := range uint32(70) { - require.NoError(t, h.store.IngestLedgerEvents(2+i, []events.Payload{p})) + require.NoError(t, ingestLedgerEvents(h.store, 2+i, []events.Payload{p})) } first, err := h.store.Lookup(context.Background(), keys[0]) @@ -238,7 +238,7 @@ func TestHotStore_LookupReturnsImmutableSnapshot(t *testing.T) { // New ingest publishes a new snapshot. The old pointer must // remain unchanged (it's the previous snapshot). - require.NoError(t, h.store.IngestLedgerEvents(72, []events.Payload{p})) + require.NoError(t, ingestLedgerEvents(h.store, 72, []events.Payload{p})) assert.Equal(t, cardBefore, first.GetCardinality(), "prior Lookup result must be an immutable snapshot — later IngestLedgerEvents must not mutate it") @@ -255,9 +255,9 @@ func TestHotStore_FetchEventsRoundTrip(t *testing.T) { p1, _ := makePayload("a") p2, _ := makePayload("b") - require.NoError(t, h.store.IngestLedgerEvents(2, []events.Payload{p1, p2})) + require.NoError(t, ingestLedgerEvents(h.store, 2, []events.Payload{p1, p2})) p3, _ := makePayload("c") - require.NoError(t, h.store.IngestLedgerEvents(3, []events.Payload{p3})) + require.NoError(t, ingestLedgerEvents(h.store, 3, []events.Payload{p3})) fetched, err := h.store.FetchEvents(context.Background(), []uint32{0, 1, 2}) require.NoError(t, err) @@ -271,7 +271,7 @@ func TestHotStore_FetchEventsErrorsOnMissingID(t *testing.T) { const chunkID = chunk.ID(0) h := openHotStoreForTest(t, chunkID) p, _ := makePayload("only") - require.NoError(t, h.store.IngestLedgerEvents(2, []events.Payload{p})) + require.NoError(t, ingestLedgerEvents(h.store, 2, []events.Payload{p})) _, err := h.store.FetchEvents(context.Background(), []uint32{99}) assert.Error(t, err) @@ -292,7 +292,7 @@ func TestHotStore_FetchEventsLargeBatch(t *testing.T) { p, _ := makePayload(fmt.Sprintf("evt-%03d", i)) payloads[i] = p } - require.NoError(t, h.store.IngestLedgerEvents(2, payloads)) + require.NoError(t, ingestLedgerEvents(h.store, 2, payloads)) ids := make([]uint32, n) for i := range n { @@ -317,7 +317,7 @@ func TestHotStore_FetchEventsHonorsContext(t *testing.T) { const chunkID = chunk.ID(0) h := openHotStoreForTest(t, chunkID) p, _ := makePayload("only") - require.NoError(t, h.store.IngestLedgerEvents(2, []events.Payload{p})) + require.NoError(t, ingestLedgerEvents(h.store, 2, []events.Payload{p})) ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -340,7 +340,7 @@ func TestHotStore_FetchEventsRejectsUnsortedInput(t *testing.T) { p2.LedgerSequence = 2 p3, _ := makePayload("c") p3.LedgerSequence = 2 - require.NoError(t, h.store.IngestLedgerEvents(2, []events.Payload{p1, p2, p3})) + require.NoError(t, ingestLedgerEvents(h.store, 2, []events.Payload{p1, p2, p3})) _, err := h.store.FetchEvents(context.Background(), []uint32{2, 0}) require.ErrorIs(t, err, ErrUnsortedEventIDs, "out-of-order input must error") @@ -356,10 +356,10 @@ func TestHotStore_AllStreamsInEventIDOrder(t *testing.T) { p1.LedgerSequence = 2 p2, _ := makePayload("b") p2.LedgerSequence = 2 - require.NoError(t, h.store.IngestLedgerEvents(2, []events.Payload{p1, p2})) + require.NoError(t, ingestLedgerEvents(h.store, 2, []events.Payload{p1, p2})) p3, _ := makePayload("c") p3.LedgerSequence = 3 - require.NoError(t, h.store.IngestLedgerEvents(3, []events.Payload{p3})) + require.NoError(t, ingestLedgerEvents(h.store, 3, []events.Payload{p3})) got := make([]string, 0, 3) gotLedgers := make([]uint32, 0, 3) @@ -385,7 +385,7 @@ func TestHotStore_AllEmptyChunkYieldsNothing(t *testing.T) { func TestHotStore_CloseRejectsWrites(t *testing.T) { h := openHotStoreForTest(t, 0) require.NoError(t, h.raw.Close()) - err := h.store.IngestLedgerEvents(2, nil) + err := ingestLedgerEvents(h.store, 2, nil) assert.ErrorIs(t, err, ErrClosed) } @@ -399,7 +399,7 @@ func TestHotStore_PostCloseReadsError(t *testing.T) { h := openHotStoreForTest(t, chunkID) p, keys := makePayload("seed") - require.NoError(t, h.store.IngestLedgerEvents(chunkID.FirstLedger(), []events.Payload{p})) + require.NoError(t, ingestLedgerEvents(h.store, chunkID.FirstLedger(), []events.Payload{p})) require.NoError(t, h.raw.Close()) // Lookup must error rather than silently returning the cached bitmap. @@ -435,14 +435,14 @@ func TestHotStore_IngestLedgerEvents_DuplicateLedgerIsNoOp(t *testing.T) { first := chunkID.FirstLedger() p1, _ := makePayload("a") - require.NoError(t, h.store.IngestLedgerEvents(first, []events.Payload{p1})) + require.NoError(t, ingestLedgerEvents(h.store, first, []events.Payload{p1})) countBefore := mustEventCount(t, h.store) nextBefore := h.store.NextEventID() // Re-ingesting the same ledger is an idempotent no-op. p2, _ := makePayload("b") - require.NoError(t, h.store.IngestLedgerEvents(first, []events.Payload{p2})) + require.NoError(t, ingestLedgerEvents(h.store, first, []events.Payload{p2})) assert.Equal(t, countBefore, mustEventCount(t, h.store), "EventCount must not advance on duplicate ingest") assert.Equal(t, nextBefore, h.store.NextEventID(), "NextEventID must not advance on duplicate ingest") @@ -473,14 +473,14 @@ func TestHotStore_IngestLedgerEvents_RejectsLedgerGap(t *testing.T) { first := chunkID.FirstLedger() p1, _ := makePayload("a") - require.NoError(t, h.store.IngestLedgerEvents(first, []events.Payload{p1})) + require.NoError(t, ingestLedgerEvents(h.store, first, []events.Payload{p1})) countBefore := mustEventCount(t, h.store) nextBefore := h.store.NextEventID() // Skip first+1; jump directly to first+2. p2, _ := makePayload("c") - err := h.store.IngestLedgerEvents(first+2, []events.Payload{p2}) + err := ingestLedgerEvents(h.store, first+2, []events.Payload{p2}) require.ErrorIs(t, err, ErrLedgerOutOfOrder) assert.Equal(t, countBefore, mustEventCount(t, h.store)) @@ -496,11 +496,11 @@ func TestHotStore_IngestLedgerEvents_RejectsOutOfRangeLedger(t *testing.T) { p, _ := makePayload("a") // Below range (chunk 0's FirstLedger is FirstLedgerSeq == 2). - err := h.store.IngestLedgerEvents(1, []events.Payload{p}) + err := ingestLedgerEvents(h.store, 1, []events.Payload{p}) require.ErrorIs(t, err, ErrLedgerOutOfRange, "ledger below chunk range") // Above range — well past chunk 0's LastLedger. - err = h.store.IngestLedgerEvents(chunkID.LastLedger()+1, []events.Payload{p}) + err = ingestLedgerEvents(h.store, chunkID.LastLedger()+1, []events.Payload{p}) require.ErrorIs(t, err, ErrLedgerOutOfRange, "ledger above chunk range") // State must be unchanged after both rejections. @@ -523,7 +523,7 @@ func TestHotStore_ReopenRecoversState(t *testing.T) { hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("before") - require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1})) + require.NoError(t, ingestLedgerEvents(hot1, 2, []events.Payload{p1})) require.NoError(t, raw1.Close()) hot2, _ := openHotStoreForTestAt(t, dir, chunkID) @@ -531,7 +531,7 @@ func TestHotStore_ReopenRecoversState(t *testing.T) { assert.Equal(t, uint32(1), hot2.NextEventID(), "warmup recovered offsets") p2, _ := makePayload("after") - require.NoError(t, hot2.IngestLedgerEvents(3, []events.Payload{p2})) + require.NoError(t, ingestLedgerEvents(hot2, 3, []events.Payload{p2})) assert.Equal(t, uint32(2), hot2.NextEventID()) } @@ -559,7 +559,7 @@ func TestHotStore_ConcurrentIngestAndLookup(t *testing.T) { go func() { defer wg.Done() for i := range uint32(N) { - if err := h.store.IngestLedgerEvents(2+i, []events.Payload{p}); err != nil { + if err := ingestLedgerEvents(h.store, 2+i, []events.Payload{p}); err != nil { t.Errorf("ingest %d: %v", i, err) return } @@ -625,7 +625,7 @@ func TestHotStore_FetchRangeMidRange(t *testing.T) { p, _ := makePayload(fmt.Sprintf("evt-%d", i)) payloads[i] = p } - require.NoError(t, h.store.IngestLedgerEvents(first, payloads)) + require.NoError(t, ingestLedgerEvents(h.store, first, payloads)) got, err := fetchRangePayloads(t, h.store, 1, 3) require.NoError(t, err) @@ -647,7 +647,7 @@ func TestHotStore_FetchRangeOutOfBoundsErrors(t *testing.T) { const chunkID = chunk.ID(0) h := openHotStoreForTest(t, chunkID) p, _ := makePayload("only") - require.NoError(t, h.store.IngestLedgerEvents(chunkID.FirstLedger(), []events.Payload{p})) + require.NoError(t, ingestLedgerEvents(h.store, chunkID.FirstLedger(), []events.Payload{p})) _, err := fetchRangePayloads(t, h.store, 0, 2) // count > EventCount require.Error(t, err) @@ -672,7 +672,7 @@ func TestHotStore_AllMatchesFetchRange(t *testing.T) { p, _ := makePayload(fmt.Sprintf("e%d", i)) payloads[i] = p } - require.NoError(t, h.store.IngestLedgerEvents(first, payloads)) + require.NoError(t, ingestLedgerEvents(h.store, first, payloads)) allSyms := make([]string, 0, len(payloads)) for p, err := range h.store.All(context.Background()) { @@ -706,3 +706,24 @@ func mustOffsets(t *testing.T, r Reader) *events.LedgerOffsets { require.NotNil(t, o) return o } + +// ingestLedgerEvents commits one ledger's events through IngestLedgerToBatch in +// a test-owned batch and runs the post-commit apply hook — the production +// write shape, reduced to a test seeding call. +func ingestLedgerEvents(h *HotStore, ledgerSeq uint32, payloads []events.Payload) error { + if h.chunkStore.IsClosed() { + return ErrClosed + } + var apply func() + if err := h.chunkStore.Batch(func(b *rocksdb.BatchWriter) error { + a, aerr := h.IngestLedgerToBatch(b, ledgerSeq, payloads) + apply = a + return aerr + }); err != nil { + return err + } + if apply != nil { + apply() + } + return nil +} diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_warmup_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_warmup_test.go index 43d5bff41..5dd71349e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_warmup_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_warmup_test.go @@ -40,7 +40,7 @@ func TestWarmup_RebuildsMirrorFromIngestedRows(t *testing.T) { hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("alpha") p2, _ := makePayload("beta") - require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1, p2})) + require.NoError(t, ingestLedgerEvents(hot1, 2, []events.Payload{p1, p2})) // Snapshot the mirror state before close. Snapshot returns a // uniquely-owned Bitmaps the test can iterate freely. @@ -68,7 +68,7 @@ func TestWarmup_RestoresEventIDsForRepeatedTerm(t *testing.T) { p1, _ := makePayload("shared") p2, _ := makePayload("shared") p3, _ := makePayload("shared") - require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1, p2, p3})) + require.NoError(t, ingestLedgerEvents(hot1, 2, []events.Payload{p1, p2, p3})) require.NoError(t, raw1.Close()) hot2, _ := openHotStoreForTestAt(t, dir, chunkID) @@ -90,9 +90,9 @@ func TestWarmup_OffsetsReconstructedAcrossLedgers(t *testing.T) { hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("a") p2, _ := makePayload("b") - require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1, p2})) + require.NoError(t, ingestLedgerEvents(hot1, 2, []events.Payload{p1, p2})) p3, _ := makePayload("c") - require.NoError(t, hot1.IngestLedgerEvents(3, []events.Payload{p3})) + require.NoError(t, ingestLedgerEvents(hot1, 3, []events.Payload{p3})) require.NoError(t, raw1.Close()) hot2, _ := openHotStoreForTestAt(t, dir, chunkID) @@ -130,7 +130,7 @@ func TestWarmup_RejectsDataEventBeyondOffsets(t *testing.T) { hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("a") p2, _ := makePayload("b") - require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1, p2})) // total = 2 + require.NoError(t, ingestLedgerEvents(hot1, 2, []events.Payload{p1, p2})) // total = 2 require.NoError(t, raw1.Close()) // An orphan data row well beyond total (id 7, total = 2): proves the @@ -152,7 +152,7 @@ func TestWarmup_RejectsOffsetsGap(t *testing.T) { hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) for _, seq := range []uint32{2, 3, 4} { p, _ := makePayload("x") - require.NoError(t, hot1.IngestLedgerEvents(seq, []events.Payload{p})) + require.NoError(t, ingestLedgerEvents(hot1, seq, []events.Payload{p})) } require.NoError(t, raw1.Close()) @@ -174,7 +174,7 @@ func TestWarmup_RejectsOffsetsOverflow(t *testing.T) { hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) for _, seq := range []uint32{2, 3} { p, _ := makePayload("x") - require.NoError(t, hot1.IngestLedgerEvents(seq, []events.Payload{p})) + require.NoError(t, ingestLedgerEvents(hot1, seq, []events.Payload{p})) } require.NoError(t, raw1.Close()) @@ -213,7 +213,7 @@ func TestWarmup_RejectsMissingTailDataEvent(t *testing.T) { hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("a") p2, _ := makePayload("b") - require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1, p2})) // total = 2 + require.NoError(t, ingestLedgerEvents(hot1, 2, []events.Payload{p1, p2})) // total = 2 require.NoError(t, raw1.Close()) // Drop the last data row (event id total-1 == 1) while offsets still @@ -233,7 +233,7 @@ func TestWarmup_RejectsIndexBeyondCommitted(t *testing.T) { hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p1, _ := makePayload("a") p2, _ := makePayload("b") - require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p1, p2})) // total = 2 + require.NoError(t, ingestLedgerEvents(hot1, 2, []events.Payload{p1, p2})) // total = 2 require.NoError(t, raw1.Close()) // An index row at exactly total (id 2): the tightest "beyond @@ -254,8 +254,8 @@ func TestWarmup_OffsetsHandleEmptyTrailingLedger(t *testing.T) { hot1, raw1 := openHotStoreForTestAt(t, dir, chunkID) p, _ := makePayload("only") - require.NoError(t, hot1.IngestLedgerEvents(2, []events.Payload{p})) - require.NoError(t, hot1.IngestLedgerEvents(3, nil)) + require.NoError(t, ingestLedgerEvents(hot1, 2, []events.Payload{p})) + require.NoError(t, ingestLedgerEvents(hot1, 3, nil)) require.NoError(t, raw1.Close()) hot2, _ := openHotStoreForTestAt(t, dir, chunkID) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/query_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/query_test.go index 88b48c48a..706639e56 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/query_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/query_test.go @@ -104,7 +104,7 @@ func newQueryFixture(t *testing.T) *queryFixture { require.NoError(t, err) first := chunkID.FirstLedger() - require.NoError(t, fx.store.IngestLedgerEvents(first, []events.Payload{ + require.NoError(t, ingestLedgerEvents(fx.store, first, []events.Payload{ payloadFor(t, fx.contractA, "evt-a-ab", fx.t0a, fx.t0b), payloadFor(t, fx.contractA, "evt-a-ac", fx.t0a, fx.t0c), payloadFor(t, fx.contractB, "evt-b-ab", fx.t0a, fx.t0b), @@ -354,7 +354,7 @@ func TestQuery_ManyFiltersAtCallerCap(t *testing.T) { contracts[i][0] = byte(i + 1) payloads[i] = payloadFor(t, contracts[i], fmt.Sprintf("evt-%02d", i)) } - require.NoError(t, h.store.IngestLedgerEvents(first, payloads)) + require.NoError(t, ingestLedgerEvents(h.store, first, payloads)) filters := make([]Filter, n) for i := range n { @@ -377,7 +377,7 @@ func newMultiLedgerQueryFixture(t *testing.T) *queryFixture { t.Helper() fx := newQueryFixture(t) first := chunk.ID(0).FirstLedger() - require.NoError(t, fx.store.IngestLedgerEvents(first+1, []events.Payload{ + require.NoError(t, ingestLedgerEvents(fx.store, first+1, []events.Payload{ payloadFor(t, fx.contractA, "evt-extra-0", fx.t0a), payloadFor(t, fx.contractA, "evt-extra-1", fx.t0a), })) @@ -609,7 +609,7 @@ func TestQuery_ChunkWithLedgersButZeroEvents(t *testing.T) { // Ingest three empty ledgers — recorded in offsets, no events. for i := range uint32(3) { - require.NoError(t, h.store.IngestLedgerEvents(first+i, nil)) + require.NoError(t, ingestLedgerEvents(h.store, first+i, nil)) } require.Equal(t, uint32(0), mustEventCount(t, h.store)) @@ -704,8 +704,8 @@ func TestQuery_EmptyLeadingLedgerRangeStaysEmpty(t *testing.T) { // real events. After ingest the chunk's offsets read: // [first] → [0, 0) (empty) // [first+1] → [0, 5) (5 events) - require.NoError(t, h.store.IngestLedgerEvents(first, nil)) - require.NoError(t, h.store.IngestLedgerEvents(first+1, []events.Payload{ + require.NoError(t, ingestLedgerEvents(h.store, first, nil)) + require.NoError(t, ingestLedgerEvents(h.store, first+1, []events.Payload{ makeSimplePayload(t, "evt-0"), makeSimplePayload(t, "evt-1"), makeSimplePayload(t, "evt-2"), @@ -804,7 +804,7 @@ func makeSimplePayload(t *testing.T, dataSymbol string) events.Payload { // Walks the hot store one ledger at a time using its Offsets snapshot // (which tracks the ingest-time ledger sequence) rather than reading // LedgerSequence off each Payload — the test fixture's payloadFor -// builder doesn't set Payload.LedgerSequence, and HotStore.IngestLedgerEvents +// builder doesn't set Payload.LedgerSequence, and IngestLedgerToBatch // stores them verbatim, so the per-event field is the zero value and // can't be used to recover ledger boundaries. // diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/reader.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/reader.go index aa46ebeec..41b5ad63e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/reader.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/reader.go @@ -83,8 +83,8 @@ type Reader interface { // // Implementations: // - HotStore allocates a fresh Snapshot from the live - // ConcurrentLedgerOffsets per call. Concurrent - // IngestLedgerEvents may extend the underlying state after + // ConcurrentLedgerOffsets per call. A concurrent + // IngestLedgerToBatch may extend the underlying state after // Offsets returns, but the returned snapshot reflects what // was visible at call time. Callers (Query) take the // snapshot once at entry and pass it through their helpers. diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index 739354402..70b4228ea 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -160,9 +160,10 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts } counts.Txhash = len(hashes) - payloads, err := eventPayloads(seq, lcm) + // A pre-Soroban ledger yields zero payloads, no error. + payloads, err := events.LCMViewToPayloads(lcm) if err != nil { - return counts, err + return counts, fmt.Errorf("LCMViewToPayloads seq %d: %w", seq, err) } counts.Events = len(payloads) counts.Ledgers = 1 @@ -197,14 +198,3 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts } return counts, nil } - -// eventPayloads derives one ledger's event payloads from the view (a pre-Soroban -// ledger yields zero, no error). Duplicated from ingest.eventPayloads rather than -// imported — ingest will depend on hotchunk, so importing it would cycle. -func eventPayloads(seq uint32, lcm xdr.LedgerCloseMetaView) ([]events.Payload, error) { - payloads, err := events.LCMViewToPayloads(lcm) - if err != nil { - return nil, fmt.Errorf("LCMViewToPayloads seq %d: %w", seq, err) - } - return payloads, nil -} diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go index 8846759bb..a0d243f19 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go @@ -41,8 +41,8 @@ type HotStore struct { store *rocksdb.Store chunkID chunk.ID dec *zstd.Decompressor - // compPool — per-store pool of zstd.Compressors; each concurrent AddLedgers - // borrows one for its Encode call. + // compPool — per-store pool of zstd.Compressors; each concurrent + // AddLedgerToBatch borrows one for its Encode call. compPool sync.Pool } @@ -65,52 +65,6 @@ func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) *HotStore { // never reads the store). func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } -// AddLedgers writes (seq, raw-bytes) entries to rocksdb. Bytes is -// the uncompressed ledger payload; AddLedgers compresses each -// entry with zstd before write. Variadic so callers can pass -// individual entries (h.AddLedgers(e)), a literal batch -// (h.AddLedgers(e1, e2, e3)), or a slice (h.AddLedgers(entries...)). -// Zero entries is a no-op; one entry uses Store.Put; multiple -// entries use Store.Batch (one atomic write, one fsync — versus N -// fsyncs for N Put calls). -func (h *HotStore) AddLedgers(entries ...Entry) error { - if h.store.IsClosed() { - return stores.ErrStoreClosed - } - if len(entries) == 0 { - return nil - } - c, _ := h.compPool.Get().(*zstd.Compressor) - defer h.compPool.Put(c) - - if len(entries) == 1 { - e := entries[0] - compressed, err := c.Encode(nil, e.Bytes) - if err != nil { - return err - } - return translateRocksErr(h.store.Put(LedgersCF, rocksdb.EncodeUint32(e.Seq), compressed)) - } - // Multi-entry path: compress each into its own fresh slice so - // the batch can hold them all simultaneously (the compressor's - // internal buffer would otherwise be overwritten on the next - // Encode call). - compressed := make([][]byte, len(entries)) - for i, e := range entries { - out, err := c.Encode(nil, e.Bytes) - if err != nil { - return err - } - compressed[i] = out - } - return translateRocksErr(h.store.Batch(func(b *rocksdb.BatchWriter) error { - for i, e := range entries { - b.Put(LedgersCF, rocksdb.EncodeUint32(e.Seq), compressed[i]) - } - return nil - })) -} - // AddLedgerToBatch compresses one ledger and queues its Put into b on LedgersCF // — the building block hotchunk uses to fold the ledger write into the one // shared per-ledger WriteBatch (decision (a)). Does not commit (caller owns the diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go index 52353b939..26d491630 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go @@ -62,20 +62,20 @@ func TestHotStore_AddGetRoundTripVerbatim(t *testing.T) { // Single-entry write. payload := []byte("arbitrary opaque bytes the store has no opinion about") - require.NoError(t, h.AddLedgers(Entry{Seq: 42, Bytes: payload})) + require.NoError(t, addLedgers(h, Entry{Seq: 42, Bytes: payload})) got, err := h.GetLedgerRaw(42) require.NoError(t, err) assert.Equal(t, payload, got) // Overwrite. updated := []byte("different bytes") - require.NoError(t, h.AddLedgers(Entry{Seq: 42, Bytes: updated})) + require.NoError(t, addLedgers(h, Entry{Seq: 42, Bytes: updated})) got, err = h.GetLedgerRaw(42) require.NoError(t, err) assert.Equal(t, updated, got) // Zero entries — no-op, no error. - require.NoError(t, h.AddLedgers()) + require.NoError(t, addLedgers(h)) } // TestHotStore_AddLedgersIdempotentRetry mirrors the events store's retry @@ -88,8 +88,8 @@ func TestHotStore_AddLedgersIdempotentRetry(t *testing.T) { h := openTestHotStore(t) payload := []byte("ledger payload") - require.NoError(t, h.AddLedgers(Entry{Seq: 7, Bytes: payload})) - require.NoError(t, h.AddLedgers(Entry{Seq: 7, Bytes: payload})) // retry + require.NoError(t, addLedgers(h, Entry{Seq: 7, Bytes: payload})) + require.NoError(t, addLedgers(h, Entry{Seq: 7, Bytes: payload})) // retry got, err := h.GetLedgerRaw(7) require.NoError(t, err) @@ -118,7 +118,7 @@ func TestHotStore_FirstLastSeq(t *testing.T) { require.False(t, ok) // Insert seqs out of order; FirstSeq/LastSeq report the min/max present. - require.NoError(t, h.AddLedgers( + require.NoError(t, addLedgers(h, Entry{Seq: 105, Bytes: []byte("c")}, Entry{Seq: 100, Bytes: []byte("a")}, Entry{Seq: 103, Bytes: []byte("b")}, @@ -142,7 +142,7 @@ func TestHotStore_AddLedgersMultipleEntries(t *testing.T) { {Seq: 101, Bytes: []byte("ledger 101 payload")}, {Seq: 102, Bytes: []byte("ledger 102 payload")}, } - require.NoError(t, h.AddLedgers(entries...)) + require.NoError(t, addLedgers(h, entries...)) for _, e := range entries { got, err := h.GetLedgerRaw(e.Seq) require.NoError(t, err) @@ -153,7 +153,7 @@ func TestHotStore_AddLedgersMultipleEntries(t *testing.T) { func TestHotStore_IterateLedgers(t *testing.T) { h := openTestHotStore(t) for _, seq := range []uint32{10, 20, 30, 40, 50} { - require.NoError(t, h.AddLedgers(Entry{Seq: seq, Bytes: []byte("v")})) + require.NoError(t, addLedgers(h, Entry{Seq: seq, Bytes: []byte("v")})) } // Full window. @@ -204,7 +204,7 @@ func TestHotStore_IterateLedgersVisibleGap(t *testing.T) { h := openTestHotStore(t) // Non-contiguous keyspace: missing 30. for _, seq := range []uint32{10, 20, 40, 50} { - require.NoError(t, h.AddLedgers(Entry{Seq: seq, Bytes: []byte("v")})) + require.NoError(t, addLedgers(h, Entry{Seq: seq, Bytes: []byte("v")})) } var seen []uint32 @@ -225,7 +225,7 @@ func TestHotStore_GracefulCloseAndReopen(t *testing.T) { } first, firstStore := openTestHotStoreAt(t, path, chunk.ID(0)) - require.NoError(t, first.AddLedgers(seeded...)) + require.NoError(t, addLedgers(first, seeded...)) require.NoError(t, firstStore.Close()) second, _ := openTestHotStoreAt(t, path, chunk.ID(0)) @@ -241,7 +241,7 @@ func TestHotStore_PostCloseOps(t *testing.T) { h, store := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) require.NoError(t, store.Close()) - require.ErrorIs(t, h.AddLedgers(Entry{Seq: 1, Bytes: []byte("v")}), stores.ErrStoreClosed) + require.ErrorIs(t, addLedgers(h, Entry{Seq: 1, Bytes: []byte("v")}), stores.ErrStoreClosed) _, err := h.GetLedgerRaw(1) require.ErrorIs(t, err, stores.ErrStoreClosed) var iterErr error @@ -250,7 +250,7 @@ func TestHotStore_PostCloseOps(t *testing.T) { } require.ErrorIs(t, iterErr, stores.ErrStoreClosed) - require.ErrorIs(t, h.AddLedgers(), stores.ErrStoreClosed) + require.ErrorIs(t, addLedgers(h), stores.ErrStoreClosed) iterErr = nil for _, e := range h.IterateLedgers(100, 50) { @@ -262,7 +262,7 @@ func TestHotStore_PostCloseOps(t *testing.T) { func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { h, store := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) for i := range uint32(50) { - require.NoError(t, h.AddLedgers(Entry{Seq: i, Bytes: []byte("v")})) + require.NoError(t, addLedgers(h, Entry{Seq: i, Bytes: []byte("v")})) } var wg sync.WaitGroup @@ -271,7 +271,7 @@ func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { for w := range workers { wg.Go(func() { for i := uint32(0); !stop.Load(); i++ { - _ = h.AddLedgers(Entry{Seq: uint32(w)*1_000_000 + i, Bytes: []byte("v")}) + _ = addLedgers(h, Entry{Seq: uint32(w)*1_000_000 + i, Bytes: []byte("v")}) } }) wg.Go(func() { @@ -295,7 +295,7 @@ func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { stop.Store(true) wg.Wait() - require.ErrorIs(t, h.AddLedgers(Entry{Seq: 1, Bytes: []byte("v")}), stores.ErrStoreClosed) + require.ErrorIs(t, addLedgers(h, Entry{Seq: 1, Bytes: []byte("v")}), stores.ErrStoreClosed) } // TestHotStore_AddLedgersEmptyBytes pins behavior on zero-length @@ -303,7 +303,7 @@ func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { // and read back as empty. func TestHotStore_AddLedgersEmptyBytes(t *testing.T) { h := openTestHotStore(t) - require.NoError(t, h.AddLedgers(Entry{Seq: 1, Bytes: nil})) + require.NoError(t, addLedgers(h, Entry{Seq: 1, Bytes: nil})) got, err := h.GetLedgerRaw(1) require.NoError(t, err) assert.Empty(t, got) @@ -326,7 +326,7 @@ func TestHotToColdMigration(t *testing.T) { b, err := lcm.MarshalBinary() require.NoError(t, err) raws[i] = b - require.NoError(t, hot.AddLedgers(Entry{Seq: firstSeq + uint32(i), Bytes: b})) + require.NoError(t, addLedgers(hot, Entry{Seq: firstSeq + uint32(i), Bytes: b})) } // Stream hot → cold. No re-encoding step on the caller side. @@ -361,7 +361,7 @@ func TestHotStore_XDRRoundTrip(t *testing.T) { require.NoError(t, err) h := openTestHotStore(t) - require.NoError(t, h.AddLedgers(Entry{Seq: ledgerSeq, Bytes: raw})) + require.NoError(t, addLedgers(h, Entry{Seq: ledgerSeq, Bytes: raw})) gotRaw, err := h.GetLedgerRaw(ledgerSeq) require.NoError(t, err) @@ -457,3 +457,16 @@ func makeRandomLedgerCloseMeta( lcm.V1.LedgerHeader.Header.LedgerSeq = xdr.Uint32(ledgerSeq) return lcm, hashes } + +// addLedgers commits entries through AddLedgerToBatch in one batch — the +// production write shape, reduced to a test seeding call. +func addLedgers(h *HotStore, entries ...Entry) error { + return translateRocksErr(h.store.Batch(func(b *rocksdb.BatchWriter) error { + for _, e := range entries { + if err := h.AddLedgerToBatch(b, e); err != nil { + return err + } + } + return nil + })) +} diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go index f7b71313f..7871e0015 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go @@ -107,28 +107,6 @@ func Tuning() rocksdb.Tuning { // never reads the store). func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } -// AddEntries writes a batch of (txhash → ledgerSeq) atomically to the -// txhash CF. One fsync per call. -func (h *HotStore) AddEntries(entries []Entry) error { - if h.store.IsClosed() { - return rocksdb.ErrStoreClosed - } - switch len(entries) { - case 0: - return nil - case 1: - e := entries[0] - return h.store.Put(txhashCF, e.Hash[:], rocksdb.EncodeUint32(e.LedgerSeq)) - default: - return h.store.Batch(func(b *rocksdb.BatchWriter) error { - for _, e := range entries { - b.Put(txhashCF, e.Hash[:], rocksdb.EncodeUint32(e.LedgerSeq)) - } - return nil - }) - } -} - // AddEntriesToBatch queues each (txhash → ledgerSeq) Put into b on the txhash // CF — the building block hotchunk uses to fold the tx-hash writes into the one // shared per-ledger WriteBatch (decision (a)). Does not commit (caller owns the diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go index b0c6ead16..4c63a16cd 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go @@ -72,20 +72,20 @@ func TestHotStore_AddGetRoundTrip(t *testing.T) { require.ErrorIs(t, err, stores.ErrNotFound) // Single-entry AddEntries. - require.NoError(t, s.AddEntries([]Entry{{Hash: h, LedgerSeq: 12345}})) + require.NoError(t, addEntries(s, []Entry{{Hash: h, LedgerSeq: 12345}})) got, err := s.Get(h) require.NoError(t, err) assert.Equal(t, uint32(12345), got) // Overwrite via a second AddEntries. - require.NoError(t, s.AddEntries([]Entry{{Hash: h, LedgerSeq: 67890}})) + require.NoError(t, addEntries(s, []Entry{{Hash: h, LedgerSeq: 67890}})) got, err = s.Get(h) require.NoError(t, err) assert.Equal(t, uint32(67890), got) // Empty slice — no-op, no error. - require.NoError(t, s.AddEntries(nil)) - require.NoError(t, s.AddEntries([]Entry{})) + require.NoError(t, addEntries(s, nil)) + require.NoError(t, addEntries(s, []Entry{})) } func TestHotStore_ManyDistinctKeys(t *testing.T) { @@ -99,7 +99,7 @@ func TestHotStore_ManyDistinctKeys(t *testing.T) { LedgerSeq: uint32(i) * 100, } } - require.NoError(t, s.AddEntries(entries)) + require.NoError(t, addEntries(s, entries)) for i := range n { got, err := s.Get(entries[i].Hash) @@ -118,7 +118,7 @@ func TestHotStore_AddEntriesMultiple(t *testing.T) { {Hash: txhashFor(0xc, 1), LedgerSeq: 40}, {Hash: txhashFor(0xf, 1), LedgerSeq: 50}, } - require.NoError(t, s.AddEntries(entries)) + require.NoError(t, addEntries(s, entries)) for _, e := range entries { got, err := s.Get(e.Hash) @@ -131,7 +131,7 @@ func TestHotStore_AddEntriesMultiple(t *testing.T) { for i, e := range entries { updated[i] = Entry{Hash: e.Hash, LedgerSeq: e.LedgerSeq + 1000} } - require.NoError(t, s.AddEntries(updated)) + require.NoError(t, addEntries(s, updated)) for _, e := range updated { got, err := s.Get(e.Hash) require.NoError(t, err) @@ -144,12 +144,12 @@ func TestHotStore_PostCloseOps(t *testing.T) { require.NoError(t, store.Close()) h := txhashFor(0x5, 1) - require.ErrorIs(t, s.AddEntries([]Entry{{Hash: h, LedgerSeq: 1}}), rocksdb.ErrStoreClosed) + require.ErrorIs(t, addEntries(s, []Entry{{Hash: h, LedgerSeq: 1}}), rocksdb.ErrStoreClosed) _, err := s.Get(h) require.ErrorIs(t, err, rocksdb.ErrStoreClosed) - require.ErrorIs(t, s.AddEntries(nil), rocksdb.ErrStoreClosed) - require.ErrorIs(t, s.AddEntries([]Entry{}), rocksdb.ErrStoreClosed) + require.ErrorIs(t, addEntries(s, nil), rocksdb.ErrStoreClosed) + require.ErrorIs(t, addEntries(s, []Entry{}), rocksdb.ErrStoreClosed) } func TestHotStore_GracefulCloseAndReopenRoundTrips(t *testing.T) { @@ -157,7 +157,7 @@ func TestHotStore_GracefulCloseAndReopenRoundTrips(t *testing.T) { first, firstStore := openTestHotStoreAt(t, path, chunk.ID(0)) for n := range 16 { - require.NoError(t, first.AddEntries([]Entry{ + require.NoError(t, addEntries(first, []Entry{ {Hash: txhashFor(byte(n), 1), LedgerSeq: uint32(n) + 1}, })) } @@ -179,7 +179,7 @@ func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { for n := range 16 { pre[n] = Entry{Hash: txhashFor(byte(n), 1), LedgerSeq: uint32(n)} } - require.NoError(t, s.AddEntries(pre)) + require.NoError(t, addEntries(s, pre)) var wg sync.WaitGroup var stop atomic.Bool @@ -187,7 +187,7 @@ func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { for w := range workers { wg.Go(func() { for i := byte(0); !stop.Load(); i++ { - _ = s.AddEntries([]Entry{ + _ = addEntries(s, []Entry{ {Hash: txhashFor(i%16, byte(w+5)), LedgerSeq: uint32(i)}, }) } @@ -205,5 +205,13 @@ func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { wg.Wait() postClose := []Entry{{Hash: txhashFor(0x1, 1), LedgerSeq: 1}} - require.ErrorIs(t, s.AddEntries(postClose), rocksdb.ErrStoreClosed) + require.ErrorIs(t, addEntries(s, postClose), rocksdb.ErrStoreClosed) +} + +// addEntries commits entries through AddEntriesToBatch in one batch — the +// production write shape, reduced to a test seeding call. +func addEntries(h *HotStore, entries []Entry) error { + return h.store.Batch(func(b *rocksdb.BatchWriter) error { + return h.AddEntriesToBatch(b, entries) + }) } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/read_assembly_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/read_assembly_test.go index d358b4adc..840c9697f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/read_assembly_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/read_assembly_test.go @@ -346,7 +346,7 @@ func TestTxReader_HotAndColdFederation(t *testing.T) { flHot := buildLedgers(t, []uint32{hotSeq}, 1) hotStore := openTestHotStore(t) for h, seq := range flHot.byHash { - require.NoError(t, hotStore.AddEntries([]Entry{{Hash: h, LedgerSeq: seq}})) + require.NoError(t, addEntries(hotStore, []Entry{{Hash: h, LedgerSeq: seq}})) } coldSeq := chunk.ID(5).FirstLedger() From a298fec385dd19c53093f38e28540f1c0fb74d68 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Wed, 1 Jul 2026 22:54:04 -0400 Subject: [PATCH 26/55] fullhistory: clear lint fallout + finish the ReadOnly WAL comment sweep golangci-lint is red at the tip with two findings: - unparam: (*preparedLedger).queue's error result is always nil (its body is pure BatchWriter puts). Drop the error return. - nolintlint: e2e_test's maintidx suppression became unused when the live-catalog test seam was removed. Trim the directive to funlen. The ReadOnly WAL doc fix missed two comments carrying the old claim: the OpenDbForReadOnly call site said a crash-left WAL is skipped (SST-only), and NewRocksHotRecoveryProbe justified its read-write open with "so RocksDB replays the WAL". A read-only open recovers the WAL into memtables too; what a read-write open uniquely adds is persisting that recovery (its Close flushes to SST). Reword both. --- cmd/stellar-rpc/internal/fullhistory/e2e_test.go | 2 +- cmd/stellar-rpc/internal/fullhistory/hotsource.go | 8 +++++--- .../internal/fullhistory/pkg/rocksdb/rocksdb.go | 4 ++-- .../fullhistory/pkg/stores/eventstore/hot_store.go | 7 ++----- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go index 947f34d18..0c5a68c46 100644 --- a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go @@ -262,7 +262,7 @@ func hashAt(n uint64) [32]byte { // // Correctness is asserted at every step. // -//nolint:funlen,maintidx // one linear end-to-end scenario asserted step by step +//nolint:funlen // one linear end-to-end scenario asserted step by step func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing.T) { if testing.Short() { t.Skip("e2e ingests a full 10k-ledger chunk; skipped in -short") diff --git a/cmd/stellar-rpc/internal/fullhistory/hotsource.go b/cmd/stellar-rpc/internal/fullhistory/hotsource.go index 0c415b911..b79cbe7f3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotsource.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotsource.go @@ -36,9 +36,11 @@ func NewRocksHotProbe(hotChunkPath func(chunk.ID) string, logger *supportlog.Ent } // NewRocksHotRecoveryProbe returns the startup progress probe. Unlike the freeze -// probe, it opens the highest ready hot DB read-write so RocksDB replays any -// synced WAL left by an ungraceful crash before MaxCommittedSeq is read. Startup -// uses it before ingestion opens a writer, then closes it immediately. +// probe, it opens the highest ready hot DB read-write: a read-only open would +// also recover a crash-left synced WAL into memtables (so MaxCommittedSeq is +// correct either way), but only a writable handle persists that recovery — its +// Close flushes to SST. Startup uses it before ingestion opens a writer, then +// closes it immediately. func NewRocksHotRecoveryProbe(hotChunkPath func(chunk.ID) string, logger *supportlog.Entry) backfill.HotProbe { return &rocksHotProbe{hotRoot: hotChunkPath, logger: logger, recover: true} } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go index 107ad9bb1..f51f42e0f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go @@ -534,8 +534,8 @@ func (s *Store) constructAndOpen() error { ) if s.cfg.ReadOnly { // errorIfWalFileExists=false: a cleanly-closed DB has no WAL; if a crash ever - // left one, read-only skips it (SST-only) and the caller's completeness gate - // falls through rather than failing the open. + // left one, the open recovers it into in-memory memtables (see Config.ReadOnly) + // rather than failing, so reads still see every synced write. db, cfHandles, err = grocksdb.OpenDbForReadOnlyColumnFamilies(opts, abs, cfNames, cfOpts, false) } else { db, cfHandles, err = grocksdb.OpenDbColumnFamilies(opts, abs, cfNames, cfOpts) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index 721875745..d8f9eae63 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -468,9 +468,7 @@ func (h *HotStore) IngestLedgerToBatch( //nolint:nilnil // (nil, nil) is the idempotent-duplicate signal; the caller runs the hook only when non-nil return nil, nil } - if qerr := prep.queue(b); qerr != nil { - return nil, qerr - } + prep.queue(b) return prep.apply, nil } @@ -488,7 +486,7 @@ type preparedLedger struct { // queue writes the prepared ledger's rows into b: one DataCF row per // event, one IndexCF row per (term, event), and one OffsetsCF row for // the ledger's per-ledger event count. -func (p *preparedLedger) queue(b *rocksdb.BatchWriter) error { +func (p *preparedLedger) queue(b *rocksdb.BatchWriter) { for i := range p.blobs { eventID := p.startID + uint32(i) b.Put(DataCF, encodeDataKey(eventID), p.blobs[i]) @@ -499,7 +497,6 @@ func (p *preparedLedger) queue(b *rocksdb.BatchWriter) error { //nolint:gosec // bounds-checked in prepareLedger's overflow guard eventCount := uint32(len(p.blobs)) b.Put(OffsetsCF, encodeOffsetKey(p.ledgerSeq), encodeLedgerEventCount(eventCount)) - return nil } // prepareLedger runs the pre-commit pipeline for one ledger (validate → derive From e70178555dfb82f6dbebe853b351e32636f0eacd Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 12:24:01 -0400 Subject: [PATCH 27/55] =?UTF-8?q?fullhistory:=20address=20PR=20#820=20revi?= =?UTF-8?q?ew=20=E2=80=94=20concrete=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applies the clearly-correct review fixes from tamirms's review; the larger interlocking reworks (hot-tier ownership, ingestion-loop rewrite, lifecycle error flow, probe-seam collapse) are left for a follow-up so the design calls stay with the author. - captive-core opener: build the toml from the bytes already read (NewCaptiveCoreTomlFromData) instead of re-reading the file — kills the double parse and the TOCTOU on NETWORK_PASSPHRASE; drop the stale "must inject Core / stub until #772" comments (it's a complete opener); fix the config.go "only two things" comment (there are three keys). - hotloop: reuse geometry.FsyncNewDirs for the create-dir barrier. - StartConfig: carry only Exec (+ RetentionChunks, Fatalf); run() assembles lifecycle.Config from Exec so backfill and lifecycle share one catalog/pool by construction rather than by comment. - lifecycle: move the hand-rolled hot-DB discard to catalog.DiscardHotChunk, a sibling of the two sweeps (crash ordering in the audited place). - prune metric: meter real swept-artifact count (from eligiblePruneOps), not op-closure count, matching the Phase 1 sweep's unit. - ingest metrics: drop the unreachable per-label resolve-on-the-fly fallback vectors and the six struct fields serving only them. - hotLedgerStream: drop the dead unbounded-range and nil-store branches; assert bounded (the freeze only ever passes a bounded chunk range). - eventstore: delete dead RemoveHotChunkDir/HotChunkDir/HotDirName and the production-dead ledger HotStore.FirstSeq; unexport Index() and correct the fold-path comments (the freeze re-derives from raw LCMs, never snapshots the mirror); note DB.Txhash()/Events() reads are the #772 seam. - CF registration: every hot facade now exports CFNames(); hotchunk exports the assembled ColumnFamilies() union so tests stop hand-stitching it. - catalog: correct the inverted hot-key-value doc. --- .../internal/fullhistory/catalog/catalog.go | 7 ++- .../fullhistory/catalog/catalog_sweep.go | 37 ++++++++++++ .../internal/fullhistory/config.go | 9 +-- .../internal/fullhistory/daemon.go | 38 ++++++------- .../internal/fullhistory/hotloop.go | 11 ++-- .../internal/fullhistory/hotsource.go | 21 ++----- .../internal/fullhistory/ingest/metrics.go | 56 ++++--------------- .../internal/fullhistory/lifecycle/discard.go | 43 -------------- .../fullhistory/lifecycle/discard_test.go | 4 +- .../fullhistory/lifecycle/eligibility.go | 26 ++++++--- .../fullhistory/lifecycle/lifecycle.go | 8 +-- .../lifecycle/lifecycle_helpers_test.go | 2 +- .../fullhistory/lifecycle/lifecycle_test.go | 3 +- .../lifecycle/progress_realdb_test.go | 6 +- .../fullhistory/lifecycle/retention_test.go | 2 +- .../pkg/stores/eventstore/cold_index.go | 14 ++--- .../pkg/stores/eventstore/hot_store.go | 37 +++--------- .../pkg/stores/eventstore/hot_store_test.go | 3 +- .../pkg/stores/eventstore/query_test.go | 2 +- .../pkg/stores/hotchunk/hotchunk.go | 16 ++++-- .../pkg/stores/hotchunk/hotchunk_test.go | 2 +- .../pkg/stores/ledger/hot_store.go | 25 ++++----- .../pkg/stores/ledger/hot_store_test.go | 18 +----- .../internal/fullhistory/startup.go | 33 +++++++---- .../internal/fullhistory/startup_test.go | 14 ++--- 25 files changed, 184 insertions(+), 253 deletions(-) delete mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go index 84536182e..a43f360c5 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go @@ -57,8 +57,11 @@ func (c *Catalog) State(chunkID chunk.ID, kind geometry.Kind) (geometry.State, e } // HotState returns the HotState of a chunk's hot-DB key, or empty (key absent). -// The key's mere existence (any value) marks the chunk as owned by ingestion; -// only the last-committed ledger derivation cares which value (see ReadyHotChunkKeys). +// The key's mere existence (any value) marks the chunk as owned by ingestion, and +// most consumers branch on the value: the freeze source and last-committed +// derivation treat only "ready" as usable (see ReadyHotChunkKeys), and +// openHotDBForChunk picks its recovery action from it. Only the discard scan is +// value-blind (any state means "a hot dir may exist, sweep it"). func (c *Catalog) HotState(chunkID chunk.ID) (geometry.HotState, error) { v, ok, err := c.get(geometry.HotChunkKey(chunkID)) if err != nil || !ok { diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_sweep.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_sweep.go index d9dcb0e44..02f1f4bbb 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_sweep.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_sweep.go @@ -1,7 +1,12 @@ package catalog import ( + "fmt" + "os" + "path/filepath" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/metastore" ) @@ -85,3 +90,35 @@ func (c *Catalog) SweepTxHashIndexKey(cov geometry.TxHashIndexCoverage) error { geometry.RmdirIfEmpty(dir) // best-effort; an empty dir is not an artifact return nil } + +// DiscardHotChunk retires a chunk's hot DB once its cold artifacts are durable +// (or it fell past retention), following the same crash order as the two sweeps +// above: mark "transient" -> rmdir -> fsync(parent) -> delete key. The key +// outlives the durable rmdir, so a crash anywhere leaves the key "transient" for +// the next scan to finish — idempotent, and an absent key is a no-op. The caller +// MUST have closed the chunk's hot write handle (discard runs after the freeze). +func (c *Catalog) DiscardHotChunk(chunkID chunk.ID) error { + state, err := c.HotState(chunkID) + if err != nil { + return fmt.Errorf("read hot key chunk %s: %w", chunkID, err) + } + if state == "" { + return nil + } + if err := c.PutHotTransient(chunkID); err != nil { + return fmt.Errorf("mark hot transient chunk %s: %w", chunkID, err) + } + dir := c.layout.HotChunkPath(chunkID) + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("rmdir hot dir %s: %w", dir, err) + } + // rmdir durable BEFORE the key delete: the key outlives the dir, so a crash + // re-runs the discard rather than leaving a key-less dir. + if err := geometry.FsyncDir(filepath.Dir(dir)); err != nil { + return fmt.Errorf("fsync hot parent dir %s: %w", filepath.Dir(dir), err) + } + if err := c.DeleteHotKey(chunkID); err != nil { + return fmt.Errorf("delete hot key chunk %s: %w", chunkID, err) + } + return nil +} diff --git a/cmd/stellar-rpc/internal/fullhistory/config.go b/cmd/stellar-rpc/internal/fullhistory/config.go index f2edb2b84..d557f5a8c 100644 --- a/cmd/stellar-rpc/internal/fullhistory/config.go +++ b/cmd/stellar-rpc/internal/fullhistory/config.go @@ -90,10 +90,11 @@ type BackfillConfig struct { // IngestionConfig is [ingestion] — the live-network ingestion (captive-core) // settings. The captive-core config FILE is the single source of truth for what -// it can hold (notably NETWORK_PASSPHRASE, read back at startup); only the two -// things that genuinely can't live in that file are separate keys — the plain -// history-archive URLs (the file's [HISTORY.*] entries are shell commands, not -// the URLs the SDK's archive client needs) and, optionally, the binary path. +// it can hold (notably NETWORK_PASSPHRASE, read back at startup); the remaining +// keys are the things that don't live in that file — the plain history-archive +// URLs (the file's [HISTORY.*] entries are shell commands, not the URLs the SDK's +// archive client needs), and, optionally, the stellar-core binary path and the +// captive-core storage directory. type IngestionConfig struct { // CaptiveCoreConfig is the path to the CaptiveStellarCore (stellar-core) config // file. Required for live ingestion. Must define NETWORK_PASSPHRASE. diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index d1af58d90..32c182aeb 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -22,7 +22,6 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/ingest" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/observability" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/metastore" ) @@ -42,9 +41,8 @@ type daemonOptions struct { Backend backfill.Backend // Core starts captive core at the resume ledger and yields the live getter the - // ingestion loop polls. nil ⇒ runDaemonWith builds a captiveCoreOpener (whose - // config plumbing is deferred to #772, so production must inject Core until then). - // Tests inject a fake getter. + // ingestion loop polls. nil ⇒ runDaemonWith builds a captiveCoreOpener from + // [ingestion] (a complete production opener). Tests inject a fake getter. Core CoreOpener // ServeReads launches the RPC read server; it must return promptly, not block. @@ -153,9 +151,8 @@ func runDaemonWith(ctx context.Context, configPath string, opts daemonOptions) e metrics, sink := buildSinks(opts, registry) // Resolve the captive-core opener: injected (tests) or built from - // [ingestion].captive_core_config. Production wiring is deferred to #772, so the - // builder errors with a clear pointer — done after validateConfig so config - // errors surface first, and a deployment must inject Core until the cutover. + // [ingestion].captive_core_config (a complete production opener) — done after + // validateConfig so config errors surface first. core := opts.Core if core == nil { built, cerr := newCaptiveCoreOpener(cfg.Ingestion, cfg.Service.DefaultDataDir, logger) @@ -176,9 +173,9 @@ func runDaemonWith(ctx context.Context, configPath string, opts daemonOptions) e return supervise(ctx, start, logger, backoff) } -// startConfig assembles the StartConfig run consumes. Exec and Lifecycle share -// ONE catalog, worker pool, and retention floor (backfill and the lifecycle -// goroutine share one set of postconditions), so Lifecycle embeds the same exec. +// startConfig assembles the StartConfig run consumes. run() builds the +// lifecycle.Config from Exec + RetentionChunks, so backfill and the lifecycle +// goroutine share ONE catalog, worker pool, and retention floor by construction. func startConfig( cfg Config, cat *catalog.Catalog, logger *supportlog.Entry, backend backfill.Backend, networkTip NetworkTipBackend, core CoreOpener, serveReads func(context.Context) error, @@ -199,15 +196,12 @@ func startConfig( return StartConfig{ Exec: exec, HotProgressProbe: NewRocksHotRecoveryProbe(cat.Layout().HotChunkPath, logger), - Lifecycle: lifecycle.Config{ - ExecConfig: exec, - RetentionChunks: deref(cfg.Retention.RetentionChunks), - }, - NetworkTip: networkTip, - Core: core, - ServeReads: serveReads, - TipBackoff: tipBackoff, - TipMaxAttempts: tipMaxAttempts, + RetentionChunks: deref(cfg.Retention.RetentionChunks), + NetworkTip: networkTip, + Core: core, + ServeReads: serveReads, + TipBackoff: tipBackoff, + TipMaxAttempts: tipMaxAttempts, } } @@ -351,7 +345,11 @@ func newCaptiveCoreOpener(ing IngestionConfig, dataDir string, logger *supportlo storagePath = filepath.Join(dataDir, "captive-core") } - coreToml, err := ledgerbackend.NewCaptiveCoreTomlFromFile(ing.CaptiveCoreConfig, ledgerbackend.CaptiveCoreTomlParams{ + // Build the toml from the bytes already read, not the path — re-reading via + // NewCaptiveCoreTomlFromFile would parse the file twice and, worse, could + // observe a different NETWORK_PASSPHRASE than the one peeked above if the file + // changed between the two reads (surfacing as the SDK's confusing mismatch error). + coreToml, err := ledgerbackend.NewCaptiveCoreTomlFromData(data, ledgerbackend.CaptiveCoreTomlParams{ HistoryArchiveURLs: ing.HistoryArchiveURLs, NetworkPassphrase: peek.NetworkPassphrase, Strict: true, diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index 6daee752d..d48ebff45 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -86,16 +86,13 @@ func openHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlo // The dir + dirent must be durable BEFORE the key flips to "ready", else a // crash between the flip and the dir's durability fabricates the "ready but - // dir missing" fatal above for a DB that was actually fine. - if syncErr := geometry.FsyncDir(dir); syncErr != nil { + // dir missing" fatal above for a DB that was actually fine. FsyncNewDirs + // syncs the leaf then its parent dirent (the one audited barrier for a + // freshly created dir). + if syncErr := geometry.FsyncNewDirs(filepath.Dir(dir), dir); syncErr != nil { _ = db.Close() return nil, fmt.Errorf("fsync hot dir %s: %w", dir, syncErr) } - parent := filepath.Dir(dir) - if syncErr := geometry.FsyncDir(parent); syncErr != nil { - _ = db.Close() - return nil, fmt.Errorf("fsync hot parent dir %s: %w", parent, syncErr) - } if flipErr := cat.FlipHotReady(chunkID); flipErr != nil { _ = db.Close() return nil, fmt.Errorf("flip hot ready chunk %s: %w", chunkID, flipErr) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotsource.go b/cmd/stellar-rpc/internal/fullhistory/hotsource.go index b79cbe7f3..464ea3e49 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotsource.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotsource.go @@ -112,23 +112,14 @@ func (st *hotLedgerStream) RawLedgers( ctx context.Context, r ledgerbackend.Range, _ ...ledgerbackend.StreamOption, ) iter.Seq2[[]byte, error] { return func(yield func([]byte, error) bool) { - if st.store == nil { - yield(nil, errors.New("hotLedgerStream has no store")) - return - } - to := r.To() + // The only caller is the freeze via Source(), which always passes a bounded + // chunk range over a constructor-set store (h.db.Ledgers()). Assert the bound + // rather than carry the dead unbounded-range and nil-store branches. if !r.Bounded() { - last, ok, err := st.store.LastSeq() - if err != nil { - yield(nil, err) - return - } - if !ok { - return - } - to = last + yield(nil, fmt.Errorf("hotLedgerStream requires a bounded range, got unbounded from %d", r.From())) + return } - for e, ierr := range st.store.IterateLedgers(r.From(), to) { + for e, ierr := range st.store.IterateLedgers(r.From(), r.To()) { if cerr := ctx.Err(); cerr != nil { yield(nil, cerr) return diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go index 86a019deb..30d3adc87 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go @@ -203,22 +203,16 @@ func (c ingestCollectors) observe(d time.Duration, items int, err error) { // ingest daemon startup path yet. This type only provides the registerable sink. type PrometheusSink struct { // Pre-resolved per-ingester children, keyed by data type, one map per - // tier (the duration histograms have per-tier buckets). + // tier (the duration histograms have per-tier buckets). Every producer + // draws its data_type/stage from the same unexported constant sets these + // maps are built from, so a lookup can never miss — the maps are indexed + // directly, with no on-the-fly vector fallback. hot map[string]ingestCollectors cold map[string]ingestCollectors - // The vectors behind the resolved children, kept for the (unexpected) - // case of a data type outside the construction-time set — resolved on - // the fly so no signal is ever silently dropped. - hotDuration *prometheus.HistogramVec - coldDuration *prometheus.HistogramVec - ingestItems *prometheus.CounterVec - ingestErrors *prometheus.CounterVec // Per-stage durations (IngestStage), pre-resolved per // (data_type, stage) with per-tier buckets, keyed "dataType/stage". - hotStage map[string]prometheus.Observer - coldStage map[string]prometheus.Observer - hotStageVec *prometheus.HistogramVec - coldStageVec *prometheus.HistogramVec + hotStage map[string]prometheus.Observer + coldStage map[string]prometheus.Observer // Aggregate per-tier wall-clock: hot per-ledger Ingest, cold per-chunk // service lifetime. Separate histograms so each tier gets fitting buckets. hotLedgerTotal prometheus.Observer @@ -311,41 +305,19 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh return &PrometheusSink{ hot: hot, cold: cold, - hotDuration: hotDuration, - coldDuration: coldDuration, - ingestItems: ingestItems, - ingestErrors: ingestErrors, hotStage: hotStage, coldStage: coldStage, - hotStageVec: hotStageVec, - coldStageVec: coldStageVec, hotLedgerTotal: hotLedgerTotal, coldChunkTotal: coldChunkTotal, } } func (p *PrometheusSink) HotIngest(dataType string, d time.Duration, items int, err error) { - c, ok := p.hot[dataType] - if !ok { - c = ingestCollectors{ - duration: p.hotDuration.WithLabelValues(dataType), - items: p.ingestItems.WithLabelValues(dataType, tierHot), - errors: p.ingestErrors.WithLabelValues(dataType, tierHot), - } - } - c.observe(d, items, err) + p.hot[dataType].observe(d, items, err) } func (p *PrometheusSink) ColdIngest(dataType string, d time.Duration, items int, err error) { - c, ok := p.cold[dataType] - if !ok { - c = ingestCollectors{ - duration: p.coldDuration.WithLabelValues(dataType), - items: p.ingestItems.WithLabelValues(dataType, tierCold), - errors: p.ingestErrors.WithLabelValues(dataType, tierCold), - } - } - c.observe(d, items, err) + p.cold[dataType].observe(d, items, err) } func (p *PrometheusSink) HotLedgerTotal(d time.Duration) { @@ -361,15 +333,9 @@ func (p *PrometheusSink) ColdChunkTotal(d time.Duration) { // items_total already carries volume); they exist on the interface for the // CSV bench sink. func (p *PrometheusSink) IngestStage(dataType, tier, stage string, d time.Duration, _ int) { - resolved, vec := p.hotStage, p.hotStageVec + resolved := p.hotStage if tier == tierCold { - resolved, vec = p.coldStage, p.coldStageVec - } - o, ok := resolved[dataType+"/"+stage] - if !ok { - // Unexpected (data_type, stage) outside the construction-time set — - // resolve on the fly so no signal is silently dropped. - o = vec.WithLabelValues(dataType, stage) + resolved = p.coldStage } - o.Observe(d.Seconds()) + resolved[dataType+"/"+stage].Observe(d.Seconds()) } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go deleted file mode 100644 index c60f1bfb6..000000000 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard.go +++ /dev/null @@ -1,43 +0,0 @@ -package lifecycle - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" -) - -// discardHotDBForChunk retires a chunk's hot DB once its cold artifacts are -// durable (or it fell past retention): transient -> rmdir+fsync parent -> delete -// key. Idempotent — a missing key is a no-op, and a crash mid-discard leaves the -// key "transient" for the next scan to finish. The caller must have closed the -// write handle (the stage runs after executePlan froze the cold artifacts). -func discardHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID) error { - state, err := cat.HotState(chunkID) - if err != nil { - return fmt.Errorf("read hot key chunk %s: %w", chunkID, err) - } - if state == "" { - return nil - } - if putErr := cat.PutHotTransient(chunkID); putErr != nil { - return fmt.Errorf("mark hot transient chunk %s: %w", chunkID, putErr) - } - - dir := cat.Layout().HotChunkPath(chunkID) - if rmErr := os.RemoveAll(dir); rmErr != nil { - return fmt.Errorf("rmdir hot dir %s: %w", dir, rmErr) - } - // rmdir must be durable BEFORE the key delete: the key outlives the dir, so a - // crash re-runs the discard rather than leaving a key-less dir. - if syncErr := geometry.FsyncDir(filepath.Dir(dir)); syncErr != nil { - return fmt.Errorf("fsync hot parent dir %s: %w", filepath.Dir(dir), syncErr) - } - if delErr := cat.DeleteHotKey(chunkID); delErr != nil { - return fmt.Errorf("delete hot key chunk %s: %w", chunkID, delErr) - } - return nil -} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard_test.go index ce958ad43..fd6253929 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/discard_test.go @@ -18,7 +18,7 @@ func TestDiscardHotTier_RemovesDirAndKey(t *testing.T) { db := openLiveHotDB(t, cat, c) require.NoError(t, db.Close()) - require.NoError(t, discardHotDBForChunk(cat, c)) + require.NoError(t, cat.DiscardHotChunk(c)) has, err := hotKeyExists(cat, c) require.NoError(t, err) @@ -26,5 +26,5 @@ func TestDiscardHotTier_RemovesDirAndKey(t *testing.T) { _, statErr := os.Stat(cat.Layout().HotChunkPath(c)) assert.True(t, os.IsNotExist(statErr), "the dir is removed") - require.NoError(t, discardHotDBForChunk(cat, c), "second discard is a no-op") + require.NoError(t, cat.DiscardHotChunk(c), "second discard is a no-op") } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go index 737e587ab..1972f4970 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go @@ -14,7 +14,7 @@ import ( // fully serve (or that fell past retention). Per chunk: below the floor → discard; // complete (last <= through), nothing pending, and the index covers it → discard; // otherwise (live, or frozen awaiting coverage) → leave alone. -// discardHotDBForChunk is idempotent, so a crash between freeze and discard +// catalog.DiscardHotChunk is idempotent, so a crash between freeze and discard // self-heals next tick. func eligibleDiscardOps(cfg Config, cat *catalog.Catalog, through uint32) ([]func() error, error) { earliest, _, err := cat.EarliestLedger() @@ -36,7 +36,7 @@ func eligibleDiscardOps(cfg Config, cat *catalog.Catalog, through uint32) ([]fun last := c.LastLedger() switch { case gate.Excludes(c): - ops = append(ops, func() error { return discardHotDBForChunk(cat, c) }) + ops = append(ops, func() error { return cat.DiscardHotChunk(c) }) case last <= through: pending, perr := pendingArtifacts(c, cat) if perr != nil { @@ -47,7 +47,7 @@ func eligibleDiscardOps(cfg Config, cat *catalog.Catalog, through uint32) ([]fun return nil, cerr } if pending.Empty() && covers { - ops = append(ops, func() error { return discardHotDBForChunk(cat, c) }) + ops = append(ops, func() error { return cat.DiscardHotChunk(c) }) } // else: frozen awaiting coverage, or still producing — leave alone. } @@ -102,19 +102,24 @@ func indexCovers(c chunk.ID, cat *catalog.Catalog) (bool, error) { // batched SweepChunkArtifacts for the chunk family). "Below the floor" is the // gate predicate shared with the discard scan and read path, so prune deletes // exactly what the reader has stopped admitting. -func eligiblePruneOps(cfg Config, cat *catalog.Catalog, through uint32) ([]func() error, error) { +// The second return is the total number of artifacts the ops will sweep (one per +// index-key op plus every ref in the single batched chunk sweep), so the caller +// meters Prune in artifacts — the same unit the Phase 1 sweep reports — rather +// than in op closures (the chunk family collapses N artifacts into one op). +func eligiblePruneOps(cfg Config, cat *catalog.Catalog, through uint32) ([]func() error, int, error) { earliest, _, err := cat.EarliestLedger() if err != nil { - return nil, err + return nil, 0, err } gate := NewRetentionFloor(through, cfg.RetentionChunks, earliest) var ops []func() error + artifacts := 0 // Index family: transient debris from any window, plus frozen keys below the floor. idxKeys, err := cat.AllTxHashIndexKeys() if err != nil { - return nil, err + return nil, 0, err } for _, cov := range idxKeys { switch { @@ -123,16 +128,18 @@ func eligiblePruneOps(cfg Config, cat *catalog.Catalog, through uint32) ([]func( // because no build is in flight when this scan runs (it follows // executePlan's return, and backfill finishes before the loop starts). ops = append(ops, func() error { return cat.SweepTxHashIndexKey(cov) }) + artifacts++ case gate.Excludes(cat.TxHashIndexLayout().LastChunk(cov.Index)): // Frozen index key below the floor; the sweep demotes it first. ops = append(ops, func() error { return cat.SweepTxHashIndexKey(cov) }) + artifacts++ } } // Chunk family: swept in one batch. refs, err := cat.ChunkArtifactKeys() if err != nil { - return nil, err + return nil, 0, err } var sweep []catalog.ArtifactRef for _, ref := range refs { @@ -151,7 +158,7 @@ func eligiblePruneOps(cfg Config, cat *catalog.Catalog, through uint32) ([]func( // redundant. redundant, rerr := txhashRedundantInFinalizedWindow(cat, ref.Chunk) if rerr != nil { - return nil, rerr + return nil, 0, rerr } if redundant { sweep = append(sweep, ref) @@ -160,8 +167,9 @@ func eligiblePruneOps(cfg Config, cat *catalog.Catalog, through uint32) ([]func( } if len(sweep) > 0 { ops = append(ops, func() error { return cat.SweepChunkArtifacts(sweep) }) + artifacts += len(sweep) } - return ops, nil + return ops, artifacts, nil } // txhashRedundantInFinalizedWindow reports whether c's window has a TERMINAL diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index 99075a328..21905c0c3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -203,7 +203,7 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu // Stage 3 — prune scan. pruneStart := time.Now() - pruneOps, err := eligiblePruneOps(cfg, cat, through) + pruneOps, prunedArtifacts, err := eligiblePruneOps(cfg, cat, through) if cfg.abortTick(ctx, err, "eligible prune ops") { return } @@ -212,9 +212,9 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu return } } - metrics.Prune(len(pruneOps), time.Since(pruneStart)) - if logger != nil && len(pruneOps) > 0 { - logger.WithField("pruned", len(pruneOps)).Info("streaming: lifecycle prune stage complete") + metrics.Prune(prunedArtifacts, time.Since(pruneStart)) + if logger != nil && prunedArtifacts > 0 { + logger.WithField("pruned", prunedArtifacts).Info("streaming: lifecycle prune stage complete") } } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go index 81bd06268..96e82c756 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go @@ -188,7 +188,7 @@ func assertQuiescent(t *testing.T, cfg Config, cat *catalog.Catalog, through uin dops, err := eligibleDiscardOps(cfg, cat, through) require.NoError(t, err) assert.Empty(t, dops, "re-scan finds no discard work at quiescence") - pops, err := eligiblePruneOps(cfg, cat, through) + pops, _, err := eligiblePruneOps(cfg, cat, through) require.NoError(t, err) assert.Empty(t, pops, "re-scan finds no prune work at quiescence") } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go index 0e6a6ebfc..de8c5b791 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go @@ -176,9 +176,10 @@ func TestRunLifecycleTick_PrunesTransientIndexDebris(t *testing.T) { through, err := deriveCompleteThrough(cat) require.NoError(t, err) - ops, err := eligiblePruneOps(cfg, cat, through) + ops, artifacts, err := eligiblePruneOps(cfg, cat, through) require.NoError(t, err) require.Len(t, ops, 1, "the freezing debris is swept") + require.Equal(t, 1, artifacts, "one index artifact swept") require.NoError(t, ops[0]()) require.False(t, rec.fired()) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go index 3fe7dce2b..6dcea305e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go @@ -1,7 +1,6 @@ package lifecycle import ( - "slices" "testing" "github.com/stretchr/testify/require" @@ -9,9 +8,8 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash" ) // seedLedgersCF reopens a CLOSED chunk hot DB raw and commits sparse ledgers-CF @@ -23,7 +21,7 @@ func seedLedgersCF(t *testing.T, cat *catalog.Catalog, c chunk.ID, entries ...le t.Helper() store, err := rocksdb.New(rocksdb.Config{ Path: cat.Layout().HotChunkPath(c), - ColumnFamilies: slices.Concat([]string{ledger.LedgersCF}, eventstore.CFNames(), txhash.CFNames()), + ColumnFamilies: hotchunk.ColumnFamilies(), Logger: silentLogger(), }) require.NoError(t, err) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go index fc954644f..fec1a2c03 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go @@ -136,7 +136,7 @@ func TestReaderRetention_WindowStraddlingFloorServesInRangeNotBelow(t *testing.T assert.False(t, floor.Excludes(wins.LastChunk(0)), "a straddling window is not wholly below the floor — its .idx is kept") cfg, _ := lifecycleTestConfig(t, cat, 2) - pops, err := eligiblePruneOps(cfg, cat, through) + pops, _, err := eligiblePruneOps(cfg, cat, through) require.NoError(t, err) for _, op := range pops { require.NoError(t, op()) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/cold_index.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/cold_index.go index b730b9986..8c02c8107 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/cold_index.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/cold_index.go @@ -50,15 +50,13 @@ import ( // hit a slow (*Bitmap).lazyOR path at query time and K≥12 regresses // catastrophically. // -// Two callers produce bitmaps: +// Bitmaps reach this function one way today: // -// - Cold backfill builds a Bitmaps single-threaded via per-event -// events.TermsFor + Bitmaps.AddTo, hands it directly to this -// function. -// - The live-chunk freeze path calls hotStore.Index().Snapshot() to -// materialize a uniquely-owned Bitmaps from the concurrent live -// mirror; that Snapshot Clones each bitmap so this function may -// mutate them freely. +// - Both cold backfill and the live-chunk freeze build a Bitmaps +// single-threaded by re-deriving terms from raw LCMs (per-event +// events.TermsFor + Bitmaps.AddTo) and hand it directly here. The +// freeze does NOT snapshot the hot in-memory mirror — that path has +// no production caller (see eventstore.HotStore.index). // // index.hash is the MPHF serialized via buildMPHF. // diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index d8f9eae63..f8bcc23d9 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -7,8 +7,6 @@ import ( "fmt" "iter" "math" - "os" - "path/filepath" "github.com/RoaringBitmap/roaring/v2" "github.com/linxGnu/grocksdb" @@ -18,11 +16,6 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" ) -// HotDirName is the subdirectory under EventsFullHistoryDataDir that -// contains one DB per active hot chunk (the current_hot_chunk plus -// any chunk currently being frozen). -const HotDirName = "hot" - // Column-family names used inside one chunk's hot RocksDB DB. The // per-Chunk DB directory encodes the chunk ID, so the CF names // themselves carry no chunk suffix. @@ -32,22 +25,6 @@ const ( OffsetsCF = "events_offsets" ) -// HotChunkDir returns the on-disk path of chunkID's per-Chunk hot DB -// rooted at dataDir. -func HotChunkDir(dataDir string, chunkID chunk.ID) string { - return filepath.Join(dataDir, HotDirName, chunkID.String()) -} - -// RemoveHotChunkDir deletes chunkID's hot DB directory. Idempotent — -// returns nil when the directory is already absent. -// -// The caller MUST close chunkID's caller-owned RocksDB handle before calling -// this; otherwise RocksDB's LOCK file is still held and the on-disk state will be -// inconsistent. -func RemoveHotChunkDir(dataDir string, chunkID chunk.ID) error { - return os.RemoveAll(HotChunkDir(dataDir, chunkID)) -} - // Per-CF tuning for the hot store, passed via rocksdb.Config.PerCFOptions: // // - DataCF holds XDR-encoded event payloads: compressible (zstd @@ -119,7 +96,7 @@ var ErrLedgerOutOfOrder = errors.New("events: ledger out of order") // via chunkStore.IsClosed() and rely on the mirror's internal locks and // RocksDB's thread-safety. // - Metadata split after the caller-owned store is closed: ChunkID, -// NextEventID, Index are infallible (cached, usable post-close); EventCount, +// NextEventID are infallible (cached, usable post-close); EventCount, // Offsets return ErrClosed after close (Reader-interface contract). type HotStore struct { chunkStore *rocksdb.Store @@ -196,12 +173,12 @@ func (h *HotStore) Offsets() (*events.LedgerOffsets, error) { return h.offsets.View(), nil } -// Index returns the in-memory term mirror. Used by the freezer to -// snapshot every (events.TermKey, bitmap) pair into WriteColdIndex -// without rebuilding from RocksDB. Callers should typically call -// h.Index().Snapshot() to get a uniquely owned Bitmaps for -// serialization. -func (h *HotStore) Index() *events.ConcurrentBitmaps { return h.mirror } +// index returns the in-memory term mirror. Test-only write hook: no +// production path reads it. The live-chunk freeze re-derives the cold +// event index from raw LCMs (see backfill), so it never snapshots this +// mirror. Kept unexported until #772 decides whether the v2 read path +// hooks a snapshot here. +func (h *HotStore) index() *events.ConcurrentBitmaps { return h.mirror } // Lookup returns the bitmap of event IDs in this Chunk that match // the given term. The returned bitmap is an immutable snapshot of diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go index 92954f160..9e7ca117c 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "iter" + "path/filepath" "sync" "testing" @@ -76,7 +77,7 @@ func tryOpenHotStoreForTest(t *testing.T, dir string, chunkID chunk.ID) (*HotSto func openRawHotChunkForTest(t *testing.T, dir string, chunkID chunk.ID) *rocksdb.Store { t.Helper() raw, err := rocksdb.New(rocksdb.Config{ - Path: HotChunkDir(dir, chunkID), + Path: filepath.Join(dir, chunkID.String()), ColumnFamilies: CFNames(), Logger: silentLogger(), PerCFOptions: CFOptions(), diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/query_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/query_test.go index 706639e56..d3d7fd08d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/query_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/query_test.go @@ -544,7 +544,7 @@ func TestQuery_PostFilterRejectsTermHashCollision(t *testing.T) { // ConcurrentBitmaps.AddTo is the writer-side API the ingest path uses // to register (term, eventID) pairs. No concurrent ingest is running // in this test, so the single-writer contract is satisfied. - fx.store.Index().AddTo(gammaKey, 4) + fx.store.index().AddTo(gammaKey, 4) after, err := fx.store.Lookup(context.Background(), gammaKey) require.NoError(t, err) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index 70b4228ea..37e4f19e5 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -41,10 +41,12 @@ type DB struct { events *eventstore.HotStore } -// columnFamilies is the full CF list for the shared per-chunk DB (ledger + 3 -// events + 1 txhash). Names are already non-colliding across the facades. -func columnFamilies() []string { - return slices.Concat([]string{ledger.LedgersCF}, eventstore.CFNames(), txhash.CFNames()) +// ColumnFamilies is the full CF list for the shared per-chunk DB (ledger + 3 +// events + 1 txhash), assembled from each facade's CFNames() — one idiom, so +// callers (including tests) never hand-stitch the union. Names are non-colliding +// across the facades. +func ColumnFamilies() []string { + return slices.Concat(ledger.CFNames(), eventstore.CFNames(), txhash.CFNames()) } // config builds the shared store's rocksdb.Config: events' per-CF options (ZSTD @@ -55,7 +57,7 @@ func columnFamilies() []string { func config(path string, logger *supportlog.Entry, readOnly bool) rocksdb.Config { return rocksdb.Config{ Path: path, - ColumnFamilies: columnFamilies(), + ColumnFamilies: ColumnFamilies(), Logger: logger, Tuning: txhash.Tuning(), PerCFOptions: eventstore.CFOptions(), @@ -110,9 +112,13 @@ func (d *DB) ChunkID() chunk.ID { return d.chunkID } func (d *DB) Ledgers() *ledger.HotStore { return d.ledger } // Txhash returns the txhash read/write facade over the shared store. +// Write side feeds the ingestion loop; the read side has no production +// caller yet — it's the intended hot read seam for the v2 cutover (#772), +// exercised by tests until then. func (d *DB) Txhash() *txhash.HotStore { return d.txhash } // Events returns the events read/write facade over the shared store. +// Same status as Txhash: writes feed ingestion, reads are the #772 seam. func (d *DB) Events() *eventstore.HotStore { return d.events } // Close releases the shared store exactly once. Idempotent. Must not be called diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go index ce1177f2a..c8015516c 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go @@ -48,7 +48,7 @@ func TestOpen_ValidatesInputs(t *testing.T) { } func TestColumnFamilies_UnionIsNonColliding(t *testing.T) { - cfs := columnFamilies() + cfs := ColumnFamilies() // 1 ledger CF + 3 events CFs + 1 txhash CF = 5. require.Len(t, cfs, 1+len(eventstore.CFNames())+len(txhash.CFNames())) seen := map[string]bool{} diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go index a0d243f19..f35734b6a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go @@ -19,6 +19,11 @@ import ( // shared per-chunk multi-CF DB (decision (a)). const LedgersCF = "ledgers" +// CFNames returns the CFs this facade owns, so the hotchunk shared-DB opener +// assembles the union the same way it does for txhash and eventstore (every +// facade exports CFNames()). +func CFNames() []string { return []string{LedgersCF} } + // Entry — one (sequence, uncompressed ledger bytes) pair. Compression is // internal to the store, so callers pass and receive raw bytes here. type Entry struct { @@ -104,22 +109,12 @@ func (h *HotStore) GetLedgerRaw(seq uint32) ([]byte, error) { return out, nil } -// FirstSeq returns the lowest ledger sequence in the store, or ok=false -// if the store is empty. Cheap (a single RocksDB boundary seek): lets a -// caller learn the store's ledger range without an external chunk hint. -func (h *HotStore) FirstSeq() (uint32, bool, error) { return h.edgeSeq(false) } - // LastSeq returns the highest ledger sequence in the store, or ok=false -// if the store is empty. -func (h *HotStore) LastSeq() (uint32, bool, error) { return h.edgeSeq(true) } - -//nolint:funcorder // helper grouped with FirstSeq/LastSeq for readability -func (h *HotStore) edgeSeq(last bool) (uint32, bool, error) { - edge := h.store.FirstKey - if last { - edge = h.store.LastKey - } - k, ok, err := edge(LedgersCF) +// if the store is empty. This is the chunk's authoritative last-committed +// ledger (hotchunk.DB.MaxCommittedSeq reads it). Cheap — a single RocksDB +// boundary seek on the last key. +func (h *HotStore) LastSeq() (uint32, bool, error) { + k, ok, err := h.store.LastKey(LedgersCF) if err != nil { return 0, false, translateRocksErr(err) } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go index 26d491630..50fc4aac5 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go @@ -96,38 +96,26 @@ func TestHotStore_AddLedgersIdempotentRetry(t *testing.T) { assert.Equal(t, payload, got) // Still a single entry — the retry overwrote rather than appended. - first, ok, err := h.FirstSeq() - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, uint32(7), first) last, ok, err := h.LastSeq() require.NoError(t, err) require.True(t, ok) assert.Equal(t, uint32(7), last) } -func TestHotStore_FirstLastSeq(t *testing.T) { +func TestHotStore_LastSeq(t *testing.T) { h := openTestHotStore(t) // Empty store: ok=false, no error. - _, ok, err := h.FirstSeq() - require.NoError(t, err) - require.False(t, ok) - _, ok, err = h.LastSeq() + _, ok, err := h.LastSeq() require.NoError(t, err) require.False(t, ok) - // Insert seqs out of order; FirstSeq/LastSeq report the min/max present. + // Insert seqs out of order; LastSeq reports the max present. require.NoError(t, addLedgers(h, Entry{Seq: 105, Bytes: []byte("c")}, Entry{Seq: 100, Bytes: []byte("a")}, Entry{Seq: 103, Bytes: []byte("b")}, )) - first, ok, err := h.FirstSeq() - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, uint32(100), first) - last, ok, err := h.LastSeq() require.NoError(t, err) require.True(t, ok) diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index a375a9aed..cee7a5b80 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -58,7 +58,7 @@ func run(ctx context.Context, cfg StartConfig) error { metrics := observability.MetricsOrNop(cfg.Exec.Metrics) metrics.LastCommitted(lastCommitted, - lifecycle.EffectiveRetentionFloor(lastCommitted, cfg.Lifecycle.RetentionChunks, earliest)) + lifecycle.EffectiveRetentionFloor(lastCommitted, cfg.RetentionChunks, earliest)) logger.WithField("last_committed", lastCommitted). WithField("earliest", earliest). WithField("pinned", pinned). @@ -120,9 +120,17 @@ func run(ctx context.Context, cfg StartConfig) error { // passes truncating the same .pack/.idx). Loop checks ctx at every step, so // the join cannot block past the current step. lifecycleCtx, cancelLifecycle := context.WithCancel(ctx) + // Assemble the lifecycle config from the SAME Exec wiring backfill uses, so + // the two share one catalog/pool by construction. WithLifecycleDefaults fills + // Fatalf when unset (Exec is already defaulted, so its re-default is a no-op). + lifecycleCfg := lifecycle.Config{ + ExecConfig: cfg.Exec, + RetentionChunks: cfg.RetentionChunks, + Fatalf: cfg.Fatalf, + }.WithLifecycleDefaults() var lifecycleWG sync.WaitGroup lifecycleWG.Go(func() { - lifecycle.Loop(lifecycleCtx, cfg.Lifecycle, cat, lifecycleCh) + lifecycle.Loop(lifecycleCtx, lifecycleCfg, cat, lifecycleCh) }) // The two return paths registered after this defer (the ingestion-loop return // and the ServeReads error path) have no live sender on lifecycleCh — ingestion @@ -150,7 +158,7 @@ func run(ctx context.Context, cfg StartConfig) error { // computes [rangeStart, rangeEnd] with the mid-chunk exclusion, and breaks on an // empty or non-advancing range. func backfillToTip(ctx context.Context, cfg StartConfig, lastCommitted, earliest uint32) (uint32, error) { - retentionChunks := cfg.Lifecycle.RetentionChunks + retentionChunks := cfg.RetentionChunks metrics := observability.MetricsOrNop(cfg.Exec.Metrics) logger := cfg.Exec.Logger @@ -269,10 +277,15 @@ type StartConfig struct { // synced WAL after crashes; nil falls back to Exec.Process.HotProbe for tests. HotProgressProbe backfill.HotProbe - // Lifecycle drives the lifecycle goroutine. Its embedded ExecConfig is the SAME - // wiring as Exec (one catalog, one pool); RetentionChunks is the backfill floor's - // width too (0 ⇒ the earliest-ledger floor only). - Lifecycle lifecycle.Config + // RetentionChunks bounds the sliding retention floor's width — the backfill + // floor's width too (0 ⇒ the earliest-ledger floor only). run() assembles the + // lifecycle.Config from Exec + this, so the lifecycle and backfill can never + // diverge on the catalog/pool (the invariant is structural, not by comment). + RetentionChunks uint32 + + // Fatalf aborts the daemon on a lifecycle tick op failure; nil ⇒ the + // lifecycle default (log.Fatalf). Tests override it. + Fatalf func(format string, args ...any) // NetworkTip samples the bulk backend's tip during backfill. Required. NetworkTip NetworkTipBackend @@ -297,11 +310,11 @@ const ( defaultTipMaxAttempts = 5 ) -// withDefaults fills the tip-backoff defaults and the embedded Exec/Lifecycle -// defaults (Workers -> GOMAXPROCS; lifecycle Fatalf -> log.Fatalf). +// withDefaults fills the tip-backoff defaults and the embedded Exec defaults +// (Workers -> GOMAXPROCS). The lifecycle.Config (including its Fatalf default) is +// assembled from Exec + RetentionChunks + Fatalf in run(). func (cfg StartConfig) withDefaults() StartConfig { cfg.Exec = cfg.Exec.WithDefaults() - cfg.Lifecycle = cfg.Lifecycle.WithLifecycleDefaults() if cfg.TipBackoff <= 0 { cfg.TipBackoff = defaultTipBackoff } diff --git a/cmd/stellar-rpc/internal/fullhistory/startup_test.go b/cmd/stellar-rpc/internal/fullhistory/startup_test.go index 4d9142c48..936964709 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup_test.go @@ -14,7 +14,6 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) @@ -94,14 +93,11 @@ func startTestConfig( }, } cfg := StartConfig{ - Exec: exec, - Lifecycle: lifecycle.Config{ - ExecConfig: exec, - RetentionChunks: 0, - // A tick op failure should fail the test loudly, not kill the process; the - // loop goroutine is joined before run() returns, so t.Errorf is safe here. - Fatalf: func(format string, args ...any) { t.Errorf("unexpected lifecycle fatal: "+format, args...) }, - }, + Exec: exec, + RetentionChunks: 0, + // A tick op failure should fail the test loudly, not kill the process; the + // loop goroutine is joined before run() returns, so t.Errorf is safe here. + Fatalf: func(format string, args ...any) { t.Errorf("unexpected lifecycle fatal: "+format, args...) }, NetworkTip: tip, Core: core, ServeReads: func(context.Context) error { return nil }, From a694d8c914305627e4cfe69a7b4f48eca7ad978c Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 12:39:17 -0400 Subject: [PATCH 28/55] fullhistory/eventstore: delete NextEventID (test-only duplicate of EventCount) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NextEventID had no production callers through the type — the two in-file uses (FetchRange, All) are just h.offsets.TotalEvents(), and every external caller was a test using it as the infallible accessor. Inline the two internal uses, drop the method and its "two mental models" doc, and migrate the test sites to EventCount() (reusing eventstore's mustEventCount; a small eventCount helper in the hotchunk and fullhistory test packages). --- .../internal/fullhistory/hotloop_test.go | 11 ++++++++- .../pkg/stores/eventstore/hot_store.go | 23 +++++++------------ .../pkg/stores/eventstore/hot_store_test.go | 21 ++++++++--------- .../pkg/stores/hotchunk/hotchunk_test.go | 17 ++++++++++---- 4 files changed, 41 insertions(+), 31 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index 37896e116..ef253557c 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -217,7 +217,7 @@ func TestRunIngestionLoop_LedgerLandsAcrossAllCFs(t *testing.T) { raw, err := reopened.Ledgers().GetLedgerRaw(first + 2) require.NoError(t, err) assert.NotEmpty(t, raw) - assert.Equal(t, uint32(0), reopened.Events().NextEventID(), "zero-tx ledgers carry no events") + assert.Equal(t, uint32(0), eventCount(t, reopened.Events()), "zero-tx ledgers carry no events") } // --------------------------------------------------------------------------- @@ -363,3 +363,12 @@ func TestRunIngestionLoop_RestartResumesFromWatermark(t *testing.T) { require.True(t, ok) assert.Equal(t, first+5, maxSeq) } + +// eventCount reads the hot events store's committed event count, failing the +// test on the (close-only) error the Reader contract allows. +func eventCount(t *testing.T, r interface{ EventCount() (uint32, error) }) uint32 { + t.Helper() + n, err := r.EventCount() + require.NoError(t, err) + return n +} diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index f8bcc23d9..42278a7be 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -95,8 +95,8 @@ var ErrLedgerOutOfOrder = errors.New("events: ledger out of order") // - Reads (Lookup, FetchEvents, All) take NO HotStore-level lock — they guard // via chunkStore.IsClosed() and rely on the mirror's internal locks and // RocksDB's thread-safety. -// - Metadata split after the caller-owned store is closed: ChunkID, -// NextEventID are infallible (cached, usable post-close); EventCount, +// - Metadata split after the caller-owned store is closed: ChunkID is +// infallible (cached, usable post-close); EventCount and // Offsets return ErrClosed after close (Reader-interface contract). type HotStore struct { chunkStore *rocksdb.Store @@ -142,13 +142,6 @@ func (h *HotStore) EventCount() (uint32, error) { return h.offsets.TotalEvents(), nil } -// NextEventID is the next chunk-relative event ID IngestLedgerToBatch -// will assign. Returns the same value as EventCount on the hot side -// and is exposed under both names for the ingest-side and reader-side -// mental models. Infallible at the type level (hot-only API, not on -// the Reader interface). -func (h *HotStore) NextEventID() uint32 { return h.offsets.TotalEvents() } - // Offsets returns a point-in-time view of the ledger-offset cache. // The coordinator uses this to stitch a multi-ledger query range // into chunk-relative event-id ranges (see Reader.Offsets). @@ -321,11 +314,11 @@ func (h *HotStore) FetchEvents(ctx context.Context, eventIDs []uint32) ([]events // // Out-of-range arguments yield an error and stop: // - count == 0 is a natural no-op (no yields). -// - start+count > NextEventID (overflow-safe via uint64) yields a -// wrapped out-of-bounds error. +// - start+count > the committed event count (overflow-safe via uint64) +// yields a wrapped out-of-bounds error. // - A short scan (fewer DataCF rows than count) yields a wrapped // error after the partial stream — the CF should be dense in -// [0, NextEventID), so a hole indicates corruption. +// [0, committed count), so a hole indicates corruption. func (h *HotStore) FetchRange(ctx context.Context, start, count uint32) iter.Seq2[events.Payload, error] { return func(yield func(events.Payload, error) bool) { if h.chunkStore.IsClosed() { @@ -339,7 +332,7 @@ func (h *HotStore) FetchRange(ctx context.Context, start, count uint32) iter.Seq if count == 0 { return } - if err := validateFetchRange(start, count, h.NextEventID(), h.chunkID); err != nil { + if err := validateFetchRange(start, count, h.offsets.TotalEvents(), h.chunkID); err != nil { yield(events.Payload{}, err) return } @@ -384,7 +377,7 @@ func (h *HotStore) FetchRange(ctx context.Context, start, count uint32) iter.Seq // ColdWriter without buffering. Thin wrapper over FetchRange; its // yielded Payloads are likewise borrowed (valid only for the step). // -// NextEventID is read inside the returned closure body, so a +// The committed event count is read inside the returned closure body, so a // concurrent ingest between r.All(ctx) returning the Seq2 and the // consumer's first range step is included in the snapshot. // @@ -393,7 +386,7 @@ func (h *HotStore) All(ctx context.Context) iter.Seq2[events.Payload, error] { return func(yield func(events.Payload, error) bool) { // FetchRange stops iterating after yielding an error; we // just forward whatever it yields and exit on the same step. - for p, err := range h.FetchRange(ctx, 0, h.NextEventID()) { + for p, err := range h.FetchRange(ctx, 0, h.offsets.TotalEvents()) { if !yield(p, err) { return } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go index 9e7ca117c..b7f8acd6e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go @@ -142,7 +142,6 @@ func TestHotStore_FreshChunkHasEmptyState(t *testing.T) { assert.Equal(t, chunkID, h.store.ChunkID()) assert.Equal(t, uint32(0), mustEventCount(t, h.store)) - assert.Equal(t, uint32(0), h.store.NextEventID()) assert.Equal(t, chunkID.FirstLedger(), mustOffsets(t, h.store).StartLedger()) } @@ -180,7 +179,7 @@ func TestHotStore_IngestLedgerWritesAllCFs(t *testing.T) { require.NotNil(t, bm) assert.True(t, bm.Contains(0)) - assert.Equal(t, uint32(1), h.store.NextEventID()) + assert.Equal(t, uint32(1), mustEventCount(t, h.store)) } func TestHotStore_EventIDsAreMonotonic(t *testing.T) { @@ -200,7 +199,7 @@ func TestHotStore_EventIDsAreMonotonic(t *testing.T) { require.NoError(t, err) assert.True(t, found, "missing event id %d", id) } - assert.Equal(t, uint32(3), h.store.NextEventID()) + assert.Equal(t, uint32(3), mustEventCount(t, h.store)) } func TestHotStore_EmptyLedgerStillWritesOffsetsAndState(t *testing.T) { @@ -439,14 +438,14 @@ func TestHotStore_IngestLedgerEvents_DuplicateLedgerIsNoOp(t *testing.T) { require.NoError(t, ingestLedgerEvents(h.store, first, []events.Payload{p1})) countBefore := mustEventCount(t, h.store) - nextBefore := h.store.NextEventID() + nextBefore := mustEventCount(t, h.store) // Re-ingesting the same ledger is an idempotent no-op. p2, _ := makePayload("b") require.NoError(t, ingestLedgerEvents(h.store, first, []events.Payload{p2})) assert.Equal(t, countBefore, mustEventCount(t, h.store), "EventCount must not advance on duplicate ingest") - assert.Equal(t, nextBefore, h.store.NextEventID(), "NextEventID must not advance on duplicate ingest") + assert.Equal(t, nextBefore, mustEventCount(t, h.store), "event count must not advance on duplicate ingest") // The original ledger's event is untouched (not overwritten by p2). got, err := h.store.FetchEvents(context.Background(), []uint32{0}) @@ -477,7 +476,7 @@ func TestHotStore_IngestLedgerEvents_RejectsLedgerGap(t *testing.T) { require.NoError(t, ingestLedgerEvents(h.store, first, []events.Payload{p1})) countBefore := mustEventCount(t, h.store) - nextBefore := h.store.NextEventID() + nextBefore := mustEventCount(t, h.store) // Skip first+1; jump directly to first+2. p2, _ := makePayload("c") @@ -485,7 +484,7 @@ func TestHotStore_IngestLedgerEvents_RejectsLedgerGap(t *testing.T) { require.ErrorIs(t, err, ErrLedgerOutOfOrder) assert.Equal(t, countBefore, mustEventCount(t, h.store)) - assert.Equal(t, nextBefore, h.store.NextEventID()) + assert.Equal(t, nextBefore, mustEventCount(t, h.store)) } // TestHotStore_IngestLedgerEvents_RejectsOutOfRangeLedger pins the @@ -506,7 +505,7 @@ func TestHotStore_IngestLedgerEvents_RejectsOutOfRangeLedger(t *testing.T) { // State must be unchanged after both rejections. assert.Equal(t, uint32(0), mustEventCount(t, h.store)) - assert.Equal(t, uint32(0), h.store.NextEventID()) + assert.Equal(t, uint32(0), mustEventCount(t, h.store)) } func TestHotStore_CloseIsIdempotent(t *testing.T) { @@ -529,11 +528,11 @@ func TestHotStore_ReopenRecoversState(t *testing.T) { hot2, _ := openHotStoreForTestAt(t, dir, chunkID) - assert.Equal(t, uint32(1), hot2.NextEventID(), "warmup recovered offsets") + assert.Equal(t, uint32(1), mustEventCount(t, hot2), "warmup recovered offsets") p2, _ := makePayload("after") require.NoError(t, ingestLedgerEvents(hot2, 3, []events.Payload{p2})) - assert.Equal(t, uint32(2), hot2.NextEventID()) + assert.Equal(t, uint32(2), mustEventCount(t, hot2)) } func TestHotStore_SatisfiesReader(t *testing.T) { @@ -578,7 +577,7 @@ func TestHotStore_ConcurrentIngestAndLookup(t *testing.T) { } }() wg.Wait() - assert.Equal(t, uint32(N), h.store.NextEventID()) + assert.Equal(t, uint32(N), mustEventCount(t, h.store)) } // fetchRangePayloads fully drains FetchRange into a slice for tests diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go index c8015516c..5eed2cd74 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go @@ -106,7 +106,7 @@ func TestIngestLedger_AllCFsAdvanceTogether(t *testing.T) { require.NoError(t, err) require.NotNil(t, bm) assert.Equal(t, uint64(2), bm.GetCardinality(), "both ledgers share the event term") - assert.Equal(t, uint32(2), db.Events().NextEventID()) + assert.Equal(t, uint32(2), eventCount(t, db.Events())) // The single authoritative watermark equals the last committed seq. maxSeq, ok, err := db.MaxCommittedSeq() @@ -144,7 +144,7 @@ func TestIngestLedger_RejectedLedgerPersistsNothingAcrossAnyCF(t *testing.T) { // events CFs — no term indexed, no event committed. _, lerr := db.Events().Lookup(context.Background(), term) require.ErrorIs(t, lerr, eventstore.ErrTermNotFound) - assert.Equal(t, uint32(0), db.Events().NextEventID()) + assert.Equal(t, uint32(0), eventCount(t, db.Events())) // The single watermark is still empty — nothing committed. _, ok, err := db.MaxCommittedSeq() @@ -201,7 +201,7 @@ func TestIngestLedger_MidBatchCommitFailurePersistsNothing(t *testing.T) { // The events CF advanced for exactly the one good ledger — the failed // ledger's event was not committed (warmup reconstructed the offsets from // disk, which hold only the good ledger). - assert.Equal(t, uint32(1), db3.Events().NextEventID(), + assert.Equal(t, uint32(1), eventCount(t, db3.Events()), "the failed ledger's event must not be committed to the events CFs") // The good ledger's data is intact; the failed ledger's is wholly absent @@ -295,7 +295,7 @@ func TestReopen_RecoversEventsMirror(t *testing.T) { db2, err := Open(dir, chunkID, silentLogger()) require.NoError(t, err) t.Cleanup(func() { _ = db2.Close() }) - assert.Equal(t, uint32(1), db2.Events().NextEventID(), "warmup recovered the events offsets") + assert.Equal(t, uint32(1), eventCount(t, db2.Events()), "warmup recovered the events offsets") } // TestOpenReadOnly_ReadsCommittedAndRejectsWrites pins the freeze source's @@ -468,3 +468,12 @@ func buildLCM(t *testing.T, seq uint32, txMetas []xdr.TransactionMeta) (xdr.Ledg } return lcm, hashes } + +// eventCount reads the hot events store's committed event count, failing the +// test on the (close-only) error the Reader contract allows. +func eventCount(t *testing.T, r interface{ EventCount() (uint32, error) }) uint32 { + t.Helper() + n, err := r.EventCount() + require.NoError(t, err) + return n +} From 86285e35d291a9c8bc5cdcfd8871fcd37066adeb Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 14:14:52 -0400 Subject: [PATCH 29/55] =?UTF-8?q?fullhistory:=20hot-DB=20failure=20policy?= =?UTF-8?q?=20=E2=80=94=20no=20auto-create,=20loss=20is=20restartable=20(#?= =?UTF-8?q?14)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reworks the hot-DB open failure handling per review: - rocksdb.Config gains a MustExist knob (read-write, create-if-missing OFF); hotchunk.OpenExisting uses it. Opening a "ready" chunk whose DB is missing or gutted now FAILS the open instead of silently fabricating a fresh empty DB (which regressed the watermark ~10k ledgers with no signal). openHotDBForChunk and the recovery probe both open must-exist. - Delete the ErrHotVolumeLost sentinel and its three wrap sites (hotloop, tryHotSource, refineWithHotDB) plus the stat guards; a "ready" DB that won't open is now an ordinary restartable error with the context in the wrap. - Delete the ErrFirstStartNoTip sentinel; first-start-no-tip returns a plain error. The "never serve incomplete history" property is enforced by returning an error at all (each restart re-checks lastCommitted < earliest), not by the exit shape, so a datastore mid-outage or young lake self-heals on restart. - Collapse supervise's unrecoverable branch: nil/ctx-canceled are clean, everything else is warn+backoff+restart. Loss can't be told from a transient inside the process, so there is no fatal-and-exit class; genuine loss presents as a crash-loop with a clear warn line. never-auto-heal lives in the open flag. - Drop the "run surgical recovery (case 4)" message (leaked private numbering). - Tests: the sentinel ErrorIs assertions become require.Error; the "fatal sentinel, no retry" supervise test inverts to "first-start-no-tip retries". Note: the design doc's "loss is fatal, never restart" prose lives in full-history-streaming-workflow.md (not tracked on this branch) and should be updated to match wherever it resides. --- .../internal/fullhistory/backfill/process.go | 35 +++++++++---------- .../internal/fullhistory/daemon.go | 16 ++++----- .../internal/fullhistory/daemon_test.go | 27 +++++++++++--- .../internal/fullhistory/hotloop.go | 28 +++++---------- .../internal/fullhistory/hotloop_test.go | 11 +++--- .../internal/fullhistory/hotsource.go | 5 ++- .../fullhistory/lifecycle/progress.go | 12 +++---- .../fullhistory/lifecycle/progress_test.go | 13 +++---- .../fullhistory/pkg/rocksdb/rocksdb.go | 14 ++++++-- .../pkg/stores/hotchunk/hotchunk.go | 24 +++++++++---- .../internal/fullhistory/startup.go | 17 ++++----- .../internal/fullhistory/startup_test.go | 16 ++++----- 12 files changed, 117 insertions(+), 101 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go index e8817549a..08827b76d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go @@ -23,19 +23,13 @@ import ( // ErrBackendCoverageTimeout is returned when the bulk backend's tip never reaches the chunk in time. var ErrBackendCoverageTimeout = errors.New("backend never covered chunk within deadline") -// ErrHotVolumeLost is the case-4 fatal: a "ready" hot:chunk key whose DB is -// missing/unopenable — unrecoverable loss (the hot DB is the sole copy of the -// chunk's recently-ingested ledgers), never auto-healed. Detected LAZILY on the -// open that needs the DB. A sentinel so the daemon's top loop owns the fatal. -var ErrHotVolumeLost = errors.New("hot storage lost; run surgical recovery (case 4)") - // HotProbe answers backfillSource's hot branch: is the hot tier COMPLETE for this // chunk (decision (a): maxCommittedSeq >= last ledger), and if so hand back a // LedgerStream over its ledgers CF so the just-closed chunk freezes without a // refetch. Injected: production wires NewRocksHotProbe, tests pass a fake. type HotProbe interface { - // OpenHotChunk borrows the chunk's hot DB for a freeze. ok==false / error under - // a "ready" key (absent or unopenable dir) is case-4 loss. + // OpenHotChunk borrows the chunk's hot DB for a freeze. Under a "ready" key an + // absent/unopenable dir is an ordinary (restartable) error — never auto-healed. OpenHotChunk(chunkID chunk.ID) (HotChunk, bool, error) } @@ -168,7 +162,8 @@ func processChunk(ctx context.Context, chunkID chunk.ID, artifacts catalog.Artif // no-op otherwise), in preference order: // 1. a ready, COMPLETE hot tier (decision (a): maxCommittedSeq >= last ledger) — // only if a HotProbe is wired; incomplete-but-present is staleness that falls -// through (re-derivation recovers it), LOSS is fatal (ErrHotVolumeLost); +// through (re-derivation recovers it); a "ready" DB that won't open is an +// ordinary restartable error (must-exist open, never auto-healed); // 2. the frozen local .pack, unless ledgers is itself requested (circular); // 3. the bulk backend, gated by a bounded waitForCoverage on its Tip. func backfillSource( @@ -183,7 +178,7 @@ func backfillSource( if cfg.HotProbe != nil { src, closer, used, herr := resolveHotSource(chunkID, cfg) if herr != nil { - return nil, noClose, herr // case-4 loss is fatal + return nil, noClose, herr // hot-DB open failure — restartable, never auto-healed } if used { cfg.Logger.Debugf("backfillSource: chunk %s from complete hot tier", chunkID) @@ -229,7 +224,8 @@ func backfillSource( // resolveHotSource applies the hot branch end to end: it reads the hot key and, // only when "ready", tries the hot tier. used=true → src/closer are the hot // source; used=false → no "ready" key or present-but-incomplete (caller falls -// through); err → case-4 loss (fatal). Keeps backfillSource's hot branch flat. +// through); err → a "ready" DB that won't open (restartable). Keeps backfillSource's +// hot branch flat. func resolveHotSource( chunkID chunk.ID, cfg ProcessConfig, ) (ledgerbackend.LedgerStream, func() error, bool, error) { @@ -245,23 +241,24 @@ func resolveHotSource( // tryHotSource handles the hot branch under a "ready" key: used=true when present // AND complete; used=false when present-but-incomplete (staleness, caller falls -// through); err only for case-4 LOSS (ErrHotVolumeLost), detected lazily on the open. +// through); err when a "ready" DB is absent or unopenable — an ordinary restartable +// error (never auto-healed), detected lazily on the open. func tryHotSource(chunkID chunk.ID, cfg ProcessConfig) (ledgerbackend.LedgerStream, func() error, bool, error) { hot, ok, err := cfg.HotProbe.OpenHotChunk(chunkID) if err != nil { - // "ready" key but the DB cannot be opened — hot-volume loss. - return nil, nil, false, fmt.Errorf("%w: chunk %s: %w", ErrHotVolumeLost, chunkID, err) + // "ready" key but the DB cannot be opened. + return nil, nil, false, fmt.Errorf("chunk %s is ready but its hot DB won't open: %w", chunkID, err) } if !ok { - // "ready" key but the dir is absent — hot-volume loss. - return nil, nil, false, fmt.Errorf("%w: chunk %s: hot directory absent", ErrHotVolumeLost, chunkID) + // "ready" key but the dir is absent. + return nil, nil, false, fmt.Errorf("chunk %s is ready but its hot directory is absent", chunkID) } maxSeq, present, merr := hot.MaxCommittedSeq() if merr != nil { _ = hot.Close() - // A read error against an opened DB is loss, not staleness: the DB opened - // but cannot answer its own progress. - return nil, nil, false, fmt.Errorf("%w: chunk %s: max committed seq: %w", ErrHotVolumeLost, chunkID, merr) + // A read error against an opened DB: the DB opened but cannot answer its + // own progress. Surface it (restartable), don't treat as staleness. + return nil, nil, false, fmt.Errorf("chunk %s: read hot max committed seq: %w", chunkID, merr) } // decision (a): complete iff the single DB's maxCommittedSeq reaches the chunk's // last ledger. An empty DB (present==false) cannot be complete. diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index 32c182aeb..7097f85cb 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -219,9 +219,14 @@ func buildSinks(opts daemonOptions, registry *prometheus.Registry) (observabilit return metrics, sink } -// supervise restarts run on a restartable error after a backoff ("startup is the -// recovery path"); a clean shutdown or ctx cancel returns nil; ErrFirstStartNoTip -// is fatal and surfaces up. +// supervise restarts run after a backoff on ANY non-clean return ("startup is the +// recovery path"): nil means a clean shutdown, a ctx cancel means a clean shutdown, +// everything else is warned and retried. Loss can't be distinguished from a +// transient inside the process (an unmounted volume looks identical to a destroyed +// one, and EMFILE / a lingering RocksDB LOCK are recoverable), so there is no +// fatal-and-exit class — genuine loss presents as a crash-loop with a clear warn +// line, the same page an operator would get from a one-shot exit. The +// never-auto-heal guarantee lives in the must-exist open (openHotDBForChunk), not here. func supervise( ctx context.Context, start StartConfig, logger *supportlog.Entry, backoff time.Duration, ) error { @@ -233,11 +238,6 @@ func supervise( if ctx.Err() != nil { return nil //nolint:nilerr // ctx canceled is a clean shutdown, not a run failure } - // Unrecoverable: a fresh start cannot heal these, so don't spin restarting — - // surface them up so an operator/supervisor sees them. - if errors.Is(err, backfill.ErrHotVolumeLost) || errors.Is(err, ErrFirstStartNoTip) { - return err - } logger.WithError(err).Warnf("daemon run failed; restarting in %s", backoff) if sleepCtx(ctx, backoff) != nil { return nil //nolint:nilerr // ctx canceled mid-backoff is a clean shutdown, not a failure diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon_test.go b/cmd/stellar-rpc/internal/fullhistory/daemon_test.go index 7d814410d..ddb9bc47b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon_test.go @@ -430,16 +430,33 @@ func TestSupervise_RetriesThenCleanShutdown(t *testing.T) { assert.GreaterOrEqual(t, attempts.Load(), int32(2), "restarted on the transient failure") } -// Fatal sentinels surface up, not retried (a fresh start cannot heal them). -func TestSupervise_FatalSentinelSurfaces(t *testing.T) { +// A first start with no reachable tip is now RESTARTABLE (previously a fatal +// sentinel): supervise retries it on a backoff rather than surfacing it, and a +// ctx cancel returns clean. Loss/misconfig can't be told from a transient inside +// the process, so there is no fatal-and-exit class. +func TestSupervise_FirstStartNoTipRetries(t *testing.T) { cat, _ := testCatalog(t) pinGenesis(t, cat) - // Unreachable tip + no local progress ⇒ fatal ErrFirstStartNoTip. + // Unreachable tip + no local progress: every run fails the first-start check. tip := &fakeTipBackend{err: errors.New("unreachable"), errFirst: 99} start := startTestConfig(t, cat, tip, &fakeCore{}, nil) + start.TipMaxAttempts = 1 // one tip poll per run, so callCount tracks restart count - err := supervise(context.Background(), start, silentLogger(), time.Hour) - require.ErrorIs(t, err, ErrFirstStartNoTip, "fatal sentinel surfaces immediately, no retry") + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { errCh <- supervise(ctx, start, silentLogger(), 5*time.Millisecond) }() + + require.Eventually(t, func() bool { + return tip.callCount() >= 2 + }, 3*time.Second, 5*time.Millisecond, "first-start-no-tip is retried, not surfaced as fatal") + cancel() + + select { + case err := <-errCh: + require.NoError(t, err, "ctx cancel returns clean, even though runs kept failing") + case <-time.After(3 * time.Second): + t.Fatal("supervise did not return after cancel") + } } // --------------------------------------------------------------------------- diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index d48ebff45..b1e74a1e8 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -9,7 +9,6 @@ import ( supportlog "github.com/stellar/go-stellar-sdk/support/log" "github.com/stellar/go-stellar-sdk/xdr" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/ingest" @@ -36,11 +35,14 @@ type LedgerGetter interface { // openHotDBForChunk opens/recovers/creates the chunk's shared hot DB, keyed on // the durable hot:chunk state: -// - "ready": open it. A MISSING dir is hot-volume loss (the hot DB is the sole -// copy of recently-ingested ledgers) — refuse with ErrHotVolumeLost, never auto-heal. +// - "ready": open it must-exist (create-if-missing OFF). A missing or gutted DB +// FAILS the open — never auto-heal into a fresh empty DB (which would silently +// regress the watermark). The open failure is an ordinary restartable error: +// a transient self-heals on the next attempt, genuine loss becomes a +// supervised crash-loop with the wrapped context. // - "transient" or absent: wipe any leftover dir and create fresh // (transient -> fsync dir+parent -> ready), so a crash mid-create can't -// fabricate the "ready but dir missing" fatal above. +// fabricate a "ready but DB gone" open failure above. func openHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlog.Entry) (*hotchunk.DB, error) { dir := cat.Layout().HotChunkPath(chunkID) @@ -50,23 +52,9 @@ func openHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlo } if state == geometry.HotReady { - if _, statErr := os.Stat(dir); statErr != nil { - if os.IsNotExist(statErr) { - // The key promises a DB the filesystem lacks — hot storage was - // lost under a surviving meta store. Surfaced as the sentinel so - // the daemon's top-level loop owns the fatal-and-surface decision. - return nil, fmt.Errorf( - "%w: chunk %s is %q but its hot dir %s is missing", - backfill.ErrHotVolumeLost, chunkID, geometry.HotReady, dir) - } - return nil, fmt.Errorf( - "%w: chunk %s: stat hot dir %s: %w", - backfill.ErrHotVolumeLost, chunkID, dir, statErr) - } - db, openErr := hotchunk.Open(dir, chunkID, logger) + db, openErr := hotchunk.OpenExisting(dir, chunkID, logger) if openErr != nil { - // The dir existed at the stat above; an open failure now is loss. - return nil, fmt.Errorf("%w: chunk %s: open hot DB: %w", backfill.ErrHotVolumeLost, chunkID, openErr) + return nil, fmt.Errorf("chunk %s is %q but its hot DB won't open: %w", chunkID, geometry.HotReady, openErr) } return db, nil } diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index ef253557c..cb61bd007 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -13,7 +13,6 @@ import ( "github.com/stellar/go-stellar-sdk/xdr" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" @@ -151,9 +150,10 @@ func TestOpenHotTier_CreatesBracketAndDir(t *testing.T) { assert.Equal(t, c.FirstLedger(), resume, "an empty resume DB resumes at the chunk's first ledger") } -// TestOpenHotTier_ReadyButDirMissingIsCase4 is the case-4 fatal: a "ready" key -// whose dir is gone is hot-volume loss, never auto-healed. -func TestOpenHotTier_ReadyButDirMissingIsCase4(t *testing.T) { +// TestOpenHotTier_ReadyButDirMissingFailsOpen: a "ready" key whose DB is gone +// FAILS the must-exist open (never auto-healed into a fresh empty DB). The error +// is ordinary/restartable — no sentinel. +func TestOpenHotTier_ReadyButDirMissingFailsOpen(t *testing.T) { cat, _ := testCatalog(t) c := chunk.ID(5) require.NoError(t, cat.PutHotTransient(c)) @@ -161,7 +161,6 @@ func TestOpenHotTier_ReadyButDirMissingIsCase4(t *testing.T) { _, err := openHotDBForChunk(cat, c, silentLogger()) require.Error(t, err) - require.ErrorIs(t, err, backfill.ErrHotVolumeLost) } // TestOpenHotTier_TransientRecreatesFresh: a "transient" key (crashed @@ -202,7 +201,6 @@ func TestRunIngestionLoop_LedgerLandsAcrossAllCFs(t *testing.T) { err := runIngestionLoop(context.Background(), getter, db, cat, ch, silentLogger(), nil, nil) require.Error(t, err, "poll ran past the prefix and the getter errored") - require.NotErrorIs(t, err, backfill.ErrHotVolumeLost) // Reopen the (loop-closed) DB and assert every CF advanced together. reopened, err := hotchunk.Open(cat.Layout().HotChunkPath(c), c, silentLogger()) @@ -313,7 +311,6 @@ func TestRunIngestionLoop_GetLedgerErrorReturnsError(t *testing.T) { err := runIngestionLoop(context.Background(), getter, db, cat, ch, silentLogger(), nil, nil) require.Error(t, err) require.ErrorIs(t, err, boom) - require.NotErrorIs(t, err, backfill.ErrHotVolumeLost) } // --------------------------------------------------------------------------- diff --git a/cmd/stellar-rpc/internal/fullhistory/hotsource.go b/cmd/stellar-rpc/internal/fullhistory/hotsource.go index 464ea3e49..c090cf191 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotsource.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotsource.go @@ -59,7 +59,10 @@ func (p *rocksHotProbe) OpenHotChunk(chunkID chunk.ID) (backfill.HotChunk, bool, err error ) if p.recover { - db, err = hotchunk.Open(dir, chunkID, p.logger) + // Recovery opens read-WRITE so a synced-WAL replay is persisted on Close, + // but must-exist (create-if-missing OFF): a "ready" chunk's DB is never + // auto-created here. + db, err = hotchunk.OpenExisting(dir, chunkID, p.logger) } else { // Open the chunk's shared multi-CF DB READ-ONLY: the freeze reads its // ledgers to re-derive the cold artifacts and must never mutate it. The diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go index abe538c0f..a6e2c4b47 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go @@ -77,24 +77,22 @@ func LastCommittedLedger(cat *catalog.Catalog, probe backfill.HotProbe) (uint32, // refineWithHotDB opens the highest ready hot chunk through probe and returns // its MaxCommittedSeq, or CompleteThrough(live-1) on an empty DB. A "ready" key -// whose dir/DB is gone surfaces as backfill.ErrHotVolumeLost (lazy loss -// detection). +// whose dir/DB is gone surfaces as an ordinary (restartable) error — the open is +// must-exist, so it is never auto-healed into a fresh empty DB. func refineWithHotDB(probe backfill.HotProbe, live int64) (uint32, error) { id := chunk.ID(live) //nolint:gosec // live > cold >= -1, so live >= 0 hot, ok, openErr := probe.OpenHotChunk(id) if openErr != nil { - return 0, fmt.Errorf("%w: chunk %s is %q but its hot DB won't open (run surgical recovery): %w", - backfill.ErrHotVolumeLost, id, geometry.HotReady, openErr) + return 0, fmt.Errorf("chunk %s is %q but its hot DB won't open: %w", id, geometry.HotReady, openErr) } if !ok { - return 0, fmt.Errorf("%w: chunk %s is %q but its hot dir is missing (run surgical recovery)", - backfill.ErrHotVolumeLost, id, geometry.HotReady) + return 0, fmt.Errorf("chunk %s is %q but its hot dir is missing", id, geometry.HotReady) } defer func() { _ = hot.Close() }() maxSeq, present, seqErr := hot.MaxCommittedSeq() if seqErr != nil { - return 0, fmt.Errorf("%w: chunk %s: max committed seq: %w", backfill.ErrHotVolumeLost, id, seqErr) + return 0, fmt.Errorf("chunk %s: read hot max committed seq: %w", id, seqErr) } if present { return maxSeq, nil diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go index 896c7b18c..13a7472f4 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/require" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" @@ -279,35 +278,33 @@ func TestDeriveWatermark(t *testing.T) { require.Equal(t, uint32(10), got, "refined to the highest ready chunk's seq") }) - t.Run("fatal: a ready HIGHEST chunk whose dir is missing (lazy loss on open)", func(t *testing.T) { + t.Run("errors: a ready HIGHEST chunk whose dir is missing (lazy detection on open)", func(t *testing.T) { cat, _ := testCatalog(t) // The highest ready chunk's dir is missing: the one open the derivation - // performs surfaces the loss as backfill.ErrHotVolumeLost with recovery guidance. + // performs surfaces an ordinary (restartable) error — the must-exist open + // never auto-heals it into a fresh empty DB. require.NoError(t, cat.PutHotTransient(5)) require.NoError(t, cat.FlipHotReady(5)) // ready key 5, NO dir probe := &fakeHotProbe{ok: false} // OpenHotChunk reports dir absent _, err := deriveWatermark(cat, probe) require.Error(t, err) - require.ErrorIs(t, err, backfill.ErrHotVolumeLost) require.Contains(t, err.Error(), "00000005") }) - t.Run("fatal: refinement open error on the highest ready chunk", func(t *testing.T) { + t.Run("errors: refinement open error on the highest ready chunk", func(t *testing.T) { cat, _ := testCatalog(t) readyHot(t, cat, 3) // dir present probe := &fakeHotProbe{openErr: errors.New("rocksdb LOCK held")} _, err := deriveWatermark(cat, probe) require.Error(t, err) - require.ErrorIs(t, err, backfill.ErrHotVolumeLost) }) - t.Run("fatal: refinement read error", func(t *testing.T) { + t.Run("errors: refinement read error", func(t *testing.T) { cat, _ := testCatalog(t) readyHot(t, cat, 3) probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{maxErr: errors.New("corrupt")}} _, err := deriveWatermark(cat, probe) require.Error(t, err) - require.ErrorIs(t, err, backfill.ErrHotVolumeLost) }) t.Run("live chunk 0 ready, empty DB => pre-genesis, no underflow", func(t *testing.T) { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go index f51f42e0f..7f681483a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go @@ -65,6 +65,13 @@ type Config struct { // reads see every synced write, not just SST/MANIFEST state. Used by the // freeze source. ReadOnly bool + + // MustExist opens read-WRITE but with create-if-missing OFF, so opening a + // missing or gutted DB fails instead of silently fabricating a fresh empty + // one. The dir is never created. Used for the "never auto-heal" hot-DB open + // under a "ready" key — a DB the filesystem should already hold. Ignored when + // ReadOnly is set (read-only never creates regardless). + MustExist bool } // Store is the Layer-1 RocksDB handle. Concrete struct: one impl, @@ -506,8 +513,9 @@ func (s *Store) constructAndOpen() error { if err != nil { return fmt.Errorf("rocksdb: canonicalize path %s: %w", s.cfg.Path, err) } - // Read-only opens an existing DB; it never creates the directory. - if !s.cfg.ReadOnly { + // Read-only and must-exist opens require a pre-existing DB; neither creates + // the directory. Only a plain read-write open (create-if-missing) does. + if !s.cfg.ReadOnly && !s.cfg.MustExist { if err := os.MkdirAll(abs, dirPerm); err != nil { return fmt.Errorf("mkdir %s: %w", abs, err) } @@ -515,7 +523,7 @@ func (s *Store) constructAndOpen() error { cfNames := resolveCFNames(s.cfg) opts := grocksdb.NewDefaultOptions() - if !s.cfg.ReadOnly { + if !s.cfg.ReadOnly && !s.cfg.MustExist { opts.SetCreateIfMissing(true) opts.SetCreateIfMissingColumnFamilies(true) } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index 37e4f19e5..dbe9dd411 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -54,7 +54,7 @@ func ColumnFamilies() []string { // per-CF fields apply to every CF — a benign over-application (ledger/events CFs // just gain a bloom + larger write buffer); the per-CF overrides keep events // distinct. -func config(path string, logger *supportlog.Entry, readOnly bool) rocksdb.Config { +func config(path string, logger *supportlog.Entry, readOnly, mustExist bool) rocksdb.Config { return rocksdb.Config{ Path: path, ColumnFamilies: ColumnFamilies(), @@ -62,31 +62,41 @@ func config(path string, logger *supportlog.Entry, readOnly bool) rocksdb.Config Tuning: txhash.Tuning(), PerCFOptions: eventstore.CFOptions(), ReadOnly: readOnly, + MustExist: mustExist, } } // Open opens (or creates) the chunk's shared multi-CF hot DB read-WRITE -// (ingestion's handle) and composes the three facades over it. On any -// facade-construction failure the shared store is closed before returning. +// (ingestion's handle for a NEW chunk) and composes the three facades over it. On +// any facade-construction failure the shared store is closed before returning. func Open(path string, chunkID chunk.ID, logger *supportlog.Entry) (*DB, error) { - return open(path, chunkID, logger, false) + return open(path, chunkID, logger, false, false) +} + +// OpenExisting opens an EXISTING hot DB read-WRITE with create-if-missing OFF — +// ingestion's handle for a chunk whose "ready" key promises the DB already exists. +// A missing or gutted DB fails the open instead of silently fabricating a fresh +// empty one (the "never auto-heal" rule); the caller treats that failure as an +// ordinary restartable error. +func OpenExisting(path string, chunkID chunk.ID, logger *supportlog.Entry) (*DB, error) { + return open(path, chunkID, logger, false, true) } // OpenReadOnly opens an EXISTING hot DB read-only — the freeze source's view. The // freeze only ever opens a chunk ingestion has already cleanly closed, so all // data is in SST (no WAL to replay); composing the facades only reads. func OpenReadOnly(path string, chunkID chunk.ID, logger *supportlog.Entry) (*DB, error) { - return open(path, chunkID, logger, true) + return open(path, chunkID, logger, true, false) } -func open(path string, chunkID chunk.ID, logger *supportlog.Entry, readOnly bool) (*DB, error) { +func open(path string, chunkID chunk.ID, logger *supportlog.Entry, readOnly, mustExist bool) (*DB, error) { if path == "" { return nil, stores.ErrInvalidConfig } if logger == nil { return nil, stores.ErrInvalidConfig } - store, err := rocksdb.New(config(path, logger, readOnly)) + store, err := rocksdb.New(config(path, logger, readOnly, mustExist)) if err != nil { return nil, fmt.Errorf("open chunk %s: %w", chunkID, err) } diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index cee7a5b80..7b821a062 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -21,8 +21,9 @@ import ( // core (injected), launch the lifecycle goroutine on a doorbell, begin serving // reads (injected), and run the live ingestion loop. Returns nil only on a clean // shutdown (ctx canceled mid-run, or the ingestion loop's clean stop); any other -// return is a restartable error the supervisor surfaces (ErrFirstStartNoTip on a -// first start with no reachable backend; a backfill/ingest failure; ErrHotVolumeLost). +// return is a restartable error the supervisor warns on and retries with backoff +// (a first start with no reachable backend, a backfill/ingest failure, or a +// "ready" hot DB that won't open — none are auto-healed, all are re-attempted). func run(ctx context.Context, cfg StartConfig) error { if err := cfg.validate(); err != nil { return err @@ -177,8 +178,12 @@ func backfillToTip(ctx context.Context, cfg StartConfig, lastCommitted, earliest tip, err := networkTip(ctx, cfg.NetworkTip, cfg.TipBackoff, cfg.TipMaxAttempts) if err != nil { if lastCommitted < earliest { - // First start, no reachable backend: FATAL — never serve incomplete history. - return 0, fmt.Errorf("%w: %w", ErrFirstStartNoTip, err) + // First start, no reachable backend: error out — the daemon must never + // serve incomplete history. Restartable: the property is enforced by + // returning an error at all (each restart re-checks lastCommitted < + // earliest), not by the exit shape, so a datastore mid-outage or a young + // lake below genesis self-heals on a later restart. + return 0, fmt.Errorf("network tip unavailable and no local history to serve: %w", err) } // Restart with local progress: serve what's below lastCommitted, skip backfill. tip = lastCommitted @@ -246,10 +251,6 @@ func lastCommittedMidChunk(lastCommitted uint32) bool { return lastCommitted != lifecycle.CompleteThrough(c) } -// ErrFirstStartNoTip is the first-start FATAL: no local progress and no reachable -// tip. A sentinel so the supervisor owns the restart and tests can assert it. -var ErrFirstStartNoTip = errors.New("network tip unavailable and no local history to serve") - // --------------------------------------------------------------------------- // Injected external boundaries (so startup is testable with fakes). // --------------------------------------------------------------------------- diff --git a/cmd/stellar-rpc/internal/fullhistory/startup_test.go b/cmd/stellar-rpc/internal/fullhistory/startup_test.go index 936964709..7e6f0b542 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup_test.go @@ -184,8 +184,9 @@ func TestNetworkTip_CtxCancelAbortsWait(t *testing.T) { // backfillToTip — backfill loop edge cases. // --------------------------------------------------------------------------- -// First start (genesis, no local history) with the tip absent is fatal. -func TestBackfill_FirstStartTipAbsentFatal(t *testing.T) { +// First start (genesis, no local history) with the tip absent errors out +// (restartable — no sentinel; the supervisor retries). +func TestBackfill_FirstStartTipAbsentErrors(t *testing.T) { cat, _ := testCatalog(t) pinGenesis(t, cat) tip := &fakeTipBackend{err: errors.New("backend unreachable"), errFirst: 99} @@ -194,7 +195,6 @@ func TestBackfill_FirstStartTipAbsentFatal(t *testing.T) { // Empty catalog ⇒ lastCommitted=1 < earliest=2 ⇒ first start with no progress. _, err := backfillToTip(context.Background(), cfg, preGenesisLedger, chunk.FirstLedgerSeq) require.Error(t, err) - require.ErrorIs(t, err, ErrFirstStartNoTip) } // First start (genesis) with the tip present computes range [chunk 0, @@ -415,9 +415,9 @@ func TestRun_ServeReadsErrorSurfaces(t *testing.T) { require.NoError(t, db.Close()) } -// run fatals with ErrFirstStartNoTip on a first start with an unavailable tip; +// run errors on a first start with an unavailable tip (restartable, no sentinel); // reads are never served and ingestion never starts. -func TestRun_FirstStartNoTipFatal(t *testing.T) { +func TestRun_FirstStartNoTipErrors(t *testing.T) { cat, _ := testCatalog(t) pinGenesis(t, cat) served := atomic.Int32{} @@ -427,9 +427,9 @@ func TestRun_FirstStartNoTipFatal(t *testing.T) { cfg.ServeReads = func(context.Context) error { served.Add(1); return nil } err := run(context.Background(), cfg) - require.ErrorIs(t, err, ErrFirstStartNoTip) - require.Zero(t, served.Load(), "reads are never served when backfill fatals") - require.Zero(t, core.openedCount.Load(), "core never starts when backfill fatals") + require.Error(t, err) + require.Zero(t, served.Load(), "reads are never served when backfill errors") + require.Zero(t, core.openedCount.Load(), "core never starts when backfill errors") } // run surfaces a missing earliest_ledger pin loudly (a wiring error, not a first From 893cc805d0bf07c2580ea5114018720e5de71021 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 14:39:30 -0400 Subject: [PATCH 30/55] hotchunk: move hotLedgerStream in, add DB.Source() (#22 prep) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relocates the ledgers-CF LedgerStream adapter from fullhistory/hotsource.go into hotchunk as DB.Source(), so the freeze source lives next to the DB it reads. rocksHotChunk.Source() now delegates. Additive/non-breaking — sets up #22, where backfill opens hotchunk.OpenReadOnly directly and calls DB.Source(), letting the HotProbe/HotChunk interfaces and both probes be deleted. --- .../internal/fullhistory/hotsource.go | 46 +---------------- .../pkg/stores/hotchunk/hotchunk.go | 49 +++++++++++++++++++ 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotsource.go b/cmd/stellar-rpc/internal/fullhistory/hotsource.go index c090cf191..2e050e83a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotsource.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotsource.go @@ -1,10 +1,8 @@ package fullhistory import ( - "context" "errors" "fmt" - "iter" "os" "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" @@ -13,7 +11,6 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" ) // rocksHotProbe is the production backfill.HotProbe: it opens the chunk's shared @@ -89,51 +86,12 @@ func (h *rocksHotChunk) MaxCommittedSeq() (uint32, bool, error) { // Source streams the chunk's LCMs from the ledgers CF as a LedgerStream the cold // writer (WriteColdChunk) drains, so a just-closed chunk freezes straight from its -// hot DB without a refetch. +// hot DB without a refetch. The adapter lives on hotchunk.DB. func (h *rocksHotChunk) Source() ledgerbackend.LedgerStream { - return &hotLedgerStream{store: h.db.Ledgers()} + return h.db.Source() } // Close releases the shared hot DB. func (h *rocksHotChunk) Close() error { return h.db.Close() } - -// hotLedgerStream is a ledgerbackend.LedgerStream over a ledger.HotStore, so the -// source-blind cold pipeline freezes a just-closed chunk from its hot DB. -type hotLedgerStream struct { - store *ledger.HotStore -} - -var _ ledgerbackend.LedgerStream = (*hotLedgerStream)(nil) - -// RawLedgers yields the range's wire bytes from the hot store. IterateLedgers -// yields BORROWED buffers (valid only to the next step); the drain loop consumes -// each fully before the next yield, so the borrow is safe. ctx cancellation is -// observed between ledgers (the LedgerStream contract drain relies on). -func (st *hotLedgerStream) RawLedgers( - ctx context.Context, r ledgerbackend.Range, _ ...ledgerbackend.StreamOption, -) iter.Seq2[[]byte, error] { - return func(yield func([]byte, error) bool) { - // The only caller is the freeze via Source(), which always passes a bounded - // chunk range over a constructor-set store (h.db.Ledgers()). Assert the bound - // rather than carry the dead unbounded-range and nil-store branches. - if !r.Bounded() { - yield(nil, fmt.Errorf("hotLedgerStream requires a bounded range, got unbounded from %d", r.From())) - return - } - for e, ierr := range st.store.IterateLedgers(r.From(), r.To()) { - if cerr := ctx.Err(); cerr != nil { - yield(nil, cerr) - return - } - if ierr != nil { - yield(nil, ierr) - return - } - if !yield(e.Bytes, nil) { - return - } - } - } -} diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index dbe9dd411..9175594e3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -9,10 +9,13 @@ package hotchunk import ( + "context" "fmt" + "iter" "slices" sdkingest "github.com/stellar/go-stellar-sdk/ingest" + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" supportlog "github.com/stellar/go-stellar-sdk/support/log" "github.com/stellar/go-stellar-sdk/xdr" @@ -131,6 +134,13 @@ func (d *DB) Txhash() *txhash.HotStore { return d.txhash } // Same status as Txhash: writes feed ingestion, reads are the #772 seam. func (d *DB) Events() *eventstore.HotStore { return d.events } +// Source streams the chunk's LCMs from the ledgers CF as a ledgerbackend.LedgerStream +// the cold writer (backfill's WriteColdChunk) drains, so a just-closed chunk freezes +// straight from its hot DB without a refetch. The freeze opens the DB read-only. +func (d *DB) Source() ledgerbackend.LedgerStream { + return &hotLedgerStream{store: d.ledger} +} + // Close releases the shared store exactly once. Idempotent. Must not be called // concurrently with in-flight reads/writes. func (d *DB) Close() error { return d.store.Close() } @@ -214,3 +224,42 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts } return counts, nil } + +// hotLedgerStream is a ledgerbackend.LedgerStream over a ledger.HotStore, so the +// source-blind cold pipeline freezes a just-closed chunk from its hot DB. +type hotLedgerStream struct { + store *ledger.HotStore +} + +var _ ledgerbackend.LedgerStream = (*hotLedgerStream)(nil) + +// RawLedgers yields the range's wire bytes from the hot store. IterateLedgers +// yields BORROWED buffers (valid only to the next step); the drain loop consumes +// each fully before the next yield, so the borrow is safe. ctx cancellation is +// observed between ledgers (the LedgerStream contract drain relies on). +func (st *hotLedgerStream) RawLedgers( + ctx context.Context, r ledgerbackend.Range, _ ...ledgerbackend.StreamOption, +) iter.Seq2[[]byte, error] { + return func(yield func([]byte, error) bool) { + // The only caller is the freeze via Source(), which always passes a bounded + // chunk range over a constructor-set store (d.ledger). Assert the bound + // rather than carry the dead unbounded-range and nil-store branches. + if !r.Bounded() { + yield(nil, fmt.Errorf("hotLedgerStream requires a bounded range, got unbounded from %d", r.From())) + return + } + for e, ierr := range st.store.IterateLedgers(r.From(), r.To()) { + if cerr := ctx.Err(); cerr != nil { + yield(nil, cerr) + return + } + if ierr != nil { + yield(nil, ierr) + return + } + if !yield(e.Bytes, nil) { + return + } + } + } +} From e0be57863fb0e39383d906b029ef6aeeb81a4890 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 15:00:22 -0400 Subject: [PATCH 31/55] =?UTF-8?q?fullhistory:=20delete=20HotProbe=20seam?= =?UTF-8?q?=20=E2=80=94=20backfill/watermark=20open=20hot=20DBs=20by=20pat?= =?UTF-8?q?h=20(#22b)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HotProbe/HotChunk interfaces had exactly one production implementation (rocksHotProbe) and one hand-copied test double that had drifted from it. The layering they were meant to protect (backfill not importing hotchunk) was already void (backfill -> catalog -> metastore -> rocksdb), so the indirection bought nothing. Delete it. - backfill.tryHotSource and lifecycle.refineWithHotDB now call hotchunk.OpenReadOnly(Layout.HotChunkPath(chunkID), ...) directly. - LastCommittedLedger takes a *supportlog.Entry instead of a probe: nil => the positional term (no open); non-nil => one read-only refinement of the highest ready hot DB. A read-only open replays any crash-left synced WAL into memtables, so MaxCommittedSeq is correct without the read-write recovery mode (deleted per #22 decision A). - backfillSource always attempts the hot branch (the "ready" key gates it); the cfg.HotProbe != nil skip is gone. - Delete hotsource.go, ProcessConfig.HotProbe, StartConfig.HotProgressProbe and their daemon wiring + validate checks. Tests: delete the fakeHotProbe/fakeHotChunk doubles and the drifted hand-copied probe; TestDeriveWatermark and new backfill hot-source tests now drive REAL temp-dir hot DBs opened by path (physically-impossible fake-only cases dropped, already covered by progress_realdb_test.go). Whole fullhistory tree builds + vets; backfill/lifecycle/root packages pass -short. --- .../fullhistory/backfill/hot_fakes_test.go | 53 -------- .../fullhistory/backfill/hotsource_test.go | 84 +++++++++++++ .../internal/fullhistory/backfill/process.go | 77 ++++-------- .../fullhistory/backfill/process_test.go | 7 +- .../internal/fullhistory/daemon.go | 20 ++- .../internal/fullhistory/e2e_test.go | 5 +- .../internal/fullhistory/hotsource.go | 97 -------------- .../fullhistory/lifecycle/helpers_test.go | 119 ++---------------- .../{hot_fakes_test.go => hotkeys_test.go} | 48 ------- .../lifecycle/lifecycle_helpers_test.go | 12 +- .../fullhistory/lifecycle/lifecycle_test.go | 2 +- .../fullhistory/lifecycle/progress.go | 34 ++--- .../lifecycle/progress_realdb_test.go | 43 ++++--- .../lifecycle/progress_shim_test.go | 16 +-- .../fullhistory/lifecycle/progress_test.go | 90 +++---------- .../internal/fullhistory/startup.go | 22 +--- .../internal/fullhistory/startup_test.go | 4 +- 17 files changed, 217 insertions(+), 516 deletions(-) delete mode 100644 cmd/stellar-rpc/internal/fullhistory/backfill/hot_fakes_test.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/backfill/hotsource_test.go delete mode 100644 cmd/stellar-rpc/internal/fullhistory/hotsource.go rename cmd/stellar-rpc/internal/fullhistory/lifecycle/{hot_fakes_test.go => hotkeys_test.go} (56%) diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/hot_fakes_test.go b/cmd/stellar-rpc/internal/fullhistory/backfill/hot_fakes_test.go deleted file mode 100644 index f71311836..000000000 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/hot_fakes_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package backfill - -import ( - "sync/atomic" - - "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" - - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" -) - -// fakeHotChunk is a test HotChunk: a hand-set MaxCommittedSeq + an injectable -// LedgerStream source, counting closes when closedTo is non-nil. -type fakeHotChunk struct { - maxSeq uint32 - present bool - maxErr error - source ledgerbackend.LedgerStream - closedTo *atomic.Int32 -} - -func (h *fakeHotChunk) MaxCommittedSeq() (uint32, bool, error) { - return h.maxSeq, h.present, h.maxErr -} -func (h *fakeHotChunk) Source() ledgerbackend.LedgerStream { return h.source } -func (h *fakeHotChunk) Close() error { - if h.closedTo != nil { - h.closedTo.Add(1) - } - return nil -} - -// fakeHotProbe is a test HotProbe: returns its fake chunk when ok, an error when -// openErr is set, or (nil,false,nil) for "no ready hot DB". Counts opens via -// openedTo when non-nil. -type fakeHotProbe struct { - chunk *fakeHotChunk - ok bool - openErr error - openedTo *atomic.Int32 -} - -func (p *fakeHotProbe) OpenHotChunk(chunk.ID) (HotChunk, bool, error) { - if p.openedTo != nil { - p.openedTo.Add(1) - } - if p.openErr != nil { - return nil, false, p.openErr - } - if !p.ok { - return nil, false, nil - } - return p.chunk, true, nil -} diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/hotsource_test.go b/cmd/stellar-rpc/internal/fullhistory/backfill/hotsource_test.go new file mode 100644 index 000000000..70ab6de4a --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/hotsource_test.go @@ -0,0 +1,84 @@ +package backfill + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" +) + +// seedReadyHotChunk brackets a "ready" hot DB for c (transient -> create -> ready) +// and commits ONE ledgers-CF entry at seq `top` so MaxCommittedSeq reads back +// `top`. It writes just the ledgers CF (the only CF the completeness gate reads) +// and closes the store, so tryHotSource's read-only reopen is not blocked by the +// RocksDB LOCK. The daemon opens this exact on-disk DB by its Layout path. +func seedReadyHotChunk(t *testing.T, cat *catalog.Catalog, c chunk.ID, top uint32) { + t.Helper() + require.NoError(t, cat.PutHotTransient(c)) + store, err := rocksdb.New(rocksdb.Config{ + Path: cat.Layout().HotChunkPath(c), + ColumnFamilies: hotchunk.ColumnFamilies(), + Logger: silentLogger(), + }) + require.NoError(t, err) + h := ledger.NewWithStore(store, c) + require.NoError(t, store.Batch(func(b *rocksdb.BatchWriter) error { + return h.AddLedgerToBatch(b, ledger.Entry{Seq: top, Bytes: []byte("ledger")}) + })) + require.NoError(t, store.Close()) + require.NoError(t, cat.FlipHotReady(c)) +} + +// TestBackfillSource_HotComplete: a "ready" hot DB whose committed frontier +// reaches the chunk's last ledger IS the source — backfillSource returns it with +// NO backend configured, so success alone proves the hot branch was taken. +func TestBackfillSource_HotComplete(t *testing.T) { + cat, _ := testCatalog(t) + cfg := testProcessConfig(t, cat) // no Backend + + c := chunk.ID(0) + seedReadyHotChunk(t, cat, c, c.LastLedger()) // complete: maxSeq == last ledger + + src, closeSrc, err := backfillSource(context.Background(), c, catalog.AllArtifacts(), cfg) + require.NoError(t, err, "complete hot tier is used; no bulk backend needed") + require.NotNil(t, src) + require.NoError(t, closeSrc()) +} + +// TestBackfillSource_HotIncompleteFallsThrough: a "ready" but incomplete hot DB is +// staleness — backfillSource falls past it. With no pack and no backend, that +// fall-through surfaces as the "no bulk backend" error (not a hot-tier error). +func TestBackfillSource_HotIncompleteFallsThrough(t *testing.T) { + cat, _ := testCatalog(t) + cfg := testProcessConfig(t, cat) // no Backend, no frozen pack + + c := chunk.ID(0) + seedReadyHotChunk(t, cat, c, c.FirstLedger()) // incomplete: maxSeq < last ledger + + _, _, err := backfillSource(context.Background(), c, catalog.AllArtifacts(), cfg) + require.Error(t, err) + require.Contains(t, err.Error(), "no bulk backend", + "an incomplete hot tier falls through; it is not itself an error") +} + +// TestBackfillSource_HotReadyButDirMissing: a "ready" key whose hot DB won't open +// (dir gone) is an ordinary restartable error — the read-only open never +// auto-heals it into a fresh empty DB. +func TestBackfillSource_HotReadyButDirMissing(t *testing.T) { + cat, _ := testCatalog(t) + cfg := testProcessConfig(t, cat) + + c := chunk.ID(0) + require.NoError(t, cat.PutHotTransient(c)) + require.NoError(t, cat.FlipHotReady(c)) // ready key, NO dir on disk + + _, _, err := backfillSource(context.Background(), c, catalog.AllArtifacts(), cfg) + require.Error(t, err) + require.Contains(t, err.Error(), "won't open") +} diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go index 08827b76d..b3f2473e3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go @@ -17,45 +17,19 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/ingest" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" ) // ErrBackendCoverageTimeout is returned when the bulk backend's tip never reaches the chunk in time. var ErrBackendCoverageTimeout = errors.New("backend never covered chunk within deadline") -// HotProbe answers backfillSource's hot branch: is the hot tier COMPLETE for this -// chunk (decision (a): maxCommittedSeq >= last ledger), and if so hand back a -// LedgerStream over its ledgers CF so the just-closed chunk freezes without a -// refetch. Injected: production wires NewRocksHotProbe, tests pass a fake. -type HotProbe interface { - // OpenHotChunk borrows the chunk's hot DB for a freeze. Under a "ready" key an - // absent/unopenable dir is an ordinary (restartable) error — never auto-healed. - OpenHotChunk(chunkID chunk.ID) (HotChunk, bool, error) -} - -// HotChunk is one chunk's opened hot tier: the single DB's completeness gate plus -// an LCM source over the ledgers CF. -type HotChunk interface { - // MaxCommittedSeq is the single authoritative last-committed ledger (decision (a)); - // ok=false on an empty DB (so the chunk cannot be complete). - MaxCommittedSeq() (seq uint32, ok bool, err error) - // Source yields the chunk's LCMs from the ledgers CF as a LedgerStream the cold - // writer (WriteColdChunk) drains. - Source() ledgerbackend.LedgerStream - // Close releases the shared hot DB. - Close() error -} - // ProcessConfig is what processChunk/backfillSource need for a freeze pass. type ProcessConfig struct { Catalog *catalog.Catalog Logger *supportlog.Entry Sink ingest.MetricSink - // HotProbe opens the hot tier for backfillSource's hot branch. Nil (cold-only - // backfill or a hot-less test) skips that branch — pack/backend sources only. - HotProbe HotProbe - // Backend is the bulk source for a chunk with no local copy (BSB now, captive // core later — see the Backend interface). It carries its own frontier Tip, so // the coverage wait needs no separate waiter. May be nil for frontfill-only; @@ -160,10 +134,10 @@ func processChunk(ctx context.Context, chunkID chunk.ID, artifacts catalog.Artif // backfillSource picks a chunk's ledger source (+ a closer for an opened hot DB; // no-op otherwise), in preference order: -// 1. a ready, COMPLETE hot tier (decision (a): maxCommittedSeq >= last ledger) — -// only if a HotProbe is wired; incomplete-but-present is staleness that falls -// through (re-derivation recovers it); a "ready" DB that won't open is an -// ordinary restartable error (must-exist open, never auto-healed); +// 1. a ready, COMPLETE hot tier (decision (a): maxCommittedSeq >= last ledger); +// incomplete-but-present is staleness that falls through (re-derivation +// recovers it); a "ready" DB that won't open is an ordinary restartable error +// (read-only open, never auto-healed); // 2. the frozen local .pack, unless ledgers is itself requested (circular); // 3. the bulk backend, gated by a bounded waitForCoverage on its Tip. func backfillSource( @@ -173,17 +147,15 @@ func backfillSource( cat := cfg.Catalog layout := cat.Layout() - // (1) Hot branch: only when a HotProbe is wired and the hot key is "ready". A - // "transient" key (mid-op or recovery-demoted) is not a read source. - if cfg.HotProbe != nil { - src, closer, used, herr := resolveHotSource(chunkID, cfg) - if herr != nil { - return nil, noClose, herr // hot-DB open failure — restartable, never auto-healed - } - if used { - cfg.Logger.Debugf("backfillSource: chunk %s from complete hot tier", chunkID) - return src, closer, nil - } + // (1) Hot branch: only when the hot key is "ready". A "transient" key (mid-op + // or recovery-demoted) is not a read source; an absent key falls through. + src, closer, used, herr := resolveHotSource(chunkID, cfg) + if herr != nil { + return nil, noClose, herr // hot-DB open failure — restartable, never auto-healed + } + if used { + cfg.Logger.Debugf("backfillSource: chunk %s from complete hot tier", chunkID) + return src, closer, nil } // (2) Frozen local .pack, only when ledgers is not requested (producing ledgers @@ -239,20 +211,21 @@ func resolveHotSource( return tryHotSource(chunkID, cfg) } -// tryHotSource handles the hot branch under a "ready" key: used=true when present -// AND complete; used=false when present-but-incomplete (staleness, caller falls -// through); err when a "ready" DB is absent or unopenable — an ordinary restartable -// error (never auto-healed), detected lazily on the open. +// tryHotSource handles the hot branch under a "ready" key: it opens the chunk's +// shared hot DB read-only (never auto-healed) straight from its Layout path. +// used=true when present AND complete; used=false when present-but-incomplete +// (staleness, caller falls through); err when a "ready" DB is absent or unopenable +// — an ordinary restartable error, detected lazily on the open. func tryHotSource(chunkID chunk.ID, cfg ProcessConfig) (ledgerbackend.LedgerStream, func() error, bool, error) { - hot, ok, err := cfg.HotProbe.OpenHotChunk(chunkID) + dir := cfg.Catalog.Layout().HotChunkPath(chunkID) + // Open the chunk's shared multi-CF DB READ-ONLY: the freeze reads its ledgers to + // re-derive the cold artifacts and must never mutate it. The freeze only targets + // chunks ingestion already released, so its data is in SST (no WAL replay). An + // absent or gutted "ready" DB fails the open — restartable, never auto-created. + hot, err := hotchunk.OpenReadOnly(dir, chunkID, cfg.Logger) if err != nil { - // "ready" key but the DB cannot be opened. return nil, nil, false, fmt.Errorf("chunk %s is ready but its hot DB won't open: %w", chunkID, err) } - if !ok { - // "ready" key but the dir is absent. - return nil, nil, false, fmt.Errorf("chunk %s is ready but its hot directory is absent", chunkID) - } maxSeq, present, merr := hot.MaxCommittedSeq() if merr != nil { _ = hot.Close() diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/process_test.go b/cmd/stellar-rpc/internal/fullhistory/backfill/process_test.go index c21109249..b1905a198 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/process_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/process_test.go @@ -123,10 +123,9 @@ func zeroTxBackend(t *testing.T) *fakeBackend { func testProcessConfig(t *testing.T, cat *catalog.Catalog) ProcessConfig { t.Helper() return ProcessConfig{ - Catalog: cat, - Logger: silentLogger(), - Sink: ingest.NopSink{}, - HotProbe: &fakeHotProbe{}, // not "ready" by default; tests override + Catalog: cat, + Logger: silentLogger(), + Sink: ingest.NopSink{}, } } diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index 7097f85cb..6b9acb0fa 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -188,20 +188,18 @@ func startConfig( Workers: deref(cfg.Backfill.Workers), MaxRetries: deref(cfg.Backfill.MaxRetries), Process: backfill.ProcessConfig{ - Backend: backend, - Sink: sink, - HotProbe: NewRocksHotProbe(cat.Layout().HotChunkPath, logger), + Backend: backend, + Sink: sink, }, } return StartConfig{ - Exec: exec, - HotProgressProbe: NewRocksHotRecoveryProbe(cat.Layout().HotChunkPath, logger), - RetentionChunks: deref(cfg.Retention.RetentionChunks), - NetworkTip: networkTip, - Core: core, - ServeReads: serveReads, - TipBackoff: tipBackoff, - TipMaxAttempts: tipMaxAttempts, + Exec: exec, + RetentionChunks: deref(cfg.Retention.RetentionChunks), + NetworkTip: networkTip, + Core: core, + ServeReads: serveReads, + TipBackoff: tipBackoff, + TipMaxAttempts: tipMaxAttempts, } } diff --git a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go index 0c5a68c46..186af8527 100644 --- a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go @@ -502,10 +502,11 @@ func e2eReadCatalog(t *testing.T, dataDir string) (*catalog.Catalog, func()) { return catalog.NewCatalog(store, NewLayoutFromPaths(paths), windows), func() { _ = store.Close() } } -// mustDeriveWatermark derives the durable watermark through the production probe. +// mustDeriveWatermark derives the durable watermark with the read-only hot-DB +// refinement (passing a logger opens the highest ready hot DB by its Layout path). func mustDeriveWatermark(t *testing.T, cat *catalog.Catalog) uint32 { t.Helper() - wm, err := lifecycle.LastCommittedLedger(cat, NewRocksHotProbe(cat.Layout().HotChunkPath, silentLogger())) + wm, err := lifecycle.LastCommittedLedger(cat, silentLogger()) require.NoError(t, err) return wm } diff --git a/cmd/stellar-rpc/internal/fullhistory/hotsource.go b/cmd/stellar-rpc/internal/fullhistory/hotsource.go deleted file mode 100644 index 2e050e83a..000000000 --- a/cmd/stellar-rpc/internal/fullhistory/hotsource.go +++ /dev/null @@ -1,97 +0,0 @@ -package fullhistory - -import ( - "errors" - "fmt" - "os" - - "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" - supportlog "github.com/stellar/go-stellar-sdk/support/log" - - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" -) - -// rocksHotProbe is the production backfill.HotProbe: it opens the chunk's shared -// multi-CF hot DB and answers backfillSource's completeness question (decision -// (a): the single maxCommittedSeq). -type rocksHotProbe struct { - hotRoot func(chunkID chunk.ID) string - logger *supportlog.Entry - recover bool -} - -// NewRocksHotProbe returns the production backfill.HotProbe (hotChunkPath maps a -// chunk to its hot-DB dir — the daemon passes Layout.HotChunkPath). -// -// Caller contract: OpenHotChunk must NOT be passed the LIVE chunk — ingestion -// holds its hot DB open read-write and a second open fails on RocksDB's LOCK. The -// freeze only ever targets chunks ingestion has already released. -func NewRocksHotProbe(hotChunkPath func(chunk.ID) string, logger *supportlog.Entry) backfill.HotProbe { - return &rocksHotProbe{hotRoot: hotChunkPath, logger: logger} -} - -// NewRocksHotRecoveryProbe returns the startup progress probe. Unlike the freeze -// probe, it opens the highest ready hot DB read-write: a read-only open would -// also recover a crash-left synced WAL into memtables (so MaxCommittedSeq is -// correct either way), but only a writable handle persists that recovery — its -// Close flushes to SST. Startup uses it before ingestion opens a writer, then -// closes it immediately. -func NewRocksHotRecoveryProbe(hotChunkPath func(chunk.ID) string, logger *supportlog.Entry) backfill.HotProbe { - return &rocksHotProbe{hotRoot: hotChunkPath, logger: logger, recover: true} -} - -func (p *rocksHotProbe) OpenHotChunk(chunkID chunk.ID) (backfill.HotChunk, bool, error) { - dir := p.hotRoot(chunkID) - if _, err := os.Stat(dir); err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil, false, nil // dir absent — caller treats as loss under "ready" - } - return nil, false, fmt.Errorf("stat hot dir %s: %w", dir, err) - } - - var ( - db *hotchunk.DB - err error - ) - if p.recover { - // Recovery opens read-WRITE so a synced-WAL replay is persisted on Close, - // but must-exist (create-if-missing OFF): a "ready" chunk's DB is never - // auto-created here. - db, err = hotchunk.OpenExisting(dir, chunkID, p.logger) - } else { - // Open the chunk's shared multi-CF DB READ-ONLY: the freeze reads its - // ledgers to re-derive the cold artifacts and must never mutate it. The - // freeze only targets chunks ingestion already released, so its data is in - // SST (no concurrent writer, no WAL replay needed). - db, err = hotchunk.OpenReadOnly(dir, chunkID, p.logger) - } - if err != nil { - return nil, false, fmt.Errorf("open hot chunk DB: %w", err) - } - return &rocksHotChunk{db: db}, true, nil -} - -// rocksHotChunk is one chunk's opened hot tier — the single shared DB. -type rocksHotChunk struct { - db *hotchunk.DB -} - -// MaxCommittedSeq returns the single authoritative last-committed ledger (decision (a)): the -// highest ledger seq the shared DB has durably committed. ok=false on an empty DB. -func (h *rocksHotChunk) MaxCommittedSeq() (uint32, bool, error) { - return h.db.MaxCommittedSeq() -} - -// Source streams the chunk's LCMs from the ledgers CF as a LedgerStream the cold -// writer (WriteColdChunk) drains, so a just-closed chunk freezes straight from its -// hot DB without a refetch. The adapter lives on hotchunk.DB. -func (h *rocksHotChunk) Source() ledgerbackend.LedgerStream { - return h.db.Source() -} - -// Close releases the shared hot DB. -func (h *rocksHotChunk) Close() error { - return h.db.Close() -} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go index d0be6d36e..9143a9edf 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go @@ -2,10 +2,7 @@ package lifecycle import ( "bytes" - "context" - "errors" "fmt" - "iter" "os" "path/filepath" "testing" @@ -13,26 +10,22 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" - "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" supportlog "github.com/stellar/go-stellar-sdk/support/log" "github.com/stellar/go-stellar-sdk/xdr" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/metastore" ) // This file provides the shared test scaffolding the lifecycle tests need. The // catalog/fixture helpers are copied verbatim from the root fullhistory package's // helpers_test.go (which still serves the root tests). The hot-tier helpers -// (openHotDBForChunk / openLiveHotDB / NewRocksHotProbe) are -// test-local equivalents of the production hot-source primitives that live in the -// root fullhistory package — the lifecycle package cannot import root (root imports -// lifecycle), so the lifecycle tests rebuild them over the same public store APIs. +// (openHotDBForChunk / openLiveHotDB) create the SAME on-disk "ready" hot DBs the +// real daemon does, so the lifecycle tick freezes and the watermark refinement +// read the genuine hot DBs by path (the way production does after #22). // testCPI is the tx-hash index width tests build layouts with; equals the // production constant so on-disk geometry reads back identically. @@ -122,12 +115,11 @@ func zeroTxLCMBytes(t *testing.T, seq uint32) []byte { } // --------------------------------------------------------------------------- -// Hot-tier test scaffolding: test-local equivalents of the root package's -// production hot-source primitives (ingest.go's openHotDBForChunk -// and hotsource.go's rocksHotProbe/NewRocksHotProbe). They use only the public -// hotchunk/ledger/catalog/backfill APIs the production code uses, so a lifecycle -// test reads and freezes the SAME on-disk hot DB the real daemon would, without -// importing the root fullhistory package (which would be an import cycle). +// Hot-tier test scaffolding: a test-local equivalent of the root package's hot +// DB opener (startup.go's openHotDBForChunk). It uses only the public +// hotchunk/catalog APIs the production code uses, so a lifecycle test creates the +// SAME on-disk "ready" hot DB the real daemon would — which the freeze and the +// watermark refinement then open by Layout path, exactly as production does. // --------------------------------------------------------------------------- // openHotDBForChunk creates a "ready" shared hot DB for chunkID under the @@ -162,98 +154,3 @@ func openLiveHotDB(t *testing.T, cat *catalog.Catalog, c chunk.ID) *hotchunk.DB require.NoError(t, err) return db } - -// NewRocksHotProbe returns a test backfill.HotProbe over real per-chunk hot DBs — -// the test equivalent of the production probe. It opens the chunk's shared hot DB -// read-only and answers MaxCommittedSeq / Source / Close over it. -func NewRocksHotProbe(hotChunkPath func(chunk.ID) string, logger *supportlog.Entry) backfill.HotProbe { - return &rocksHotProbe{hotRoot: hotChunkPath, logger: logger} -} - -type rocksHotProbe struct { - hotRoot func(chunkID chunk.ID) string - logger *supportlog.Entry -} - -func (p *rocksHotProbe) OpenHotChunk(chunkID chunk.ID) (backfill.HotChunk, bool, error) { - dir := p.hotRoot(chunkID) - if _, err := os.Stat(dir); err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil, false, nil - } - return nil, false, fmt.Errorf("stat hot dir %s: %w", dir, err) - } - db, err := hotchunk.OpenReadOnly(dir, chunkID, p.logger) - if err != nil { - return nil, false, fmt.Errorf("open hot chunk DB: %w", err) - } - return &rocksHotChunk{chunkID: chunkID, db: db}, true, nil -} - -type rocksHotChunk struct { - chunkID chunk.ID - db *hotchunk.DB -} - -func (h *rocksHotChunk) MaxCommittedSeq() (uint32, bool, error) { - seq, ok, err := h.db.MaxCommittedSeq() - if err != nil { - return 0, false, fmt.Errorf("hot DB max committed seq: %w", err) - } - return seq, ok, nil -} - -func (h *rocksHotChunk) Source() ledgerbackend.LedgerStream { - return &hotLedgerStream{store: h.db.Ledgers()} -} - -func (h *rocksHotChunk) Close() error { - if h.db == nil { - return nil - } - return h.db.Close() -} - -// hotLedgerStream is a ledgerbackend.LedgerStream backed by a ledger.HotStore so -// the cold pipeline can freeze a just-closed chunk straight from its hot DB. -type hotLedgerStream struct { - store *ledger.HotStore -} - -var _ ledgerbackend.LedgerStream = (*hotLedgerStream)(nil) - -func (st *hotLedgerStream) RawLedgers( - ctx context.Context, r ledgerbackend.Range, _ ...ledgerbackend.StreamOption, -) iter.Seq2[[]byte, error] { - return func(yield func([]byte, error) bool) { - if st.store == nil { - yield(nil, errors.New("lifecycle test: hotLedgerStream has no store")) - return - } - to := r.To() - if !r.Bounded() { - last, ok, err := st.store.LastSeq() - if err != nil { - yield(nil, err) - return - } - if !ok { - return - } - to = last - } - for e, ierr := range st.store.IterateLedgers(r.From(), to) { - if cerr := ctx.Err(); cerr != nil { - yield(nil, cerr) - return - } - if ierr != nil { - yield(nil, ierr) - return - } - if !yield(e.Bytes, nil) { - return - } - } - } -} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/hot_fakes_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/hotkeys_test.go similarity index 56% rename from cmd/stellar-rpc/internal/fullhistory/lifecycle/hot_fakes_test.go rename to cmd/stellar-rpc/internal/fullhistory/lifecycle/hotkeys_test.go index 0b7296aa9..e0fb16c79 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/hot_fakes_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/hotkeys_test.go @@ -3,63 +3,15 @@ package lifecycle import ( "os" "path/filepath" - "sync/atomic" "testing" "github.com/stretchr/testify/require" - "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" - - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) -// fakeHotChunk is a test backfill.HotChunk: a hand-set MaxCommittedSeq + an -// injectable LedgerStream source, counting closes when closedTo is non-nil. -type fakeHotChunk struct { - maxSeq uint32 - present bool - maxErr error - source ledgerbackend.LedgerStream - closedTo *atomic.Int32 -} - -func (h *fakeHotChunk) MaxCommittedSeq() (uint32, bool, error) { - return h.maxSeq, h.present, h.maxErr -} -func (h *fakeHotChunk) Source() ledgerbackend.LedgerStream { return h.source } -func (h *fakeHotChunk) Close() error { - if h.closedTo != nil { - h.closedTo.Add(1) - } - return nil -} - -// fakeHotProbe is a test backfill.HotProbe: returns its fake chunk when ok, an -// error when openErr is set, or (nil,false,nil) for "no ready hot DB". Counts -// opens via openedTo when non-nil. -type fakeHotProbe struct { - chunk *fakeHotChunk - ok bool - openErr error - openedTo *atomic.Int32 -} - -func (p *fakeHotProbe) OpenHotChunk(chunk.ID) (backfill.HotChunk, bool, error) { - if p.openedTo != nil { - p.openedTo.Add(1) - } - if p.openErr != nil { - return nil, false, p.openErr - } - if !p.ok { - return nil, false, nil - } - return p.chunk, true, nil -} - // writeArtifact writes a placeholder artifact file at path (creating parents), // so a test can assert presence/absence around the catalog protocol. func writeArtifact(t *testing.T, path string) { diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go index 96e82c756..ed2a1d6fa 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go @@ -104,9 +104,11 @@ func ingestFullHotChunk(t *testing.T, cat *catalog.Catalog, c chunk.ID) { require.NoError(t, db.Close()) // release the write handle (boundary handoff) } -// lifecycleTestConfig wires a Config over the real production primitives -// (a real RocksHotProbe over the catalog's hot layout) plus a fatal recorder so a -// tick abort is observable instead of killing the test process. +// lifecycleTestConfig wires a Config over the real production primitives plus a +// fatal recorder so a tick abort is observable instead of killing the test +// process. The freeze reads the hot tier by opening the chunk's real on-disk DB +// (created by ingestFullHotChunk) straight from its Layout path — the same open +// production does after #22. func lifecycleTestConfig(t *testing.T, cat *catalog.Catalog, retentionChunks uint32) (Config, *fatalRecorder) { t.Helper() rec := &fatalRecorder{} @@ -115,9 +117,7 @@ func lifecycleTestConfig(t *testing.T, cat *catalog.Catalog, retentionChunks uin Catalog: cat, Logger: silentLogger(), Workers: 2, - Process: backfill.ProcessConfig{ - HotProbe: NewRocksHotProbe(cat.Layout().HotChunkPath, silentLogger()), - }, + Process: backfill.ProcessConfig{}, }, RetentionChunks: retentionChunks, Fatalf: rec.fatalf, diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go index de8c5b791..253d3d340 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go @@ -202,7 +202,7 @@ func TestRunLifecycleTick_PrunesTransientIndexDebris(t *testing.T) { func genuineFailureTickSetup(t *testing.T) (Config, *fatalRecorder, *catalog.Catalog) { t.Helper() cat, _ := smallTxHashIndexCatalog(t, 1) - cfg, rec := lifecycleTestConfig(t, cat, 0) // HotProbe wired, no Backend + cfg, rec := lifecycleTestConfig(t, cat, 0) // hot tier read by path, no Backend readyHot(t, cat, 1) // ready live chunk => through = chunk 0 last ledger require.NoError(t, cat.PutHotTransient(0)) // chunk 0 below live, no frozen artifacts, not a ready source return cfg, rec, cat diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go index a6e2c4b47..e3d6b8a07 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go @@ -3,10 +3,12 @@ package lifecycle import ( "fmt" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" + supportlog "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" ) // Progress is derived, never stored. "Highest complete chunk" arithmetic runs in @@ -31,12 +33,13 @@ func CompleteThrough(c int64) uint32 { // // - COLD — highest chunk with all artifacts durable (highestDurableChunk; -1 on // a fresh start). Leads at startup before any hot key exists. -// - HOT — only when hot > cold, only over "ready" keys. probe == nil gives the -// positional term CompleteThrough(hot-1); probe != nil refines with one -// MaxCommittedSeq read (safe: derivation runs before ingestion locks the DB). +// - HOT — only when hot > cold, only over "ready" keys. logger == nil gives the +// positional term CompleteThrough(hot-1); logger != nil refines with one +// read-only MaxCommittedSeq read (safe: derivation runs before ingestion locks +// the DB). // - FLOOR — EarliestLedger()-1 as int64(earliest)-1, so an absent/zero pin // yields the pre-genesis sentinel rather than underflowing. -func LastCommittedLedger(cat *catalog.Catalog, probe backfill.HotProbe) (uint32, error) { +func LastCommittedLedger(cat *catalog.Catalog, logger *supportlog.Entry) (uint32, error) { cold, err := highestDurableChunk(cat) if err != nil { return 0, err @@ -48,13 +51,13 @@ func LastCommittedLedger(cat *catalog.Catalog, probe backfill.HotProbe) (uint32, return 0, err } if hot > cold { - if probe == nil { + if logger == nil { // Positional term: everything below the live (highest ready) chunk. through = max(through, CompleteThrough(hot-1)) } else { // One refinement read of the highest ready hot DB; loss detected lazily // on this open (no eager scan over every ready key). - refined, rerr := refineWithHotDB(probe, hot) + refined, rerr := refineWithHotDB(cat, logger, hot) if rerr != nil { return 0, rerr } @@ -75,19 +78,18 @@ func LastCommittedLedger(cat *catalog.Catalog, probe backfill.HotProbe) (uint32, return through, nil } -// refineWithHotDB opens the highest ready hot chunk through probe and returns -// its MaxCommittedSeq, or CompleteThrough(live-1) on an empty DB. A "ready" key -// whose dir/DB is gone surfaces as an ordinary (restartable) error — the open is -// must-exist, so it is never auto-healed into a fresh empty DB. -func refineWithHotDB(probe backfill.HotProbe, live int64) (uint32, error) { +// refineWithHotDB opens the highest ready hot chunk read-only straight from its +// Layout path and returns its MaxCommittedSeq, or CompleteThrough(live-1) on an +// empty DB. A "ready" key whose dir/DB is gone surfaces as an ordinary +// (restartable) error — the read-only open never auto-heals it into a fresh empty +// DB. A read-only open replays any crash-left synced WAL into memtables, so +// MaxCommittedSeq is correct even after an ungraceful crash. +func refineWithHotDB(cat *catalog.Catalog, logger *supportlog.Entry, live int64) (uint32, error) { id := chunk.ID(live) //nolint:gosec // live > cold >= -1, so live >= 0 - hot, ok, openErr := probe.OpenHotChunk(id) + hot, openErr := hotchunk.OpenReadOnly(cat.Layout().HotChunkPath(id), id, logger) if openErr != nil { return 0, fmt.Errorf("chunk %s is %q but its hot DB won't open: %w", id, geometry.HotReady, openErr) } - if !ok { - return 0, fmt.Errorf("chunk %s is %q but its hot dir is missing", id, geometry.HotReady) - } defer func() { _ = hot.Close() }() maxSeq, present, seqErr := hot.MaxCommittedSeq() diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go index 6dcea305e..582c75a7e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go @@ -37,12 +37,25 @@ func seedLedgersCF(t *testing.T, cat *catalog.Catalog, c chunk.ID, entries ...le require.NoError(t, store.Close()) } +// seedReadyLiveDB brackets a "ready" hot DB for chunk c (via the production +// opener) and commits a single ledgers-CF entry at seq `top` so MaxCommittedSeq +// reads back `top`. top==0 leaves the DB empty (present=false). It closes the DB +// so the refinement's read-only reopen is not blocked by the RocksDB LOCK. +func seedReadyLiveDB(t *testing.T, cat *catalog.Catalog, c chunk.ID, top uint32) { + t.Helper() + db := openLiveHotDB(t, cat, c) // ready key + real dir + empty DB + require.NoError(t, db.Close()) + if top > 0 { + seedLedgersCF(t, cat, c, ledger.Entry{Seq: top, Bytes: []byte("ledger")}) + } +} + // TestDeriveWatermark_RealHotDB_RefinementIsNotStale exercises the watermark -// refinement against a REAL per-chunk hotchunk DB read through the production -// rocksHotProbe — the path the fakeHotProbe table tests stub out. It proves the -// single-DB MaxCommittedSeq refinement reads the actual committed ledger frontier -// (the ledgers CF's last key) and is not a stale/constant value: the bound rises -// to exactly the highest seq committed to the live chunk's real DB. +// refinement against a REAL per-chunk hotchunk DB opened read-only by its Layout +// path (the same open production does). It proves the single-DB MaxCommittedSeq +// refinement reads the actual committed ledger frontier (the ledgers CF's last +// key) and is not a stale/constant value: the bound rises to exactly the highest +// seq committed to the live chunk's real DB. func TestDeriveWatermark_RealHotDB_RefinementIsNotStale(t *testing.T) { cat, _ := testCatalog(t) @@ -69,8 +82,7 @@ func TestDeriveWatermark_RealHotDB_RefinementIsNotStale(t *testing.T) { require.Equal(t, chunk.ID(4).LastLedger(), baseline) require.Greater(t, committedTop, baseline, "fixture must put the real frontier above the baseline") - probe := NewRocksHotProbe(cat.Layout().HotChunkPath, silentLogger()) - got, err := deriveWatermark(cat, probe) + got, err := deriveWatermark(cat, silentLogger()) require.NoError(t, err) require.Equal(t, committedTop, got, "watermark must equal the REAL ledgers-CF last key, not the positional baseline") @@ -79,10 +91,9 @@ func TestDeriveWatermark_RealHotDB_RefinementIsNotStale(t *testing.T) { // TestDeriveWatermark_RealHotDB_OpensHighestReady proves the refinement opens the // HIGHEST ready chunk (the live chunk), not just any ready chunk. Two ready chunks // have independent real hot DBs with DIFFERENT committed frontiers; the watermark -// must reflect the higher chunk's DB. The fakeHotProbe table tests CANNOT cover -// this: fakeHotProbe.OpenHotChunk ignores its chunk-id argument and returns one -// canned DB, so a "open ready[0] instead of ready[len-1]" regression is invisible -// to them — only a real per-chunk probe distinguishes the two. +// must reflect the higher chunk's DB. Only opening the real per-chunk DB by its +// Layout path distinguishes the two — a "open ready[0] instead of ready[len-1]" +// regression would land on the wrong frontier. func TestDeriveWatermark_RealHotDB_OpensHighestReady(t *testing.T) { cat, _ := testCatalog(t) @@ -105,8 +116,7 @@ func TestDeriveWatermark_RealHotDB_OpensHighestReady(t *testing.T) { // top, so reading the wrong chunk yields a strictly different (lower) answer. require.Greater(t, highMid, lowTop) - probe := NewRocksHotProbe(cat.Layout().HotChunkPath, silentLogger()) - got, err := deriveWatermark(cat, probe) + got, err := deriveWatermark(cat, silentLogger()) require.NoError(t, err) require.Equal(t, highMid, got, "refinement must open the HIGHEST ready chunk (7), reading its committed mid-seq") @@ -115,7 +125,7 @@ func TestDeriveWatermark_RealHotDB_OpensHighestReady(t *testing.T) { // TestDeriveWatermark_RealHotDB_EmptyLiveFallsBack is the count-only-ready case // against a real DB: a "ready" live chunk whose real hot DB has NO committed // ledger (MaxCommittedSeq ok=false) must fall back to deriveCompleteThrough, not -// fabricate a frontier. Read through the production probe. +// fabricate a frontier. Read through a real read-only open by Layout path. func TestDeriveWatermark_RealHotDB_EmptyLiveFallsBack(t *testing.T) { cat, _ := testCatalog(t) makeChunkDurable(t, cat, 0) // cold term => chunk 0 last ledger @@ -124,9 +134,8 @@ func TestDeriveWatermark_RealHotDB_EmptyLiveFallsBack(t *testing.T) { db := openLiveHotDB(t, cat, live) // ready key + real dir, but NOTHING committed require.NoError(t, db.Close()) - // Real probe reads the empty ledgers CF: ok=false, no refinement. - probe := NewRocksHotProbe(cat.Layout().HotChunkPath, silentLogger()) - got, err := deriveWatermark(cat, probe) + // A read-only open of the empty ledgers CF: ok=false, no refinement. + got, err := deriveWatermark(cat, silentLogger()) require.NoError(t, err) require.Equal(t, chunk.ID(2).LastLedger(), got, "empty live DB ⇒ positional baseline (max ready 3 - 1 = chunk 2), no fabricated frontier") diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_shim_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_shim_test.go index cd68fdced..c62afc007 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_shim_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_shim_test.go @@ -1,17 +1,19 @@ package lifecycle import ( - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" + supportlog "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" ) // Test-only aliases for the consolidated progress derivation. The design folded -// deriveCompleteThrough + deriveWatermark into ONE LastCommittedLedger(cat[, probe]): +// deriveCompleteThrough + deriveWatermark into ONE LastCommittedLedger(cat[, logger]): // -// - deriveCompleteThrough(cat) == LastCommittedLedger(cat, nil) (chunk +// - deriveCompleteThrough(cat) == LastCommittedLedger(cat, nil) (chunk // granularity, pure catalog read — the positional term, no hot DB open). -// - deriveWatermark(cat, probe) == LastCommittedLedger(cat, probe) (one -// refinement read of the highest ready hot DB, loss detected LAZILY on it). +// - deriveWatermark(cat, logger) == LastCommittedLedger(cat, logger) (one +// read-only refinement of the highest ready hot DB opened by its Layout path, +// loss detected LAZILY on it). // // These shims keep the tests' intent legible; production callers use // LastCommittedLedger directly. @@ -19,6 +21,6 @@ func deriveCompleteThrough(cat *catalog.Catalog) (uint32, error) { return LastCommittedLedger(cat, nil) } -func deriveWatermark(cat *catalog.Catalog, probe backfill.HotProbe) (uint32, error) { - return LastCommittedLedger(cat, probe) +func deriveWatermark(cat *catalog.Catalog, logger *supportlog.Entry) (uint32, error) { + return LastCommittedLedger(cat, logger) } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go index 13a7472f4..9587fb428 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go @@ -1,7 +1,6 @@ package lifecycle import ( - "errors" "os" "testing" @@ -195,73 +194,40 @@ func TestLastCommittedLedger(t *testing.T) { } // --------------------------------------------------------------------------- -// deriveWatermark — deriveCompleteThrough + one refinement read + the -// per-ready-key dir-existence fatal loop. +// deriveWatermark — deriveCompleteThrough + one read-only refinement of the +// highest ready hot DB, opened lazily by its Layout path. These read REAL +// per-chunk hot DBs; the sub-chunk-precision / opens-highest / empty-fallback +// value cases are covered against real DBs in progress_realdb_test.go. // --------------------------------------------------------------------------- func TestDeriveWatermark(t *testing.T) { t.Run("no ready hot keys => equals deriveCompleteThrough, no open", func(t *testing.T) { cat, _ := testCatalog(t) makeChunkDurable(t, cat, 0) - probe := &fakeHotProbe{} // would error if opened with ok=false under "ready", but none ready - got, err := deriveWatermark(cat, probe) + // No ready key above the cold term ⇒ the hot>cold gate skips the open entirely. + got, err := deriveWatermark(cat, silentLogger()) require.NoError(t, err) require.Equal(t, chunk.ID(0).LastLedger(), got) }) - t.Run("sub-chunk precision: refinement reads mid-chunk seq inside the live chunk", func(t *testing.T) { - cat, _ := testCatalog(t) - readyHot(t, cat, 5) // live chunk 5; positional term = chunk 4 last ledger - midLive := chunk.ID(5).FirstLedger() + 123 - probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{maxSeq: midLive, present: true}} - got, err := deriveWatermark(cat, probe) - require.NoError(t, err) - require.Equal(t, midLive, got, "refined to the live chunk's committed seq") - }) - t.Run("boundary-crash under-count recovered by refinement", func(t *testing.T) { // Live chunk crashed at a boundary and was demoted to "transient": the // highest READY key is the just-completed predecessor (chunk 4), whose // completion no key advertises (positional term = chunk 3). The refinement - // opens chunk 4 and reads its full committed seq = chunk 4's last ledger, - // recovering the frontier the positional term under-counted. + // opens chunk 4's real DB and reads its full committed seq = chunk 4's last + // ledger, recovering the frontier the positional term under-counted. cat, _ := testCatalog(t) - readyHot(t, cat, 4) + chunk4Last := chunk.ID(4).LastLedger() + seedReadyLiveDB(t, cat, 4, chunk4Last) require.NoError(t, cat.PutHotTransient(5)) // the crashed live chunk require.Equal(t, chunk.ID(3).LastLedger(), mustDeriveCompleteThrough(t, cat), "positional term alone under-counts to chunk 3") - chunk4Last := chunk.ID(4).LastLedger() - probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{maxSeq: chunk4Last, present: true}} - got, err := deriveWatermark(cat, probe) + got, err := deriveWatermark(cat, silentLogger()) require.NoError(t, err) require.Equal(t, chunk4Last, got, "refinement recovers the chunk-4 frontier") }) - t.Run("count-only-ready: an empty refinement DB falls back to deriveCompleteThrough", func(t *testing.T) { - cat, _ := testCatalog(t) - makeChunkDurable(t, cat, 0) - readyHot(t, cat, 3) // positional => chunk 2 last ledger - // DB present but empty (present=false): no refinement, w stays positional. - probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{present: false}} - got, err := deriveWatermark(cat, probe) - require.NoError(t, err) - require.Equal(t, chunk.ID(2).LastLedger(), got) - }) - - t.Run("refinement only RAISES the bound, never lowers it", func(t *testing.T) { - cat, _ := testCatalog(t) - makeChunkDurable(t, cat, 0) - makeChunkDurable(t, cat, 1) - makeChunkDurable(t, cat, 2) // cold term => chunk 2 last ledger - readyHot(t, cat, 3) // positional => chunk 2 last ledger - // Live DB reports a seq below the cold bound (e.g. just opened); max wins. - probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{maxSeq: 5, present: true}} - got, err := deriveWatermark(cat, probe) - require.NoError(t, err) - require.Equal(t, chunk.ID(2).LastLedger(), got) - }) - t.Run("LAZY loss (item R2-6): only the highest ready chunk is opened; a lower"+ " ready key's missing dir is NOT eagerly flagged", func(t *testing.T) { cat, _ := testCatalog(t) @@ -271,47 +237,29 @@ func TestDeriveWatermark(t *testing.T) { // later, when ingestion/discard reaches that chunk via openHotDBForChunk. require.NoError(t, cat.PutHotTransient(2)) require.NoError(t, cat.FlipHotReady(2)) // ready key 2, NO dir (not opened here) - readyHot(t, cat, 5) // highest ready key 5 WITH dir (opened) - probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{maxSeq: 10, present: true}} - got, err := deriveWatermark(cat, probe) + highSeq := chunk.ID(5).FirstLedger() + 10 + seedReadyLiveDB(t, cat, 5, highSeq) // highest ready key 5 WITH real DB (opened) + got, err := deriveWatermark(cat, silentLogger()) require.NoError(t, err) - require.Equal(t, uint32(10), got, "refined to the highest ready chunk's seq") + require.Equal(t, highSeq, got, "refined to the highest ready chunk's seq") }) t.Run("errors: a ready HIGHEST chunk whose dir is missing (lazy detection on open)", func(t *testing.T) { cat, _ := testCatalog(t) // The highest ready chunk's dir is missing: the one open the derivation - // performs surfaces an ordinary (restartable) error — the must-exist open + // performs surfaces an ordinary (restartable) error — the read-only open // never auto-heals it into a fresh empty DB. require.NoError(t, cat.PutHotTransient(5)) require.NoError(t, cat.FlipHotReady(5)) // ready key 5, NO dir - probe := &fakeHotProbe{ok: false} // OpenHotChunk reports dir absent - _, err := deriveWatermark(cat, probe) + _, err := deriveWatermark(cat, silentLogger()) require.Error(t, err) require.Contains(t, err.Error(), "00000005") }) - t.Run("errors: refinement open error on the highest ready chunk", func(t *testing.T) { - cat, _ := testCatalog(t) - readyHot(t, cat, 3) // dir present - probe := &fakeHotProbe{openErr: errors.New("rocksdb LOCK held")} - _, err := deriveWatermark(cat, probe) - require.Error(t, err) - }) - - t.Run("errors: refinement read error", func(t *testing.T) { - cat, _ := testCatalog(t) - readyHot(t, cat, 3) - probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{maxErr: errors.New("corrupt")}} - _, err := deriveWatermark(cat, probe) - require.Error(t, err) - }) - t.Run("live chunk 0 ready, empty DB => pre-genesis, no underflow", func(t *testing.T) { cat, _ := testCatalog(t) - readyHot(t, cat, 0) - probe := &fakeHotProbe{ok: true, chunk: &fakeHotChunk{present: false}} - got, err := deriveWatermark(cat, probe) + seedReadyLiveDB(t, cat, 0, 0) // ready + real dir, nothing committed + got, err := deriveWatermark(cat, silentLogger()) require.NoError(t, err) require.Equal(t, preGenesisLedger, got) }) diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index 7b821a062..7b64f41e1 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -44,15 +44,11 @@ func run(ctx context.Context, cfg StartConfig) error { } // Derived, never stored: highest durably-committed ledger (frozen cold artifacts - // vs the highest ready hot DB's max committed seq), clamped by earliest-1. The - // startup probe opens the highest ready hot DB before ingestion opens a writer, - // so production uses a recovery probe that replays any synced WAL from an - // ungraceful crash before MaxCommittedSeq is read. - hotProgressProbe := cfg.HotProgressProbe - if hotProgressProbe == nil { - hotProgressProbe = cfg.Exec.Process.HotProbe - } - lastCommitted, err := lifecycle.LastCommittedLedger(cat, hotProgressProbe) + // vs the highest ready hot DB's max committed seq), clamped by earliest-1. Passing + // the logger refines with one read-only open of the highest ready hot DB before + // ingestion opens a writer; a read-only open replays any synced WAL from an + // ungraceful crash into memtables, so MaxCommittedSeq is correct. + lastCommitted, err := lifecycle.LastCommittedLedger(cat, logger) if err != nil { return fmt.Errorf("startup derive last-committed: %w", err) } @@ -273,11 +269,6 @@ type StartConfig struct { // Exec drives backfill's RunBackfill; its Catalog/Logger are the shared ones. Exec backfill.ExecConfig - // HotProgressProbe refines startup's last-committed ledger from the highest - // ready hot DB. Production uses a read-write recovery probe so RocksDB replays - // synced WAL after crashes; nil falls back to Exec.Process.HotProbe for tests. - HotProgressProbe backfill.HotProbe - // RetentionChunks bounds the sliding retention floor's width — the backfill // floor's width too (0 ⇒ the earliest-ledger floor only). run() assembles the // lifecycle.Config from Exec + this, so the lifecycle and backfill can never @@ -332,9 +323,6 @@ func (cfg StartConfig) validate() error { if cfg.Exec.Logger == nil { return errors.New("nil StartConfig.Exec.Logger") } - if cfg.Exec.Process.HotProbe == nil { - return errors.New("nil StartConfig.Exec.Process.HotProbe (last-committed derivation needs it)") - } if cfg.NetworkTip == nil { return errors.New("nil StartConfig.NetworkTip") } diff --git a/cmd/stellar-rpc/internal/fullhistory/startup_test.go b/cmd/stellar-rpc/internal/fullhistory/startup_test.go index 7e6f0b542..8cb615b1a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup_test.go @@ -88,9 +88,7 @@ func startTestConfig( Catalog: cat, Logger: silentLogger(), Workers: 2, - Process: backfill.ProcessConfig{ - HotProbe: NewRocksHotProbe(cat.Layout().HotChunkPath, silentLogger()), - }, + Process: backfill.ProcessConfig{}, } cfg := StartConfig{ Exec: exec, From 4ad32fb7ce3e2b1fdc101c08664926e1cbd3766f Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 15:47:40 -0400 Subject: [PATCH 32/55] =?UTF-8?q?fullhistory:=20ingestion-loop=20rework=20?= =?UTF-8?q?=E2=80=94=20raw=20stream=20in,=20atomic=20cell=20out=20(#17,=20?= =?UTF-8?q?#33,=20#21)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live ingestion loop was the one seam still polling decoded structs and coordinating with the lifecycle through a bounded doorbell. Rework it into a single sequence-validated stream consumer with a latest-cell handoff. - Raw stream in: OpenCore returns a ledgerbackend.LedgerStream (captive core via NewCaptiveCoreStream, which owns the core process lifecycle — started on the first pull, torn down when the loop exits). The loop ranges over RawLedgers(UnboundedRange(resume)), consuming the cached raw frame directly. Deletes LedgerGetter + backendGetter and the GetLedger->MarshalBinary round-trip. - Shared seq-validated cursor: ingest.SeqValidatedCursor validates contiguity (gap / duplicate / out-of-order) once, consumed by BOTH the cold drain and the hot loop — so the sole writer of recent history no longer trusts its source blindly. Renames HotIngester -> LedgerIngester (drain's per-ledger consumer; #33 falls out of the cursor extraction). - Resume passed in once: run() derives resume (lastCommitted+1); the loop asserts it against the hot DB's implied resume (nextIngestLedger) instead of re-deriving independently. - Atomic cell out: lifecycle.BoundarySignal (latest-cell + 1-buf wake) replaces the depth-8 channel doorbell and its "fell behind" logger.Fatalf. A slow lifecycle can't overflow a cell, so the fatal path is gone. - Loop config struct replaces eight positional params. - #21 (folded in): the loop is documented as the hot-tier OWNER — the sole writer that opens/closes/hands off per-chunk hot DBs. The live-chunk exclusion for the read-only freeze/refinement opens is a correctness invariant kept here by the handoff fence + the tick targeting <= highest-complete, not a lock. Tests: fakeLedgerGetter -> fakeCoreStream (a LedgerStream); fakeCore/e2eCore return streams; boundary handoff observed via a recordingBoundary; resume asserted via the stream's recorded From(). Whole fullhistory tree builds + vets; ingest / lifecycle / backfill / root pass -short; the non-short E2E passes (~100s). --- .../internal/fullhistory/daemon.go | 51 +--- .../internal/fullhistory/e2e_test.go | 86 ++++--- .../internal/fullhistory/hotloop.go | 215 +++++++++-------- .../internal/fullhistory/hotloop_test.go | 226 +++++++++++------- .../internal/fullhistory/ingest/doc.go | 2 +- .../internal/fullhistory/ingest/driver.go | 92 ++++--- .../internal/fullhistory/ingest/ingester.go | 29 ++- .../fullhistory/lifecycle/lifecycle.go | 72 ++++-- .../lifecycle/lifecycle_loop_test.go | 34 +-- .../internal/fullhistory/startup.go | 67 +++--- .../internal/fullhistory/startup_test.go | 41 ++-- 11 files changed, 517 insertions(+), 398 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index 6b9acb0fa..a3be66c88 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -15,7 +15,6 @@ import ( "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" supportlog "github.com/stellar/go-stellar-sdk/support/log" - "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/daemon/interfaces" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" @@ -286,9 +285,9 @@ func buildBackfillBackend( // --------------------------------------------------------------------------- // captiveCoreOpener is the production CoreOpener. It holds a resolved -// CaptiveCoreConfig and builds a FRESH captive-core ledgerbackend per run (each -// supervised restart reopens core anew), prepares it at the resume ledger, and -// hands back a LedgerGetter the ingestion loop polls by sequence plus a closer. +// CaptiveCoreConfig and hands back a captive-core LedgerStream that builds a FRESH +// core per run (each supervised restart reopens core anew) — the stream owns the +// process lifecycle, so there is no eager prepare or explicit closer here. // Construction mirrors the RPC daemon's newCaptiveCore so the full-history daemon // runs captive core and the ledgerbackend the same way (#772 can unify them at // the cutover). @@ -373,42 +372,17 @@ func newCaptiveCoreOpener(ing IngestionConfig, dataDir string, logger *supportlo }, nil } -// OpenCore builds a fresh captive-core backend, prepares it over the unbounded -// range from resumeLedger, and returns a getter wrapping GetLedger plus the -// backend's Close. A fresh backend per call keeps supervised restarts clean (the -// prior run's core was closed on its way out). -func (c *captiveCoreOpener) OpenCore( - ctx context.Context, resumeLedger uint32, -) (LedgerGetter, func() error, error) { +// OpenCore returns the live ingestion stream backed by captive stellar-core. The +// stream OWNS the core process lifecycle — a fresh core is started on the first +// RawLedgers pull and torn down when iteration ends (the ingestion loop exits) — +// so there is no eager PrepareRange and no separate closer here; a fresh core per +// run keeps supervised restarts clean. The loop pulls RawLedgers over the +// unbounded range from its resume ledger, consuming the cached raw frame directly +// (no GetLedger→MarshalBinary round-trip). +func (c *captiveCoreOpener) OpenCore(ctx context.Context) (ledgerbackend.LedgerStream, error) { cfg := c.config cfg.Context = ctx - backend, err := ledgerbackend.NewCaptive(cfg) - if err != nil { - return nil, nil, fmt.Errorf("build captive core: %w", err) - } - if err := backend.PrepareRange(ctx, ledgerbackend.UnboundedRange(resumeLedger)); err != nil { - _ = backend.Close() - return nil, nil, fmt.Errorf("captive core prepare range from %d: %w", resumeLedger, err) - } - return backendGetter{backend: backend}, backend.Close, nil -} - -// backendGetter adapts a ledgerbackend.LedgerBackend to LedgerGetter: GetLedger -// blocks until the ledger is available and returns its raw wire bytes. -type backendGetter struct { - backend ledgerbackend.LedgerBackend -} - -func (g backendGetter) GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseMetaView, error) { - lcm, err := g.backend.GetLedger(ctx, seq) - if err != nil { - return nil, err - } - raw, err := lcm.MarshalBinary() - if err != nil { - return nil, fmt.Errorf("marshal ledger %d: %w", seq, err) - } - return xdr.LedgerCloseMetaView(raw), nil + return ledgerbackend.NewCaptiveCoreStream(cfg, c.config.Log), nil } // resolveNetworkTip adapts the backfill backend to backfill's tip sampler — its Tip @@ -454,7 +428,6 @@ func newLogger(cfg LoggingConfig) (*supportlog.Entry, error) { // compile-time interface checks. var ( _ CoreOpener = (*captiveCoreOpener)(nil) - _ LedgerGetter = backendGetter{} _ NetworkTipBackend = notConfiguredTip{} _ NetworkTipBackend = backendTip{} ) diff --git a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go index 186af8527..a2f754297 100644 --- a/cmd/stellar-rpc/internal/fullhistory/e2e_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/e2e_test.go @@ -10,7 +10,7 @@ package fullhistory // validateConfig gate (pins the floor), and the supervised run loop. // - run → backfillToTip → openHotDBForChunk → runIngestionLoop (the real // atomic per-ledger WriteBatch across all CFs of the real per-chunk -// hotchunk RocksDB), the real boundary handoff, the real doorbell. +// hotchunk RocksDB), the real boundary handoff, the real boundary signal. // - lifecycle.Loop / runLifecycle: the real resolve + executePlan // freeze (cold artifacts derived FROM the live hot DB), the real txhash // index fold (a real streamhash .idx on disk), the real discard + prune. @@ -34,6 +34,7 @@ package fullhistory import ( "context" "fmt" + "iter" "os" "path/filepath" "sync" @@ -44,6 +45,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" "github.com/stellar/go-stellar-sdk/keypair" "github.com/stellar/go-stellar-sdk/xdr" @@ -57,51 +59,60 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash" ) -// e2eCore is the CoreOpener handing back a fresh e2eGetter per daemon run (a -// restart opens core anew). frames is the seq→raw backlog every getter serves; +// e2eCore is the CoreOpener handing back a fresh e2eStream per daemon run (a +// restart opens core anew). frames is the seq→raw backlog every stream serves; // the atomics aggregate observations across opens for the restart assertions. type e2eCore struct { - frames map[uint32][]byte - resumeSeen atomic.Uint32 - fromSeen atomic.Uint32 - delivered atomic.Uint32 - opens atomic.Int32 + frames map[uint32][]byte + fromSeen atomic.Uint32 + delivered atomic.Uint32 + opens atomic.Int32 } -func (c *e2eCore) OpenCore(_ context.Context, resume uint32) (LedgerGetter, func() error, error) { +func (c *e2eCore) OpenCore(context.Context) (ledgerbackend.LedgerStream, error) { c.opens.Add(1) - c.resumeSeen.Store(resume) - return &e2eGetter{core: c}, func() error { return nil }, nil + return &e2eStream{core: c}, nil } -// e2eGetter is the FAKE captive-core ledger getter: a resumable LedgerGetter the -// ingestion loop polls by sequence. It returns the frame for the requested seq -// when its core has one, and once the poll runs past the synthetic backlog it -// blocks until ctx is canceled (a live tip stream ends only on shutdown). It -// records (into its core) the FIRST seq it was asked for, so the restart step can -// assert the daemon re-derived the watermark and resumed with no gap. -type e2eGetter struct { +// e2eStream is the FAKE captive-core LedgerStream the ingestion loop consumes: it +// yields the backlog frames contiguously from the range's From() and, once it runs +// past the synthetic backlog, blocks until ctx is canceled (a live tip stream ends +// only on shutdown). It records (into its core) the FIRST seq it was asked for +// (the range From), so the restart step can assert the daemon re-derived the +// watermark and resumed with no gap. +type e2eStream struct { core *e2eCore sawFrom atomic.Bool } -var _ LedgerGetter = (*e2eGetter)(nil) +var _ ledgerbackend.LedgerStream = (*e2eStream)(nil) -func (s *e2eGetter) GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseMetaView, error) { - if s.sawFrom.CompareAndSwap(false, true) { - s.core.fromSeen.Store(seq) - } - if ctx.Err() != nil { - return nil, ctx.Err() - } - if raw, ok := s.core.frames[seq]; ok { - s.core.delivered.Store(seq) - return xdr.LedgerCloseMetaView(raw), nil +func (s *e2eStream) RawLedgers( + ctx context.Context, r ledgerbackend.Range, _ ...ledgerbackend.StreamOption, +) iter.Seq2[[]byte, error] { + return func(yield func([]byte, error) bool) { + if s.sawFrom.CompareAndSwap(false, true) { + s.core.fromSeen.Store(r.From()) + } + for seq := r.From(); ; seq++ { + if ctx.Err() != nil { + yield(nil, ctx.Err()) + return + } + if raw, ok := s.core.frames[seq]; ok { + s.core.delivered.Store(seq) + if !yield(raw, nil) { + return + } + continue + } + // Past the synthetic backlog: a live tip blocks until shutdown so the loop + // does not see an error that would look like a core crash. + <-ctx.Done() + yield(nil, ctx.Err()) + return + } } - // Past the synthetic backlog: a live tip blocks until shutdown so the loop - // does not see an error that would look like a core crash. - <-ctx.Done() - return nil, ctx.Err() } // e2eMetrics is a concurrency-safe observability.Metrics that records the @@ -335,8 +346,8 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing }, 60*time.Second, 50*time.Millisecond, "the boundary ticks must freeze+fold+discard chunks 0 and 1") require.GreaterOrEqual(t, served.Load(), int32(1), "reads were served") - require.Equal(t, c0First, core.resumeSeen.Load(), - "first start resumes captive core at genesis (watermark+1)") + require.Equal(t, c0First, core.fromSeen.Load(), + "first start resumes the ingestion stream at genesis (watermark+1)") // ===================================================================== // STEP 2 — clean shutdown. The supervised loop returns nil on ctx cancel. @@ -424,7 +435,6 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing // ===================================================================== closePost() // release the inspection metastore handle before the daemon reopens it core.opens.Store(0) - core.resumeSeen.Store(0) core.fromSeen.Store(0) cancel2, done2 := runDaemonInBackground(t, cfgPath, core, &served, &e2eMetrics{}) @@ -434,10 +444,8 @@ func TestE2E_DaemonLifecycle_FirstStartIngestFreezeLookupRestartPrune(t *testing "the restarted ingestion loop requested a resume range") wantResume := wmBeforeRestart + 1 - assert.Equal(t, wantResume, core.resumeSeen.Load(), - "restart resumes captive core at the re-derived watermark+1 (no gap, no re-fetch of the bottom)") assert.Equal(t, wantResume, core.fromSeen.Load(), - "the ingestion loop streamed from watermark+1 — the durable frontier, re-derived not stored") + "restart streams from the re-derived watermark+1 — the durable frontier, re-derived not stored, no gap") waitClean(t, cancel2, done2) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index b1e74a1e8..3cb7cb9ee 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -6,32 +6,25 @@ import ( "os" "path/filepath" + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" supportlog "github.com/stellar/go-stellar-sdk/support/log" - "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/ingest" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/observability" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" ) -// The hot-DB ingestion loop (decision (a)). One goroutine polls ledgers by seq -// (core.GetLedger) into the per-chunk shared multi-CF hot DB, committing each as -// one atomic synced WriteBatch across all CFs. It keeps NO progress variable — -// the last synced batch IS the last-committed ledger, re-derived at startup. Its only -// coupling to the lifecycle is the channel: at each boundary it sends the -// just-completed chunk id (the two goroutines share no memory). Clean-shutdown vs -// crash is decided at the daemon top level (a ctx-canceled return is clean). - -// LedgerGetter is the indexed-poll source the ingestion loop drives: it returns -// one ledger's view, blocking until that ledger is available (the design's -// core.GetLedger(ctx, seq)). Production wraps captive core; tests pass a fake. -type LedgerGetter interface { - GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseMetaView, error) -} +// The hot-DB ingestion loop (decision (a)). One goroutine consumes a single +// sequence-validated ledger stream into the per-chunk shared multi-CF hot DB, +// committing each ledger as one atomic synced WriteBatch across all CFs. It keeps +// NO progress variable — the last synced batch IS the last-committed ledger, +// re-derived at startup. Its only coupling to the lifecycle is the boundary +// signal: at each boundary it publishes the just-completed chunk id (the two +// goroutines share no memory). Clean-shutdown vs crash is decided at the daemon +// top level (a ctx-canceled return is clean). // openHotDBForChunk opens/recovers/creates the chunk's shared hot DB, keyed on // the durable hot:chunk state: @@ -88,44 +81,66 @@ func openHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlo return db, nil } -// runIngestionLoop polls core for LCMs by seq into hotDB (one atomic synced -// WriteBatch each), and at each chunk boundary hands the frontier forward by -// closing the just-filled DB and opening the next. It never returns nil; the -// daemon classifies a ctx-canceled return as clean shutdown, any other as -// RESTARTABLE (startup re-derives the last-committed ledger, losing nothing). +// boundaryPublisher is the ingestion loop's handoff sink: it publishes the +// just-completed chunk id to the lifecycle at each boundary. +// *lifecycle.BoundarySignal is the production impl; tests inject a recorder. +type boundaryPublisher interface { + Publish(c chunk.ID) +} + +// ingestionLoopConfig bundles the ingestion loop's dependencies (previously eight +// positional params). +type ingestionLoopConfig struct { + Stream ledgerbackend.LedgerStream + Resume uint32 + HotDB *hotchunk.DB + Catalog *catalog.Catalog + Boundary boundaryPublisher + Logger *supportlog.Entry + Metrics observability.Metrics + Sink ingest.MetricSink +} + +// runIngestionLoop is the hot tier's OWNER: the single goroutine that opens, +// writes, closes, and hands off the per-chunk hot DBs. It consumes ONE continuous +// sequence-validated ledger stream from Resume (the stream owns the captive-core +// process — started on the first pull, torn down when this loop exits), commits +// each ledger as one atomic synced WriteBatch (decision (a)), and at each chunk +// boundary closes the just-filled DB, opens the next, and publishes the completed +// chunk to the lifecycle. A ctx-canceled return is a clean shutdown; any other +// error is RESTARTABLE (startup re-derives the last-committed ledger, losing nothing). // -// HANDOFF FENCE: the DB is CLOSED before the next chunk's hot:chunk key is -// created — that key is what makes THIS chunk complete to the lifecycle, which -// could then discard a dir a still-live writer holds. notify() fires only after -// the next DB is open. The HotService is rebuilt each boundary. -func runIngestionLoop( - ctx context.Context, - core LedgerGetter, - hotDB *hotchunk.DB, - cat *catalog.Catalog, - lifecycleCh chan<- chunk.ID, - logger *supportlog.Entry, - metrics observability.Metrics, - sink ingest.MetricSink, -) (err error) { - metrics = observability.MetricsOrNop(metrics) - - // notify hands the just-completed chunk id to the lifecycle. A FULL buffer - // (LifecycleQueueDepth) means freeze has fallen that many boundaries behind — - // fail loud (a wedged lifecycle ingesting on cannot recover). - notify := func(complete chunk.ID) { - select { - case lifecycleCh <- complete: - default: - logger.Fatalf("lifecycle fell %d boundaries behind ingestion; investigate", - lifecycle.LifecycleQueueDepth) - } +// HANDOFF FENCE: the DB is CLOSED before the next chunk's hot:chunk key is created +// — that key is what makes THIS chunk complete to the lifecycle, which could then +// discard a dir a still-live writer holds. Publish fires only after the next DB is +// open. The HotService is rebuilt each boundary. +// +// LIVE-CHUNK EXCLUSION (one home): this loop is the SOLE writer of a chunk's hot +// DB, and closes the live DB before publishing the completed chunk (the fence +// above). The lifecycle tick only ever targets chunks at or below the highest +// durably-complete chunk — strictly below the live chunk — so the read-only freeze +// and watermark-refinement opens never touch a DB this loop holds. A read-only +// open skips the RocksDB LOCK, so that separation is a correctness invariant kept +// here in the producer by construction, not a lock the readers rely on. +func runIngestionLoop(ctx context.Context, cfg ingestionLoopConfig) (err error) { + metrics := observability.MetricsOrNop(cfg.Metrics) + hotDB := cfg.HotDB + + // Startup assertion: the resume passed in must equal what the live hot DB + // implies. run() derives resume (lastCommitted+1) and opens this DB; the two + // always agree, but only by case analysis — so assert it and fail loudly on a + // disagreement (a bug), rather than silently trusting one over the other. + if implied, ierr := nextIngestLedger(hotDB); ierr != nil { + return fmt.Errorf("derive resume assertion: %w", ierr) + } else if implied != cfg.Resume { + return fmt.Errorf("resume ledger %d disagrees with hot DB %s implied resume %d", + cfg.Resume, hotDB.ChunkID(), implied) } // The loop is hotDB's single writer and reopens it at every boundary. On any // exit, close the live handle so the rocksdb instance does not leak (the - // boundary handoff already closed every prior chunk's DB); no writer races - // this close (the loop has stopped on every exit path). + // boundary handoff already closed every prior chunk's DB); no writer races this + // close (the loop has stopped on every exit path). defer func() { if hotDB != nil { if cerr := hotDB.Close(); cerr != nil && err == nil { @@ -134,66 +149,68 @@ func runIngestionLoop( } }() - // Resume point: one past the live chunk's durable last-committed ledger (re-derived, not - // stored — a re-delivered committed ledger is an idempotent retry). - resume, err := nextIngestLedger(hotDB) - if err != nil { - return fmt.Errorf("derive resume ledger: %w", err) - } - - // hotService binds the metrics sink to THIS hotDB instance; the boundary - // handoff rebuilds it for the reopened chunk DB below. - hotService := ingest.NewHotService(hotDB, sink) - - // Indexed poll from the resume ledger. GetLedger blocks until seq is - // available; its error ends the loop for the daemon top level to classify. - for seq := resume; ; seq++ { - lcm, gerr := core.GetLedger(ctx, seq) - if gerr != nil { - return fmt.Errorf("get ledger %d: %w", seq, gerr) + // hotService binds the metrics sink to THIS hotDB instance; the boundary handoff + // rebuilds it for the reopened chunk DB below. + hotService := ingest.NewHotService(hotDB, cfg.Sink) + + // One continuous sequence-validated stream from the resume ledger. The cursor + // restores the per-ledger sequence guard the cold drain also uses (defense in + // depth against a mis-keyed source writing the sole copy of recent history). A + // stream / decode / sequence error ends the loop for the daemon to classify. + raw := cfg.Stream.RawLedgers(ctx, ledgerbackend.UnboundedRange(cfg.Resume)) + for vl, verr := range ingest.SeqValidatedCursor(raw, cfg.Resume) { + if verr != nil { + return fmt.Errorf("ingestion stream: %w", verr) } - // One atomic synced WriteBatch across all hot CFs (via - // hotDB.IngestLedger), reporting per-type LedgerCounts to the sink. - if ierr := hotService.Ingest(ctx, seq, lcm); ierr != nil { - return fmt.Errorf("ingest ledger %d: %w", seq, ierr) + // One atomic synced WriteBatch across all hot CFs (via hotDB.IngestLedger), + // reporting per-type LedgerCounts to the sink. + if ierr := hotService.Ingest(ctx, vl.Seq, vl.View); ierr != nil { + return fmt.Errorf("ingest ledger %d: %w", vl.Seq, ierr) } // Chunk boundary: this seq is the chunk's last ledger. - closed := chunk.IDFromLedger(seq) - if seq == closed.LastLedger() { - next := closed + 1 - // Handoff fence: close the write handle BEFORE the next chunk's key is - // created (that key is what makes THIS chunk complete to a tick, which - // may then freeze and discard its hot DB — no writer may hold it then). - if cerr := hotDB.Close(); cerr != nil { - hotDB = nil // closed (failed) — do not double-close in defer - return fmt.Errorf("close hot DB at boundary chunk %s: %w", closed, cerr) - } - hotDB = nil // released; reopen below republishes it for the defer + closed := chunk.IDFromLedger(vl.Seq) + if vl.Seq != closed.LastLedger() { + continue + } + next := closed + 1 + // Handoff fence: close the write handle BEFORE the next chunk's key is + // created (that key is what makes THIS chunk complete to a tick, which may + // then freeze and discard its hot DB — no writer may hold it then). + if cerr := hotDB.Close(); cerr != nil { + hotDB = nil // closed (failed) — do not double-close in defer + return fmt.Errorf("close hot DB at boundary chunk %s: %w", closed, cerr) + } + hotDB = nil // released; reopen below republishes it for the defer - nextDB, oerr := openHotDBForChunk(cat, next, logger) - if oerr != nil { - return fmt.Errorf("open hot DB for chunk %s at boundary: %w", next, oerr) - } - hotDB = nextDB - hotService = ingest.NewHotService(hotDB, sink) - // next's key (created inside openHotDBForChunk) moved the partition; - // only now notify the lifecycle of the completed chunk. - notify(closed) - - // Boundary observability (the woken tick reports the freeze/discard/prune). - metrics.ChunkBoundary() - logger.WithField("closed_chunk", closed.String()). - WithField("next_chunk", next.String()). - WithField("last_ledger", seq). - Info("streaming: ingestion chunk boundary — handed off to lifecycle") + nextDB, oerr := openHotDBForChunk(cfg.Catalog, next, cfg.Logger) + if oerr != nil { + return fmt.Errorf("open hot DB for chunk %s at boundary: %w", next, oerr) } + hotDB = nextDB + hotService = ingest.NewHotService(hotDB, cfg.Sink) + // next's key (created inside openHotDBForChunk) moved the partition; only now + // publish the completed chunk to the lifecycle. + cfg.Boundary.Publish(closed) + + // Boundary observability (the woken tick reports the freeze/discard/prune). + metrics.ChunkBoundary() + cfg.Logger.WithField("closed_chunk", closed.String()). + WithField("next_chunk", next.String()). + WithField("last_ledger", vl.Seq). + Info("streaming: ingestion chunk boundary — handed off to lifecycle") } + // The unbounded stream only ends on ctx cancellation or a source error, both + // surfaced as the cursor's error element above; a nil return here means the + // source stopped cleanly (no more ledgers, no error). + return nil } -// nextIngestLedger is the resume point for a just-opened live hot DB: one past -// its authoritative last-committed ledger, or the bound chunk's first ledger on an empty DB. +// nextIngestLedger is the resume point a live hot DB implies: one past its +// authoritative last-committed ledger, or the bound chunk's first ledger on an +// empty DB. run() derives the same value independently (lastCommitted+1); +// runIngestionLoop asserts the two agree. func nextIngestLedger(db *hotchunk.DB) (uint32, error) { maxSeq, ok, err := db.MaxCommittedSeq() if err != nil { diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index cb61bd007..991eeca0d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -3,7 +3,9 @@ package fullhistory import ( "context" "errors" + "iter" "os" + "sync" "sync/atomic" "testing" "time" @@ -11,73 +13,122 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/stellar/go-stellar-sdk/xdr" + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" ) // --------------------------------------------------------------------------- -// fakeLedgerGetter — an injectable LedgerGetter the ingestion loop polls by -// sequence (the design's indexed core.GetLedger(ctx, seq)). For seqs it has a -// programmed frame it returns those bytes; once the poll runs past the last +// fakeCoreStream — an injectable ledgerbackend.LedgerStream the ingestion loop +// consumes (the design's raw captive-core stream). RawLedgers yields programmed +// frames contiguously from the range's From(); once it runs past the last // programmed seq it either blocks until ctx is canceled (a live tip stream that -// only ends on shutdown) or returns endErr (a crashed backend). It records the -// FIRST seq it was asked for (the restart resume point) and the GetLedger call -// count. +// only ends on shutdown) or yields endErr (a crashed backend). It records the +// FIRST seq it was asked for (the loop's resume point) and a per-seq consideration +// count so a test can wait for the loop to reach the blocking pull. // --------------------------------------------------------------------------- -type fakeLedgerGetter struct { +type fakeCoreStream struct { frames map[uint32][]byte // seq -> raw LCM bytes - maxSeq uint32 // highest programmed seq blockOnCtx bool // past the last frame, block until ctx.Done - endErr error // past the last frame, return this (when not blocking) - yieldErrAt uint32 // if non-zero, return errAt at this seq instead of bytes + endErr error // past the last frame, yield this (when not blocking) + yieldErrAt uint32 // if non-zero, yield errAt at this seq instead of bytes errAt error - calls atomic.Int32 + calls atomic.Int32 // seqs considered (mirrors the old per-GetLedger count) firstSeen atomic.Uint32 sawFirst atomic.Bool } -var _ LedgerGetter = (*fakeLedgerGetter)(nil) +var _ ledgerbackend.LedgerStream = (*fakeCoreStream)(nil) -func (g *fakeLedgerGetter) GetLedger(ctx context.Context, seq uint32) (xdr.LedgerCloseMetaView, error) { - g.calls.Add(1) - if g.sawFirst.CompareAndSwap(false, true) { - g.firstSeen.Store(seq) - } - if ctx.Err() != nil { - return nil, ctx.Err() - } - if g.yieldErrAt != 0 && seq == g.yieldErrAt { - return nil, g.errAt - } - if raw, ok := g.frames[seq]; ok { - return xdr.LedgerCloseMetaView(raw), nil - } - // Past the programmed frames. - if g.blockOnCtx { - <-ctx.Done() - return nil, ctx.Err() - } - if g.endErr != nil { - return nil, g.endErr +func (s *fakeCoreStream) RawLedgers( + ctx context.Context, r ledgerbackend.Range, _ ...ledgerbackend.StreamOption, +) iter.Seq2[[]byte, error] { + return func(yield func([]byte, error) bool) { + if s.sawFirst.CompareAndSwap(false, true) { + s.firstSeen.Store(r.From()) + } + for seq := r.From(); ; seq++ { + s.calls.Add(1) + if ctx.Err() != nil { + yield(nil, ctx.Err()) + return + } + if s.yieldErrAt != 0 && seq == s.yieldErrAt { + yield(nil, s.errAt) + return + } + if raw, ok := s.frames[seq]; ok { + if !yield(raw, nil) { + return + } + continue + } + // Past the programmed frames. + if s.blockOnCtx { + <-ctx.Done() + yield(nil, ctx.Err()) + return + } + if s.endErr != nil { + yield(nil, s.endErr) + return + } + yield(nil, errors.New("fakeCoreStream: no frame for seq")) + return + } } - return nil, errors.New("fakeLedgerGetter: no frame for seq") } -// getterForSeqs builds a fakeLedgerGetter with zero-tx LCM frames for [from,to]. -func getterForSeqs(t *testing.T, from, to uint32) *fakeLedgerGetter { +// streamForSeqs builds a fakeCoreStream with zero-tx LCM frames for [from,to]. +func streamForSeqs(t *testing.T, from, to uint32) *fakeCoreStream { t.Helper() - g := &fakeLedgerGetter{frames: map[uint32][]byte{}, maxSeq: to} + s := &fakeCoreStream{frames: map[uint32][]byte{}} for seq := from; seq <= to; seq++ { - g.frames[seq] = zeroTxLCMBytes(t, seq) + s.frames[seq] = zeroTxLCMBytes(t, seq) } - return g + return s +} + +// recordingBoundary is a test boundaryPublisher capturing the completed chunk ids +// the loop publishes at each boundary, so a test can assert the handoff without +// wiring a real lifecycle Loop. +type recordingBoundary struct { + mu sync.Mutex + ids []chunk.ID +} + +func (r *recordingBoundary) Publish(c chunk.ID) { + r.mu.Lock() + defer r.mu.Unlock() + r.ids = append(r.ids, c) +} + +func (r *recordingBoundary) list() []chunk.ID { + r.mu.Lock() + defer r.mu.Unlock() + return append([]chunk.ID(nil), r.ids...) +} + +// loopConfig builds an ingestionLoopConfig for a test: the stream + hot DB + a +// recording boundary, with Resume derived from the DB (the value the loop asserts). +func loopConfig(t *testing.T, stream ledgerbackend.LedgerStream, db *hotchunk.DB, cat *catalog.Catalog) (ingestionLoopConfig, *recordingBoundary) { + t.Helper() + resume, err := nextIngestLedger(db) + require.NoError(t, err) + rec := &recordingBoundary{} + return ingestionLoopConfig{ + Stream: stream, + Resume: resume, + HotDB: db, + Catalog: cat, + Boundary: rec, + Logger: silentLogger(), + }, rec } // openLiveHotDB opens (and brackets ready) the live hot DB for a chunk via the @@ -194,13 +245,13 @@ func TestRunIngestionLoop_LedgerLandsAcrossAllCFs(t *testing.T) { db := openLiveHotDB(t, cat, c) // A short contiguous prefix from the chunk's first ledger (events require - // strict contiguity from FirstLedger), then the poll runs dry and errs. - getter := getterForSeqs(t, first, first+2) - getter.endErr = errors.New("backend crashed") - ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) + // strict contiguity from FirstLedger), then the stream runs dry and errs. + stream := streamForSeqs(t, first, first+2) + stream.endErr = errors.New("backend crashed") + cfg, _ := loopConfig(t, stream, db, cat) - err := runIngestionLoop(context.Background(), getter, db, cat, ch, silentLogger(), nil, nil) - require.Error(t, err, "poll ran past the prefix and the getter errored") + err := runIngestionLoop(context.Background(), cfg) + require.Error(t, err, "stream ran past the prefix and errored") // Reopen the (loop-closed) DB and assert every CF advanced together. reopened, err := hotchunk.Open(cat.Layout().HotChunkPath(c), c, silentLogger()) @@ -223,10 +274,8 @@ func TestRunIngestionLoop_LedgerLandsAcrossAllCFs(t *testing.T) { // --------------------------------------------------------------------------- // TestRunIngestionLoop_BoundaryNotifiesCompletedChunk: crossing the chunk 0 -> 1 -// boundary sends chunk 0 into the buffered lifecycle channel. The watermark is -// seeded just below the boundary so the poll crosses it in one step. The buffer -// is far above the at-most-one a healthy daemon holds, so it never blocks the -// loop. +// boundary publishes chunk 0 to the lifecycle. The watermark is seeded just below +// the boundary so the stream crosses it in one step. func TestRunIngestionLoop_BoundaryNotifiesCompletedChunk(t *testing.T) { t.Parallel() // seeds a near-full chunk (one synced commit per ledger) cat, _ := testCatalog(t) @@ -234,26 +283,25 @@ func TestRunIngestionLoop_BoundaryNotifiesCompletedChunk(t *testing.T) { c1 := c + 1 db := seedWatermark(t, cat, c, c.LastLedger()-1) - getter := &fakeLedgerGetter{frames: map[uint32][]byte{ + stream := &fakeCoreStream{frames: map[uint32][]byte{ c.LastLedger(): zeroTxLCMBytes(t, c.LastLedger()), // boundary 0->1 c1.FirstLedger(): zeroTxLCMBytes(t, c1.FirstLedger()), // a ledger in chunk 1 }, endErr: errors.New("end")} - ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) + cfg, rec := loopConfig(t, stream, db, cat) done := make(chan error, 1) go func() { - done <- runIngestionLoop(context.Background(), getter, db, cat, ch, silentLogger(), nil, nil) + done <- runIngestionLoop(context.Background(), cfg) }() select { case err := <-done: - require.Error(t, err, "poll ran dry") + require.Error(t, err, "stream ran dry") case <-time.After(10 * time.Second): t.Fatal("ingestion loop deadlocked") } - sent := drainLifecycle(ch) - assert.Equal(t, []chunk.ID{c}, sent, "the completed chunk id was sent at the boundary") + assert.Equal(t, []chunk.ID{c}, rec.list(), "the completed chunk id was published at the boundary") } // --------------------------------------------------------------------------- @@ -261,8 +309,8 @@ func TestRunIngestionLoop_BoundaryNotifiesCompletedChunk(t *testing.T) { // level: ctx-canceled return is clean, any other error is restartable). // --------------------------------------------------------------------------- -// TestRunIngestionLoop_CtxCancelReturnsCtxErr: a ctx cancellation while the poll -// is blocking on the tip makes GetLedger return ctx.Err(); the loop returns that +// TestRunIngestionLoop_CtxCancelReturnsCtxErr: a ctx cancellation while the stream +// is blocking on the tip makes RawLedgers yield ctx.Err(); the loop returns that // (the daemon top level classifies a ctx-canceled return as a clean shutdown). func TestRunIngestionLoop_CtxCancelReturnsCtxErr(t *testing.T) { cat, _ := testCatalog(t) @@ -270,45 +318,45 @@ func TestRunIngestionLoop_CtxCancelReturnsCtxErr(t *testing.T) { first := c.FirstLedger() db := openLiveHotDB(t, cat, c) - getter := getterForSeqs(t, first, first+1) - getter.blockOnCtx = true // after the frames, behave like a live tip stream - ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) + stream := streamForSeqs(t, first, first+1) + stream.blockOnCtx = true // after the frames, behave like a live tip stream + cfg, _ := loopConfig(t, stream, db, cat) ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { - done <- runIngestionLoop(ctx, getter, db, cat, ch, silentLogger(), nil, nil) + done <- runIngestionLoop(ctx, cfg) }() require.Eventually(t, func() bool { - return getter.calls.Load() >= 3 // ingested 2 frames, blocked on the 3rd + return stream.calls.Load() >= 3 // ingested 2 frames, blocked on the 3rd }, 5*time.Second, 5*time.Millisecond) cancel() select { case err := <-done: require.Error(t, err) - require.ErrorIs(t, err, context.Canceled, "the loop surfaces the ctx-canceled GetLedger error") + require.ErrorIs(t, err, context.Canceled, "the loop surfaces the ctx-canceled stream error") case <-time.After(10 * time.Second): t.Fatal("ingestion loop did not stop on ctx cancellation") } } -// TestRunIngestionLoop_GetLedgerErrorReturnsError: a GetLedger error (not a -// shutdown) propagates as a restartable failure. -func TestRunIngestionLoop_GetLedgerErrorReturnsError(t *testing.T) { +// TestRunIngestionLoop_StreamErrorReturnsError: a stream error (not a shutdown) +// propagates as a restartable failure. +func TestRunIngestionLoop_StreamErrorReturnsError(t *testing.T) { cat, _ := testCatalog(t) c := chunk.ID(0) first := c.FirstLedger() db := openLiveHotDB(t, cat, c) boom := errors.New("backend exploded") - getter := getterForSeqs(t, first, first) - getter.yieldErrAt = first + 1 - getter.errAt = boom - ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) + stream := streamForSeqs(t, first, first) + stream.yieldErrAt = first + 1 + stream.errAt = boom + cfg, _ := loopConfig(t, stream, db, cat) - err := runIngestionLoop(context.Background(), getter, db, cat, ch, silentLogger(), nil, nil) + err := runIngestionLoop(context.Background(), cfg) require.Error(t, err) require.ErrorIs(t, err, boom) } @@ -318,23 +366,23 @@ func TestRunIngestionLoop_GetLedgerErrorReturnsError(t *testing.T) { // --------------------------------------------------------------------------- // TestRunIngestionLoop_RestartResumesFromWatermark: after a first run commits a -// prefix and exits, a second run over a FRESH open of the SAME hot dir resumes -// at watermark+1 (asserted via the FIRST seq the getter is asked for) and a -// re-delivered already-committed ledger is the idempotent retry the hot stores -// tolerate — the final watermark is exactly the last delivered seq. +// prefix and exits, a second run over a FRESH open of the SAME hot dir resumes at +// watermark+1 (asserted via the FIRST seq the stream is asked for) — the stream +// range starts at the derived resume, and the final watermark is exactly the last +// delivered seq. func TestRunIngestionLoop_RestartResumesFromWatermark(t *testing.T) { cat, _ := testCatalog(t) c := chunk.ID(0) first := c.FirstLedger() - // First run: commit [first, first+2], then the getter errs. + // First run: commit [first, first+2], then the stream errs. db1 := openLiveHotDB(t, cat, c) - getter1 := getterForSeqs(t, first, first+2) - getter1.endErr = errors.New("end") - ch := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) - err := runIngestionLoop(context.Background(), getter1, db1, cat, ch, silentLogger(), nil, nil) + stream1 := streamForSeqs(t, first, first+2) + stream1.endErr = errors.New("end") + cfg1, _ := loopConfig(t, stream1, db1, cat) + err := runIngestionLoop(context.Background(), cfg1) require.Error(t, err) - assert.Equal(t, first, getter1.firstSeen.Load(), "first run resumed at the chunk's first ledger") + assert.Equal(t, first, stream1.firstSeen.Load(), "first run resumed at the chunk's first ledger") // Restart: re-open the live DB the way startup would. The resume point must // be watermark+1. @@ -344,13 +392,13 @@ func TestRunIngestionLoop_RestartResumesFromWatermark(t *testing.T) { require.NoError(t, err) assert.Equal(t, first+3, resume, "restart resumes one past the durable watermark") - // Second run re-delivers the last already-committed ledger (idempotent) plus - // two new ones. - getter2 := getterForSeqs(t, first+2, first+5) - getter2.endErr = errors.New("end") - err = runIngestionLoop(context.Background(), getter2, db2, cat, ch, silentLogger(), nil, nil) + // Second run resumes at watermark+1 and commits two more ledgers. + stream2 := streamForSeqs(t, first+3, first+5) + stream2.endErr = errors.New("end") + cfg2, _ := loopConfig(t, stream2, db2, cat) + err = runIngestionLoop(context.Background(), cfg2) require.Error(t, err) - assert.Equal(t, first+3, getter2.firstSeen.Load(), "second run resumed at watermark+1") + assert.Equal(t, first+3, stream2.firstSeen.Load(), "second run resumed at watermark+1") reopened, err := hotchunk.Open(cat.Layout().HotChunkPath(c), c, silentLogger()) require.NoError(t, err) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go b/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go index ae42b0079..1c387ec5e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go @@ -58,7 +58,7 @@ // // Inputs are borrowed: every Ingest receives a view over the source // stream's buffer, valid only until the next ledger is pulled, and -// each ingester copies what it retains (see HotIngester). The raw +// each ingester copies what it retains (see LedgerIngester). The raw // ledger iterator's contract includes yielding an error on ctx // cancellation — the drain loop relies on it for cancellation rather // than polling ctx itself. Metrics flow through MetricSink (Prometheus in prod, diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go index cc7820d9b..099a10bee 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go @@ -44,47 +44,73 @@ func closeColdAll(ings []ColdIngester, err error) error { return err } -// drain feeds each of the chunk's raw ledgers (as a view) to the service, then -// verifies the full [first,last] range was consumed — for cold this runs before -// Finalize, so a short stream never finalizes a truncated artifact. Cancellation -// is the iterator's job (RawLedgers errors on canceled ctx), so no ctx poll here. -func drain(ctx context.Context, ledgers iter.Seq2[[]byte, error], chunkID chunk.ID, ing HotIngester) error { +// ValidatedLedger is one sequence-validated ledger from a raw stream: its +// verified sequence and the borrowed view (valid only until the next iteration +// step, per the LedgerStream contract). +type ValidatedLedger struct { + Seq uint32 + View xdr.LedgerCloseMetaView +} + +// SeqValidatedCursor adapts a raw ledger stream into contiguous, sequence-checked +// ledgers starting at `from`: for each yielded frame it reads the view's own +// LedgerSequence() and rejects a gap, duplicate, or out-of-order ledger before +// handing it on. Both the cold drain and the hot ingestion loop consume it, so the +// sole writer of recent history never trusts an injected source blindly (the SDK +// backend also validates its own output — this is defense-in-depth, a zero-copy +// header read). A source error, a decode error, or a non-contiguous sequence is +// yielded as the error element and ends iteration; the view is borrowed. +func SeqValidatedCursor( + ledgers iter.Seq2[[]byte, error], from uint32, +) iter.Seq2[ValidatedLedger, error] { + return func(yield func(ValidatedLedger, error) bool) { + seq := from + for raw, serr := range ledgers { + if serr != nil { + yield(ValidatedLedger{Seq: seq}, fmt.Errorf("RawLedgers(%d): %w", seq, serr)) + return + } + lcm := xdr.LedgerCloseMetaView(raw) + actual, aerr := lcm.LedgerSequence() + if aerr != nil { + yield(ValidatedLedger{Seq: seq}, fmt.Errorf("ledger sequence at expected %d: %w", seq, aerr)) + return + } + if actual != seq { + yield(ValidatedLedger{Seq: seq}, fmt.Errorf("yielded ledger %d, expected %d", actual, seq)) + return + } + if !yield(ValidatedLedger{Seq: seq, View: lcm}, nil) { + return + } + seq++ + } + } +} + +// drain feeds each of the chunk's raw ledgers (as a validated view) to the +// service, then verifies the full [first,last] range was consumed — for cold this +// runs before Finalize, so a short stream never finalizes a truncated artifact. +// Cancellation is the iterator's job (RawLedgers errors on canceled ctx), so no +// ctx poll here. The per-ledger sequence guard lives in the shared cursor. +func drain(ctx context.Context, ledgers iter.Seq2[[]byte, error], chunkID chunk.ID, ing LedgerIngester) error { first, last := chunkID.FirstLedger(), chunkID.LastLedger() seq := first - for raw, serr := range ledgers { - if serr != nil { - return fmt.Errorf("RawLedgers(%d): %w", seq, serr) + for vl, verr := range SeqValidatedCursor(ledgers, first) { + if verr != nil { + return fmt.Errorf("ingest: stream for chunk %d: %w", uint32(chunkID), verr) } - // Reject a stream that runs PAST the chunk before ingesting anything - // out-of-chunk. Without this, an in-order overrun would only trip the - // post-loop count check after the extra ledgers were durably ingested - // (the ledger and txhash hot stores accept any sequence). All in-repo - // sources bound themselves; this guards custom iterators. - if seq > last { + // Reject a stream that runs PAST the chunk before ingesting out-of-chunk. + // The cursor already validated vl.Seq is contiguous; this bounds it above. + // All in-repo sources bound themselves; this guards custom iterators. + if vl.Seq > last { return fmt.Errorf("ingest: stream for chunk %d yielded a ledger past %d (chunk overrun)", uint32(chunkID), last) } - lcm := xdr.LedgerCloseMetaView(raw) - // Validate the actual ledger sequence before ingesting. The final - // count check below only catches a short/long stream; a source that - // yields a duplicate or out-of-order ledger with the right total - // count would otherwise pass silently (e.g. on the txhash and - // ledger-hot paths, which key on the LCM's own seq). - actual, aerr := lcm.LedgerSequence() - if aerr != nil { - return fmt.Errorf("ingest: stream for chunk %d: ledger sequence at expected %d: %w", - uint32(chunkID), seq, aerr) - } - if actual != seq { - return fmt.Errorf("ingest: stream for chunk %d yielded ledger %d, expected %d", - uint32(chunkID), actual, seq) - } - // seq is now VALIDATED as lcm's sequence — pass it through so the - // ingesters consume it instead of each re-deriving it from the view. - if err := ing.Ingest(ctx, seq, lcm); err != nil { + if err := ing.Ingest(ctx, vl.Seq, vl.View); err != nil { return err } - seq++ + seq = vl.Seq + 1 } if seq != last+1 { return fmt.Errorf("ingest: stream for chunk %d ended at %d, expected through %d", uint32(chunkID), seq-1, last) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go index 55a0e200f..6b4d1638b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go @@ -6,27 +6,26 @@ import ( "github.com/stellar/go-stellar-sdk/xdr" ) -// HotIngester ingests one ledger by sequence into a long-lived, caller-owned -// store. The hot tier's HotService implements it — one atomic synced WriteBatch -// across all column families per ledger (decision (a); NO per-type fan-out, no -// errgroup) — and the cold drain loop (drain) consumes the same shape, so -// ColdService satisfies it too. (The "Hot" name is historical: this is just the -// per-ledger ingest contract now.) +// LedgerIngester is drain's per-ledger consumer: it ingests one ledger by +// sequence into a caller-owned store. ColdService implements it (drain drives the +// cold materializer through it); the hot tier's HotService satisfies the same +// shape and the ingestion loop calls it, though the loop drives HotService +// directly rather than through this interface. // // Ownership: the store is INJECTED into the implementation's constructor and // owned by the caller (the daemon). The implementation does NOT open the store // and does NOT close it — Close is intentionally absent from this interface. // -// Input: seq is the DRIVER-VALIDATED ledger sequence of lcm — the drain loop -// has already read it off the view and checked it against the chunk's expected -// position (duplicate / out-of-order / overrun), so implementations consume it -// directly instead of re-deriving and re-error-handling it. lcm is a zero-copy -// xdr.LedgerCloseMetaView (a []byte alias over the source stream's BORROWED -// buffer), valid only for the current iteration step; an implementation must -// copy any bytes it retains. Ledgers are ingested sequentially — the source +// Input: seq is the CURSOR-VALIDATED ledger sequence of lcm — the shared +// seq-validated cursor (SeqValidatedCursor) has already read it off the view and +// checked it is contiguous (no gap / duplicate / out-of-order), so implementations +// consume it directly instead of re-deriving and re-error-handling it. lcm is a +// zero-copy xdr.LedgerCloseMetaView (a []byte alias over the source stream's +// BORROWED buffer), valid only for the current iteration step; an implementation +// must copy any bytes it retains. Ledgers are ingested sequentially — the source // pulls the next only after Ingest returns — so synchronous consumption inside // Ingest is safe. -type HotIngester interface { +type LedgerIngester interface { Ingest(ctx context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error } @@ -45,7 +44,7 @@ type HotIngester interface { // artifact; implementations are encouraged to latch the failure and refuse // (eventsCold does). // -// Input: same driver-validated-seq and borrowed-view contract as HotIngester. +// Input: same cursor-validated-seq and borrowed-view contract as LedgerIngester. // ColdService drives the per-ledger Ingest calls sequentially, so each view is // fully consumed before the next. type ColdIngester interface { diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index 21905c0c3..05dd9556d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "sync/atomic" "time" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" @@ -218,33 +219,58 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu } } -// LifecycleQueueDepth is the notification buffer depth — far above the at-most-one -// boundary a healthy daemon holds in flight. A FULL buffer means freeze has fallen -// this many boundaries behind ingestion, a fatal condition notify() reports. -const LifecycleQueueDepth = 8 - -// Loop is the event-driven lifecycle goroutine. Each notification carries -// the just-completed chunk id; the loop drains the buffer to the most-recent id -// (one tick over [floor, lastChunk] subsumes the rest) and runs one tick. It -// selects on both ctx.Done() and the channel, so it never blocks or fatals on -// shutdown. -func Loop(ctx context.Context, cfg Config, cat *catalog.Catalog, ch <-chan chunk.ID) { +// BoundarySignal couples ingestion (the producer) to the lifecycle Loop (the +// consumer): ingestion stores the latest completed chunk id and pings a +// 1-buffered wake; the Loop blocks on the wake, then reads the latest id. A +// latest-CELL (not a queue) means a slow lifecycle can never fall behind — one +// tick over [floor, latest] subsumes every skipped boundary — so there is no +// bounded buffer to overflow and thus no "fell behind" fatal path. Safe for one +// producer and one consumer. +type BoundarySignal struct { + latest atomic.Uint32 + set atomic.Bool + wake chan struct{} +} + +// NewBoundarySignal returns a ready signal with an empty latest cell. +func NewBoundarySignal() *BoundarySignal { + return &BoundarySignal{wake: make(chan struct{}, 1)} +} + +// Publish records c as the latest completed chunk and wakes the Loop. The wake is +// non-blocking: a pending wake already covers this boundary (the Loop will read +// the newest latest when it runs), so a full buffer is dropped, never blocked on. +func (s *BoundarySignal) Publish(c chunk.ID) { + s.latest.Store(uint32(c)) + s.set.Store(true) + select { + case s.wake <- struct{}{}: + default: + } +} + +// take returns the latest published chunk id; ok=false when nothing has been +// published (chunk 0 is a valid id, so a separate flag distinguishes it). +func (s *BoundarySignal) take() (chunk.ID, bool) { + if !s.set.Load() { + return 0, false + } + return chunk.ID(s.latest.Load()), true +} + +// Loop is the event-driven lifecycle goroutine. It blocks on the boundary signal's +// wake, reads the latest completed chunk id, and runs one tick over +// [floor, lastChunk] (which subsumes every boundary skipped while it was busy). It +// selects on ctx.Done() too, so it never blocks past shutdown. +func Loop(ctx context.Context, cfg Config, cat *catalog.Catalog, sig *BoundarySignal) { for { select { case <-ctx.Done(): return - case lastChunk := <-ch: - // Drain to the most-recent queued chunk: one tick over [floor, lastChunk] - // subsumes every earlier boundary still sitting in the buffer. - drain: - for { - select { - case lastChunk = <-ch: - case <-ctx.Done(): - return - default: - break drain - } + case <-sig.wake: + lastChunk, ok := sig.take() + if !ok { + continue } runLifecycle(ctx, cfg, cat, lastChunk) } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go index e71bb6e67..326cebad3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go @@ -12,11 +12,11 @@ import ( ) // --------------------------------------------------------------------------- -// Loop: selects on BOTH ctx.Done and the notification channel; drains -// to the most-recent queued chunk id. +// Loop: selects on BOTH ctx.Done and the boundary signal's wake; reads the +// most-recent published chunk id from the latest-cell. // --------------------------------------------------------------------------- -// TestLifecycleLoop_RunsTickPerNotifyThenStopsOnCtx: a notification (a completed +// TestLifecycleLoop_RunsTickPerNotifyThenStopsOnCtx: a boundary signal (a completed // chunk id) runs a tick; a ctx cancellation returns the loop. The loop never // blocks forever and never fatals on shutdown. func TestLifecycleLoop_RunsTickPerNotifyThenStopsOnCtx(t *testing.T) { @@ -33,19 +33,19 @@ func TestLifecycleLoop_RunsTickPerNotifyThenStopsOnCtx(t *testing.T) { live := openLiveHotDB(t, cat, 1) t.Cleanup(func() { _ = live.Close() }) - ch := make(chan chunk.ID, LifecycleQueueDepth) + sig := NewBoundarySignal() ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { - Loop(ctx, cfg, cat, ch) + Loop(ctx, cfg, cat, sig) close(done) }() - ch <- chunk.ID(0) // ingestion hands over the just-completed chunk 0 + sig.Publish(chunk.ID(0)) // ingestion hands over the just-completed chunk 0 require.Eventually(t, func() bool { has, err := hotKeyExists(cat, 0) return err == nil && !has - }, 10*time.Second, 20*time.Millisecond, "the notification ran a tick that discarded chunk 0") + }, 10*time.Second, 20*time.Millisecond, "the signal ran a tick that discarded chunk 0") require.False(t, rec.fired()) cancel() @@ -56,10 +56,10 @@ func TestLifecycleLoop_RunsTickPerNotifyThenStopsOnCtx(t *testing.T) { } } -// TestLifecycleLoop_DrainsToMostRecent: several chunk ids queued behind one -// notification are coalesced into ONE tick over the most-recent. With chunks 0 -// and 1 both frozen+covered and a live chunk 2, sending 0 then 1 runs a single -// tick up to chunk 1 that discards both. +// TestLifecycleLoop_DrainsToMostRecent: the latest-cell coalesces rapid +// boundaries — publishing 0 then 1 lands a tick over the most-recent (chunk 1) +// that subsumes chunk 0. With chunks 0 and 1 both frozen+covered and a live chunk +// 2, both are discarded (whether that takes one coalesced tick or two). func TestLifecycleLoop_DrainsToMostRecent(t *testing.T) { cat, _ := smallTxHashIndexCatalog(t, 1) cfg, rec := lifecycleTestConfig(t, cat, 0) @@ -72,17 +72,17 @@ func TestLifecycleLoop_DrainsToMostRecent(t *testing.T) { live := openLiveHotDB(t, cat, 2) t.Cleanup(func() { _ = live.Close() }) - ch := make(chan chunk.ID, LifecycleQueueDepth) + sig := NewBoundarySignal() ctx, cancel := context.WithCancel(context.Background()) defer cancel() done := make(chan struct{}) go func() { - Loop(ctx, cfg, cat, ch) + Loop(ctx, cfg, cat, sig) close(done) }() - ch <- chunk.ID(0) - ch <- chunk.ID(1) // drained-to: one tick over [floor, 1] discards both + sig.Publish(chunk.ID(0)) + sig.Publish(chunk.ID(1)) // latest-cell coalesces: a tick over [floor, 1] discards both require.Eventually(t, func() bool { h0, e0 := hotKeyExists(cat, 0) h1, e1 := hotKeyExists(cat, 1) @@ -108,10 +108,10 @@ func TestLifecycleLoop_ReturnsImmediatelyOnAlreadyCancelledCtx(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - ch := make(chan chunk.ID) // unbuffered, never sent to + sig := NewBoundarySignal() // never published to done := make(chan struct{}) go func() { - Loop(ctx, cfg, cat, ch) + Loop(ctx, cfg, cat, sig) close(done) }() select { diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index 7b64f41e1..fe845fe56 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -9,6 +9,8 @@ import ( "github.com/cenkalti/backoff/v4" + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/lifecycle" @@ -82,31 +84,29 @@ func run(ctx context.Context, cfg StartConfig) error { return fmt.Errorf("startup open resume hot tier chunk %s: %w", resumeChunk, err) } - // Start captive core from the resume ledger. On failure the resume hot DB is - // already open; close it so a restart re-opens cleanly (the rocksdb LOCK must - // be released). - core, closeCore, err := cfg.Core.OpenCore(ctx, resumeLedger) + // The live ingestion stream. It owns the captive-core process (started on the + // loop's first pull, torn down when the loop exits), so there is no eager + // prepare and no closer to defer — the loop's ctx-scoped iteration is the + // teardown. OpenCore only constructs, so a start failure surfaces as the loop's + // first stream error for the daemon to classify (and restart). + stream, err := cfg.Core.OpenCore(ctx) if err != nil { _ = hotDB.Close() - return fmt.Errorf("startup start captive core at ledger %d: %w", resumeLedger, err) + return fmt.Errorf("startup open ingestion stream: %w", err) } - defer func() { - if closeCore != nil { - _ = closeCore() - } - }() - // The lifecycle goroutine runs one tick per notification carrying the just- - // completed chunk id. Buffered to LifecycleQueueDepth; ingestion sends at each - // boundary. It shares NO in-memory state with ingestion — all derived from durable keys. - lifecycleCh := make(chan chunk.ID, lifecycle.LifecycleQueueDepth) + // The lifecycle goroutine runs one tick per boundary signal. Ingestion Publishes + // the just-completed chunk id into a latest-cell (a slow lifecycle can't fall + // behind — no bounded buffer, no fatal). It shares NO in-memory state with + // ingestion — all derived from durable keys. + boundary := lifecycle.NewBoundarySignal() // Seed the first tick with the last complete chunk at the resume point so it // fires at once — clearing crash/downtime leftovers concurrently with serving. // Skipped on a young network where no chunk is complete (the first real boundary // triggers the first tick). if seed := geometry.LastCompleteChunkAt(lastCommitted); seed >= 0 { - lifecycleCh <- chunk.ID(seed) //nolint:gosec // seed >= 0 + boundary.Publish(chunk.ID(seed)) //nolint:gosec // seed >= 0 } // The lifecycle goroutine is tied to a PER-ITERATION child ctx (not the daemon @@ -127,12 +127,12 @@ func run(ctx context.Context, cfg StartConfig) error { }.WithLifecycleDefaults() var lifecycleWG sync.WaitGroup lifecycleWG.Go(func() { - lifecycle.Loop(lifecycleCtx, lifecycleCfg, cat, lifecycleCh) + lifecycle.Loop(lifecycleCtx, lifecycleCfg, cat, boundary) }) // The two return paths registered after this defer (the ingestion-loop return - // and the ServeReads error path) have no live sender on lifecycleCh — ingestion - // is a same-goroutine call whose inline notify has stopped, and the serve path - // never starts it — so no send can race the cancel. + // and the ServeReads error path) have no live producer on the boundary signal — + // ingestion is a same-goroutine call whose inline Publish has stopped, and the + // serve path never starts it — so no Publish can race the cancel. defer func() { cancelLifecycle() lifecycleWG.Wait() @@ -144,10 +144,20 @@ func run(ctx context.Context, cfg StartConfig) error { return fmt.Errorf("startup serve reads: %w", err) } - // The ingestion loop owns hotDB for the rest of its life (closes it on any exit, - // reopens at each boundary). Returns the GetLedger/boundary error; the daemon top - // level classifies a ctx-canceled return as a clean shutdown. - return runIngestionLoop(ctx, core, hotDB, cat, lifecycleCh, logger, metrics, cfg.Exec.Process.Sink) + // The ingestion loop is the hot-tier owner: it owns hotDB for the rest of its + // life (closes it on any exit, reopens at each boundary) and consumes the stream + // from the resume ledger. Returns the stream/boundary error; the daemon top level + // classifies a ctx-canceled return as a clean shutdown. + return runIngestionLoop(ctx, ingestionLoopConfig{ + Stream: stream, + Resume: resumeLedger, + HotDB: hotDB, + Catalog: cat, + Boundary: boundary, + Logger: logger, + Metrics: metrics, + Sink: cfg.Exec.Process.Sink, + }) } // backfillToTip runs the backfill loop, returning lastCommitted as backfill makes @@ -257,11 +267,14 @@ type NetworkTipBackend interface { NetworkTip(ctx context.Context) (uint32, error) } -// CoreOpener prepares captive core at resumeLedger and hands back a LedgerGetter -// the ingestion loop polls plus a closer the caller defers. Production wraps -// captive core's PrepareRange + GetLedger; tests pass a fake getter. +// CoreOpener hands back the live ingestion stream the loop consumes. The stream +// OWNS its source's lifecycle (started on the first RawLedgers pull over the +// unbounded range from the loop's resume ledger, torn down when the loop exits), +// so there is no resume arg, no PrepareRange, and no closer for the caller to +// sequence. Production returns a captive-core stream; tests pass a fake +// LedgerStream. type CoreOpener interface { - OpenCore(ctx context.Context, resumeLedger uint32) (LedgerGetter, func() error, error) + OpenCore(ctx context.Context) (ledgerbackend.LedgerStream, error) } // StartConfig is run's resolved dependency bundle. diff --git a/cmd/stellar-rpc/internal/fullhistory/startup_test.go b/cmd/stellar-rpc/internal/fullhistory/startup_test.go index 8cb615b1a..79b4ddfc2 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup_test.go @@ -11,6 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" @@ -111,28 +113,35 @@ func startTestConfig( return cfg } -// fakeCore is a CoreOpener handing back a programmed LedgerGetter and recording -// the resume ledger it was started from. +// fakeCore is a CoreOpener handing back a programmed LedgerStream. The loop opens +// the stream at its resume ledger via RawLedgers(UnboundedRange(resume)), so the +// resume the loop started from is the stream's recorded firstSeen (resumeSeen()). type fakeCore struct { - getter LedgerGetter + stream *fakeCoreStream // programmed; nil → default block-on-ctx stream openErr error - resumeSeen atomic.Uint32 openedCount atomic.Int32 } -func (c *fakeCore) OpenCore(_ context.Context, resumeLedger uint32) (LedgerGetter, func() error, error) { +func (c *fakeCore) OpenCore(context.Context) (ledgerbackend.LedgerStream, error) { c.openedCount.Add(1) - c.resumeSeen.Store(resumeLedger) if c.openErr != nil { - return nil, nil, c.openErr + return nil, c.openErr + } + if c.stream == nil { + // Default: a live stream that blocks until ctx is canceled (the daemon's + // steady state). Tests that need a finite stream set c.stream. + c.stream = &fakeCoreStream{frames: map[uint32][]byte{}, blockOnCtx: true} } - getter := c.getter - if getter == nil { - // Default: a live getter that blocks until ctx is canceled (the daemon's - // steady state). Tests that need a finite poll set c.getter. - getter = &fakeLedgerGetter{frames: map[uint32][]byte{}, blockOnCtx: true} + return c.stream, nil +} + +// resumeSeen returns the resume ledger the loop opened the stream at (the range's +// From()), 0 before the loop has pulled. +func (c *fakeCore) resumeSeen() uint32 { + if c.stream == nil { + return 0 } - return getter, func() error { return nil }, nil + return c.stream.firstSeen.Load() } // pinGenesis pins earliest_ledger to genesis (as validateConfig does for a @@ -357,7 +366,7 @@ func TestRun_FirstStartServeIngestCleanShutdown(t *testing.T) { pinGenesis(t, cat) served := atomic.Int32{} - core := &fakeCore{getter: &fakeLedgerGetter{frames: map[uint32][]byte{}, blockOnCtx: true}} + core := &fakeCore{stream: &fakeCoreStream{frames: map[uint32][]byte{}, blockOnCtx: true}} tip := &fakeTipBackend{tips: []uint32{chunk.FirstLedgerSeq + 10}} // young: no backfill cfg := startTestConfig(t, cat, tip, core, nil) cfg.ServeReads = func(context.Context) error { served.Add(1); return nil } @@ -380,7 +389,7 @@ func TestRun_FirstStartServeIngestCleanShutdown(t *testing.T) { require.Equal(t, int32(1), served.Load(), "reads were served exactly once") require.Equal(t, int32(1), core.openedCount.Load(), "captive core started once") - require.Equal(t, uint32(chunk.FirstLedgerSeq), core.resumeSeen.Load(), + require.Equal(t, uint32(chunk.FirstLedgerSeq), core.resumeSeen(), "resume ledger is genesis on a fresh start (watermark+1)") // The resume chunk's hot key is "ready" (opened, boundary never crossed). @@ -396,7 +405,7 @@ func TestRun_FirstStartServeIngestCleanShutdown(t *testing.T) { func TestRun_ServeReadsErrorSurfaces(t *testing.T) { cat, _ := testCatalog(t) pinGenesis(t, cat) - core := &fakeCore{getter: &fakeLedgerGetter{frames: map[uint32][]byte{}, blockOnCtx: true}} + core := &fakeCore{stream: &fakeCoreStream{frames: map[uint32][]byte{}, blockOnCtx: true}} tip := &fakeTipBackend{tips: []uint32{chunk.FirstLedgerSeq + 10}} cfg := startTestConfig(t, cat, tip, core, nil) cfg.ServeReads = func(context.Context) error { return errors.New("rpc bind failed") } From e13fb2151ef65c13c32277389cb89cbbe03ef741 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 16:00:04 -0400 Subject: [PATCH 33/55] fullhistory/stores: delete unreachable hot-write-path guards + events staging (#30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three guards on the one hot write chain, none reachable, plus a staging layer defending edges the composition precludes. - Closed-store check: dropped the four redundant IsClosed re-checks inside the batch callback (hotchunk.IngestLedger's pre-check + the ledger/txhash/events facade *ToBatch methods). Store.Batch's lifecycle RLock + checkOpen is the only race-free guard and the sole writer closes its handle only between ledgers, so none of the four could fire first. The surfaced closed sentinel is now rocksdb.ErrStoreClosed (the authoritative one). - Idempotent-duplicate branch: deleted. Under decision (a) resume is always MaxCommittedSeq+1 and the events frontier commits in the same atomic batch as the ledgers row, so seq == expected every iteration; a non-expected ledger is a mis-sequencing source (the ingestion loop's cursor guard, #17) — now ErrLedgerOutOfOrder, not a silent (nil,nil) no-op. hotchunk drops the applyEvents-nil branch. - Events prepare/queue staging: collapsed. The preparedLedger struct + separate prepareLedger/queue passes existed to hold many ledgers' rows across one batch and to reject a bad ledger "without a half-built batch" — but the only caller opens exactly one batch per ledger and Store.Batch discards the whole WriteBatch on any callback error. So validate + derive terms + marshal + Put now run inline in IngestLedgerToBatch with ONE reused scratch buffer (was one heap alloc per event), keeping only the post-commit apply hook the in-memory mirror needs. Tests: DuplicateLedgerIsNoOp -> DuplicateLedgerErrors (asserts ErrLedgerOutOfOrder + untouched state); ClosedDBFails asserts rocksdb.ErrStoreClosed. Store packages + downstream (root/lifecycle/backfill/ingest) pass; non-short E2E green (~90s). --- .../pkg/stores/eventstore/hot_store.go | 146 +++++------------- .../pkg/stores/eventstore/hot_store_test.go | 36 ++--- .../pkg/stores/hotchunk/hotchunk.go | 18 +-- .../pkg/stores/hotchunk/hotchunk_test.go | 7 +- .../pkg/stores/ledger/hot_store.go | 6 +- .../pkg/stores/txhash/hot_store.go | 6 +- 6 files changed, 76 insertions(+), 143 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index 42278a7be..054a0e542 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -394,30 +394,25 @@ func (h *HotStore) All(ctx context.Context) iter.Seq2[events.Payload, error] { } } -// IngestLedgerToBatch validates+marshals one ledger's events and queues their CF -// Puts into the SHARED batch b, returning the post-commit apply hook the caller -// runs AFTER b commits (decision (a)). Returns (nil, nil) for an idempotent -// duplicate. All validation + term derivation happen up front, so a rejected -// ledger leaves b untouched. +// IngestLedgerToBatch validates one ledger's events, marshals them, and queues +// their CF Puts into the SHARED batch b, returning the post-commit apply hook the +// caller runs AFTER b commits (decision (a)). Validation + term derivation happen +// before any Put; on any error Store.Batch discards the whole WriteBatch, so a +// rejected ledger never leaves committed rows behind. // // payloads is produced by events.LCMViewToPayloads, which emits each ledger's -// events in ascending getEvents cursor order — write order here IS the -// cursor contract (event IDs are assigned by arrival position). Terms are -// derived internally via events.TermsForBytes on each payload's -// ContractEventBytes. +// events in ascending getEvents cursor order — write order here IS the cursor +// contract (event IDs are assigned by arrival position). Terms are derived via +// events.TermsForBytes on each payload's ContractEventBytes. // -// Sequence validation is performed up front, before any queue or mirror -// mutation: +// Sequence validation, before any Put or mirror mutation: // -// - ledgerSeq must lie within [chunkID.FirstLedger(), -// chunkID.LastLedger()] — out-of-range returns ErrLedgerOutOfRange. -// - ledgerSeq == the next expected ledger (StartLedger + LedgerCount) -// is appended normally. -// - ledgerSeq < expected (an already-ingested ledger) is an idempotent -// no-op returning (nil, nil), so a restarted ingester can blindly -// re-deliver the in-flight ledger; the re-delivered events are not -// re-verified. -// - ledgerSeq > expected (a gap) returns ErrLedgerOutOfOrder. +// - ledgerSeq must lie within [chunkID.FirstLedger(), chunkID.LastLedger()] — +// out-of-range returns ErrLedgerOutOfRange. +// - ledgerSeq must equal the next expected ledger (StartLedger + LedgerCount). +// Under decision (a) resume is always MaxCommittedSeq+1, so a non-expected +// ledger is a mis-sequencing source (the ingestion loop's seq guard should +// have caught it) — an error (ErrLedgerOutOfOrder), never silent tolerance. // // Post-batch atomicity: once the batch commits, the apply hook's in-memory // mirror + offsets updates are infallible by construction. Any failure there @@ -427,78 +422,21 @@ func (h *HotStore) All(ctx context.Context) iter.Seq2[events.Payload, error] { func (h *HotStore) IngestLedgerToBatch( b *rocksdb.BatchWriter, ledgerSeq uint32, payloads []events.Payload, ) (func(), error) { - if h.chunkStore.IsClosed() { - return nil, ErrClosed - } - prep, err := h.prepareLedger(ledgerSeq, payloads) - if err != nil { - return nil, err - } - if prep == nil { - //nolint:nilnil // (nil, nil) is the idempotent-duplicate signal; the caller runs the hook only when non-nil - return nil, nil - } - prep.queue(b) - return prep.apply, nil -} - -// preparedLedger is one validated, marshaled ledger ready to queue into -// a write batch (queue) and, once that batch is durable, apply to the -// in-memory mirror + offsets (apply). -type preparedLedger struct { - ledgerSeq uint32 - startID uint32 - blobs [][]byte // marshaled payload XDR, positional with payloads - termKeys [][]events.TermKey // per-payload term keys - apply func() // post-commit mirror + offsets update (infallible) -} - -// queue writes the prepared ledger's rows into b: one DataCF row per -// event, one IndexCF row per (term, event), and one OffsetsCF row for -// the ledger's per-ledger event count. -func (p *preparedLedger) queue(b *rocksdb.BatchWriter) { - for i := range p.blobs { - eventID := p.startID + uint32(i) - b.Put(DataCF, encodeDataKey(eventID), p.blobs[i]) - for _, key := range p.termKeys[i] { - b.Put(IndexCF, encodeIndexKey(key, eventID), nil) - } - } - //nolint:gosec // bounds-checked in prepareLedger's overflow guard - eventCount := uint32(len(p.blobs)) - b.Put(OffsetsCF, encodeOffsetKey(p.ledgerSeq), encodeLedgerEventCount(eventCount)) -} - -// prepareLedger runs the pre-commit pipeline for one ledger (validate → derive -// terms → marshal into fresh per-event buffers), returning a *preparedLedger -// ready to queue + apply, or (nil, nil) for an idempotent duplicate. It does NO -// disk write and NO mirror mutation, so it is safe to call before touching a -// shared batch. -func (h *HotStore) prepareLedger(ledgerSeq uint32, payloads []events.Payload) (*preparedLedger, error) { - // Validate BEFORE marshaling: failing after a shared batch holds this - // ledger's rows would orphan them. + // Validate BEFORE any Put. On error Store.Batch discards the whole WriteBatch, + // so a mid-loop failure never orphans rows — no separate staging buffer needed. if ledgerSeq < h.chunkID.FirstLedger() || ledgerSeq > h.chunkID.LastLedger() { return nil, fmt.Errorf("%w: ledger %d not in chunk %s [%d, %d]", ErrLedgerOutOfRange, ledgerSeq, h.chunkID, h.chunkID.FirstLedger(), h.chunkID.LastLedger()) } expected := h.offsets.StartLedger() + uint32(h.offsets.LedgerCount()) //nolint:gosec - if ledgerSeq < expected { - // Already ingested: idempotent no-op (a restarted ingester may - // re-deliver). Re-delivered events are not re-verified. - //nolint:nilnil // (nil, nil) is the idempotent-duplicate signal; callers branch on a nil *preparedLedger - return nil, nil - } - if ledgerSeq > expected { + if ledgerSeq != expected { return nil, fmt.Errorf("%w: expected ledger %d, got %d", ErrLedgerOutOfOrder, expected, ledgerSeq) } - // Pre-derive term keys per payload so the post-commit mirror - // update doesn't re-hash. Surfacing TermsForBytes errors here - // (pre-batch) cleanly rejects the ledger commit without touching disk — - // a decode failure on stellar-core-validated XDR is a corruption - // signal worth aborting on. + // Derive term keys per payload up front (a TermsForBytes error rejects the + // ledger without any Put) and retain them for the post-commit mirror update. termKeys := make([][]events.TermKey, len(payloads)) for i := range payloads { keys, err := events.TermsForBytes(payloads[i].ContractEventBytes) @@ -514,44 +452,44 @@ func (h *HotStore) prepareLedger(ledgerSeq uint32, payloads []events.Payload) (* h.chunkID, ledgerSeq) } - // Marshal each payload into its OWN fresh buffer (not reused scratch): a - // shared batch may hold many ledgers' rows before commit, so each blob must - // outlive prepare until the Write copies it. BatchWriter.Put copies - // synchronously, so the buffers are free after queue returns. - blobs := make([][]byte, len(payloads)) + // Marshal + queue each event directly into b. BatchWriter.Put copies + // synchronously, so ONE reused scratch buffer serves every event — the caller + // opens exactly one batch per ledger, so no row must outlive this call. + var scratch []byte for i := range payloads { - blob, err := payloads[i].MarshalInto(nil) + blob, err := payloads[i].MarshalInto(scratch[:0]) if err != nil { return nil, fmt.Errorf("marshal payload %d for ledger %d: %w", i, ledgerSeq, err) } - blobs[i] = blob + scratch = blob + eventID := startID + uint32(i) //nolint:gosec // i < len(payloads), overflow-guarded above + b.Put(DataCF, encodeDataKey(eventID), blob) + for _, key := range termKeys[i] { + b.Put(IndexCF, encodeIndexKey(key, eventID), nil) + } } + //nolint:gosec // len bounded by the overflow guard above + b.Put(OffsetsCF, encodeOffsetKey(ledgerSeq), encodeLedgerEventCount(uint32(len(payloads)))) - prep := &preparedLedger{ - ledgerSeq: ledgerSeq, - startID: startID, - blobs: blobs, - termKeys: termKeys, - } - prep.apply = func() { h.applyLedger(prep) } - return prep, nil + return func() { h.applyLedger(startID, termKeys) }, nil } // applyLedger updates the mirror + offsets for a ledger whose rows are durable. -// Infallible by construction (prepare validated seq under the single-writer -// contract); the only non-completion is a crash, after which warmup rebuilds. +// Infallible by construction (IngestLedgerToBatch validated seq under the +// single-writer contract); the only non-completion is a crash, after which warmup +// rebuilds. // // Ordering invariant: mirror BEFORE offsets. A concurrent Query that snapshots // offsets then reads the mirror must see either the prior state or a consistent // later one. Reversing it would let a reader see an offsets count including IDs // the mirror hasn't published — FetchEvents would then miss them, silently. -func (h *HotStore) applyLedger(p *preparedLedger) { +func (h *HotStore) applyLedger(startID uint32, termKeys [][]events.TermKey) { // Batch by key so each AddTo clones at most once per (key, ledger), not per // (key, event) — turns N COW clones into 1 for popular terms. Cap 64 ≈ a few // × unique-terms per ledger; the map grows past that. perKeyIDs := make(map[events.TermKey][]uint32, 64) - for i, keys := range p.termKeys { - eventID := p.startID + uint32(i) + for i, keys := range termKeys { + eventID := startID + uint32(i) //nolint:gosec // i < len(termKeys), overflow-guarded in IngestLedgerToBatch for _, key := range keys { perKeyIDs[key] = append(perKeyIDs[key], eventID) } @@ -559,8 +497,8 @@ func (h *HotStore) applyLedger(p *preparedLedger) { for key, ids := range perKeyIDs { h.mirror.AddTo(key, ids...) } - //nolint:gosec // len bounded by prepareLedger's overflow guard - h.offsets.Append(uint32(len(p.blobs))) + //nolint:gosec // len bounded by IngestLedgerToBatch's overflow guard + h.offsets.Append(uint32(len(termKeys))) } // ────────────────────────────────────────────────────────────────── diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go index b7f8acd6e..49523bed3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go @@ -423,13 +423,14 @@ func TestHotStore_PostCloseReadsError(t *testing.T) { require.ErrorIs(t, err, ErrClosed) } -// TestHotStore_IngestLedgerEvents_DuplicateLedgerIsNoOp pins the -// idempotency contract: re-ingesting an already-committed ledger is a -// no-op (returns nil) that leaves state untouched — it neither advances -// eventID/offsets nor writes the re-delivered payload, and the original -// ledger's events remain intact. A restarted ingester can blindly -// re-deliver the in-flight ledger. -func TestHotStore_IngestLedgerEvents_DuplicateLedgerIsNoOp(t *testing.T) { +// TestHotStore_IngestLedgerEvents_DuplicateLedgerErrors pins the sequencing +// contract after the staging collapse (#30): re-ingesting an already-committed +// ledger is NOT a silent no-op — it is a mis-sequencing error (ErrLedgerOutOfOrder) +// that leaves state untouched (Store.Batch discards the WriteBatch on the error). +// Under decision (a) the ingestion loop always resumes at MaxCommittedSeq+1 and +// the shared cursor validates contiguity, so a duplicate can only mean a broken +// source — an error, never silent tolerance. +func TestHotStore_IngestLedgerEvents_DuplicateLedgerErrors(t *testing.T) { const chunkID = chunk.ID(0) h := openHotStoreForTest(t, chunkID) first := chunkID.FirstLedger() @@ -438,30 +439,29 @@ func TestHotStore_IngestLedgerEvents_DuplicateLedgerIsNoOp(t *testing.T) { require.NoError(t, ingestLedgerEvents(h.store, first, []events.Payload{p1})) countBefore := mustEventCount(t, h.store) - nextBefore := mustEventCount(t, h.store) - // Re-ingesting the same ledger is an idempotent no-op. + // Re-ingesting the same ledger errors (expected is now first+1). p2, _ := makePayload("b") - require.NoError(t, ingestLedgerEvents(h.store, first, []events.Payload{p2})) + err := ingestLedgerEvents(h.store, first, []events.Payload{p2}) + require.ErrorIs(t, err, ErrLedgerOutOfOrder, "a re-delivered committed ledger must error, not no-op") - assert.Equal(t, countBefore, mustEventCount(t, h.store), "EventCount must not advance on duplicate ingest") - assert.Equal(t, nextBefore, mustEventCount(t, h.store), "event count must not advance on duplicate ingest") + assert.Equal(t, countBefore, mustEventCount(t, h.store), "event count must not advance on the rejected ingest") - // The original ledger's event is untouched (not overwritten by p2). + // The original ledger's event is untouched, and the rejected batch committed + // nothing (Store.Batch discards the WriteBatch on the callback error). got, err := h.store.FetchEvents(context.Background(), []uint32{0}) require.NoError(t, err) require.Len(t, got, 1) - assert.Equal(t, "a", dataSym(t, got[0]), "original event must survive the no-op") + assert.Equal(t, "a", dataSym(t, got[0]), "original event must survive the rejected re-ingest") - // The dropped payload must not reach the mirror. makePayload emits + // The rejected payload must not reach the mirror. makePayload emits // [contractID, topic0, ...]; contractID is shared across symbols - // (hardcoded 0xab), so we check topic0 (index 1), which is - // symbol-specific. + // (hardcoded 0xab), so we check topic0 (index 1), which is symbol-specific. _, secondKeys := makePayload("b") require.GreaterOrEqual(t, len(secondKeys), 2, "test fixture expected to have a topic0 term") bm, lookupErr := h.store.Lookup(context.Background(), secondKeys[1]) require.ErrorIs(t, lookupErr, ErrTermNotFound, - "the no-op'd payload's topic0 term must not appear in the mirror") + "the rejected payload's topic0 term must not appear in the mirror") assert.Nil(t, bm) } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index 9175594e3..c7653777d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -166,13 +166,10 @@ type LedgerCounts struct { // update. // // lcm is a borrowed zero-copy view; every extractor copies what it retains, so -// the view need not outlive this call. An idempotent-duplicate events ledger -// contributes nothing (nil apply hook) while the upsert-keyed CFs still write. +// the view need not outlive this call. Store.Batch's lifecycle RLock + checkOpen +// is the authoritative closed-store guard, so there is no separate pre-check here. func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts, error) { var counts LedgerCounts - if d.store.IsClosed() { - return counts, stores.ErrStoreClosed - } // Pre-extract anything that can fail BEFORE opening the batch, so a decode // error rejects the ledger without a half-built batch. @@ -194,9 +191,10 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts counts.Events = len(payloads) counts.Ledgers = 1 - // The events facade validates + marshals up front (so a rejected ledger - // never touches the batch) and returns the post-commit apply hook (nil for - // an idempotent duplicate). + // The events facade validates + marshals inside the batch callback (so a + // rejected ledger never leaves committed rows) and returns the post-commit + // apply hook. Under decision (a) resume is always MaxCommittedSeq+1, so seq is + // never a duplicate — the hook is always non-nil on success. var applyEvents func() cerr := d.store.Batch(func(b *rocksdb.BatchWriter) error { if err := d.ledger.AddLedgerToBatch(b, ledger.Entry{Seq: seq, Bytes: []byte(lcm)}); err != nil { @@ -219,9 +217,7 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts } // Batch is durable — now and only now apply the events mirror/offsets update. - if applyEvents != nil { - applyEvents() - } + applyEvents() return counts, nil } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go index 5eed2cd74..778aa871d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go @@ -331,7 +331,10 @@ func TestOpenReadOnly_ReadsCommittedAndRejectsWrites(t *testing.T) { require.Error(t, err, "read-only DB must reject writes") } -// TestIngestLedger_ClosedDBFails confirms a closed shared DB rejects ingest. +// TestIngestLedger_ClosedDBFails confirms a closed shared DB rejects ingest. The +// closed-store guard is Store.Batch's authoritative lifecycle RLock + checkOpen +// (the per-facade pre-checks were dropped in #30), so the surfaced sentinel is +// rocksdb.ErrStoreClosed. func TestIngestLedger_ClosedDBFails(t *testing.T) { chunkID := chunk.ID(0) db, err := Open(t.TempDir(), chunkID, silentLogger()) @@ -340,7 +343,7 @@ func TestIngestLedger_ClosedDBFails(t *testing.T) { raw := zeroTxLCM(t, chunkID.FirstLedger()) _, err = db.IngestLedger(chunkID.FirstLedger(), xdr.LedgerCloseMetaView(raw)) - require.ErrorIs(t, err, stores.ErrStoreClosed) + require.ErrorIs(t, err, rocksdb.ErrStoreClosed) } // ──────────────────────────── LCM fixtures ──────────────────────────── diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go index f35734b6a..3077cddb2 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go @@ -74,11 +74,9 @@ func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } // — the building block hotchunk uses to fold the ledger write into the one // shared per-ledger WriteBatch (decision (a)). Does not commit (caller owns the // batch). Compresses into a fresh buffer BatchWriter.Put copies, so e.Bytes need -// not outlive this call. +// not outlive this call. The caller runs inside Store.Batch, whose lifecycle +// RLock + checkOpen is the authoritative closed-store guard, so this adds none. func (h *HotStore) AddLedgerToBatch(b *rocksdb.BatchWriter, e Entry) error { - if h.store.IsClosed() { - return stores.ErrStoreClosed - } c, _ := h.compPool.Get().(*zstd.Compressor) defer h.compPool.Put(c) compressed, err := c.Encode(nil, e.Bytes) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go index 7871e0015..da2bb38b6 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go @@ -110,11 +110,9 @@ func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } // AddEntriesToBatch queues each (txhash → ledgerSeq) Put into b on the txhash // CF — the building block hotchunk uses to fold the tx-hash writes into the one // shared per-ledger WriteBatch (decision (a)). Does not commit (caller owns the -// batch). A closed store returns ErrStoreClosed. +// batch). The caller runs inside Store.Batch, whose lifecycle RLock + checkOpen +// is the authoritative closed-store guard, so this adds none. func (h *HotStore) AddEntriesToBatch(b *rocksdb.BatchWriter, entries []Entry) error { - if h.store.IsClosed() { - return rocksdb.ErrStoreClosed - } for _, e := range entries { b.Put(txhashCF, e.Hash[:], rocksdb.EncodeUint32(e.LedgerSeq)) } From 26f81dfb5730d0692a0ccd7d10445c2a01d2b91f Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 16:09:35 -0400 Subject: [PATCH 34/55] fullhistory/lifecycle: delete the two dead plan-range clamps (#25) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tick re-derived what its inputs already guarantee. tamirms confirmed both plan-range clamps are dead, so delete them and their feeder scans; the tick now plans [floor, lastChunk] as the design says. - rangeEnd cap to the highest durably-complete chunk (fed by LastCommittedLedger(cat, nil)): gone. The Loop only ever fires for a genuinely completed lastChunk — the boundary handoff fence completes the notification and the startup seed is sent only after backfill froze everything up to it — so lastChunk is already the right upper bound. - Floor raise to lowestMaterializedChunk: gone. The gap it guarded needs a rewound mid-chunk watermark, which no mechanism produces (recovery leaves chunk-aligned watermarks), and even a hypothetical gap is re-download churn, not wrong data. Producibility is enforced lazily per chunk in resolve. Deletes lowestMaterializedChunk (a per-tick catalog scan) and its arith test, and lastCompleteChunkAtID (now prod-dead; moved to the test-mirroring helpers). The tick-mirroring test helpers (runTickForCatalog, assertQuiescent) drop the raise too; runTickForCatalog no longer runs a tick on a young network (no complete chunk) — matching production, where the Loop is never triggered in that state. Whole tree builds + vets; lifecycle -short and the non-short E2E pass. --- .../fullhistory/lifecycle/lifecycle.go | 96 ++++--------------- .../lifecycle/lifecycle_arith_test.go | 23 ----- .../lifecycle/lifecycle_helpers_test.go | 31 +++--- 3 files changed, 37 insertions(+), 113 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index 05dd9556d..94440ec0a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -9,7 +9,6 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/observability" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) @@ -22,10 +21,9 @@ import ( // The retention floor has two roles with OPPOSITE safe directions (design // "Lifecycle"): as a RETENTION boundary erring low is harmless (an extra chunk // lingers, or a read returns not-found via the missing-file rule); as a -// PRODUCTION boundary erring low is DANGEROUS (it would plan a build below -// existing storage from an unvalidated source). So the plan range never starts -// below storage — start is RAISED to lowestMaterializedChunk; extending the -// bottom is backfill's job, producibility enforced lazily per chunk. +// PRODUCTION boundary erring low would in principle plan a build below existing +// storage — but producibility is enforced lazily per chunk in resolve, so the +// plan simply spans [floor, lastChunk] and extending the bottom is backfill's job. // Config bundles the tick/loop dependencies. It composes the scheduler's // ExecConfig (shared postconditions + worker pool with backfill) plus the @@ -66,45 +64,6 @@ func (cfg Config) abortTick(ctx context.Context, err error, what string) bool { return true } -// lastCompleteChunkAtID maps geometry.LastCompleteChunkAt to a chunk.ID; -// ok=false when no complete chunk exists (negative result). -func lastCompleteChunkAtID(ledger uint32) (chunk.ID, bool) { - c := geometry.LastCompleteChunkAt(ledger) - if c < 0 { - return 0, false - } - return chunk.ID(c), true //nolint:gosec // c >= 0 -} - -// lowestMaterializedChunk is the lowest chunk holding any chunk:* artifact key -// or hot:chunk key — the bottom of existing storage, and the production-boundary -// anchor (the plan never starts below it). ok=false on an empty catalog. -func lowestMaterializedChunk(cat *catalog.Catalog) (chunk.ID, bool, error) { - lowest := chunk.ID(0) - found := false - note := func(c chunk.ID) { - if !found || c < lowest { - lowest, found = c, true - } - } - - refs, err := cat.ChunkArtifactKeys() - if err != nil { - return 0, false, err - } - for _, ref := range refs { - note(ref.Chunk) - } - - hot, err := cat.HotChunkKeys() - if err != nil { - return 0, false, err - } - for _, c := range hot { - note(c) - } - return lowest, found, nil -} // runLifecycle runs one tick over the three stages for just-completed chunk // lastChunk. through = lastChunk.LastLedger() is the single snapshot every stage @@ -138,45 +97,26 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu Debug("streaming: lifecycle tick — derived snapshot") } - // Plan start = chunkID(floor), RAISED to lowestMaterializedChunk when higher - // — the production-boundary rule (never plan below existing storage). - start := ChunkIDOfLedger(floor) - low, hasLow, err := lowestMaterializedChunk(cat) - if cfg.abortTick(ctx, err, "lowest materialized chunk") { - return - } - if hasLow && int64(low) > start { - start = int64(low) - } - - // Stage 1 — plan-and-execute (freeze + index fold). + // Stage 1 — plan-and-execute (freeze + index fold) over [floor, lastChunk], via + // the same entry point backfill uses (resolve → executePlan → Freeze metric, + // recorded internally). A canceled ctx makes RunBackfill return ctx.Err(), + // which abortTick treats as a clean shutdown (no Fatalf). // - // rangeEnd is lastChunk CLAMPED to the highest durably-complete chunk: the - // production stage must never target the live or not-yet-complete chunk (whose - // hot DB ingestion holds open). In the running daemon lastChunk IS that chunk, - // so the clamp is a no-op; it only bites on seed/young-network/recovery edges. - // No complete chunk ⇒ empty range, production skipped, scans below still run. + // No rangeEnd clamp to the highest-complete chunk and no floor raise to + // lowestMaterializedChunk (both traced dead, #25): the Loop only ever fires for + // a genuinely completed lastChunk (the upstream boundary-handoff fence + seed + // guard), and recovery leaves chunk-aligned watermarks, so neither clamp can + // fire with a consequence beyond re-download churn. The only guard left is the + // empty-range check (floor above lastChunk when retention outran production). freezeStart := time.Now() - durableThrough, derr := LastCommittedLedger(cat, nil) // chunk-granularity, no hot DB read - if cfg.abortTick(ctx, derr, "derive durable through") { - return - } - highestComplete, haveComplete := lastCompleteChunkAtID(durableThrough) - rangeEnd := lastChunk - if haveComplete && highestComplete < rangeEnd { - rangeEnd = highestComplete - } - if haveComplete && start >= 0 && start <= int64(rangeEnd) { - // Plan-and-execute over [start, rangeEnd] via the same entry point backfill - // uses (resolve → executePlan → Freeze metric, recorded internally). A - // canceled ctx makes RunBackfill return ctx.Err(), which abortTick treats - // as a clean shutdown (no Fatalf). - eerr := backfill.RunBackfill(ctx, cfg.ExecConfig, chunk.ID(start), rangeEnd) //nolint:gosec // start >= 0 - if cfg.abortTick(ctx, eerr, fmt.Sprintf("run backfill [%d,%s]", start, rangeEnd)) { + start := ChunkIDOfLedger(floor) + if start >= 0 && start <= int64(lastChunk) { + eerr := backfill.RunBackfill(ctx, cfg.ExecConfig, chunk.ID(start), lastChunk) //nolint:gosec // start in [0, lastChunk] + if cfg.abortTick(ctx, eerr, fmt.Sprintf("run backfill [%d,%s]", start, lastChunk)) { return } } else { - // No complete chunk in range: skip production but report an empty freeze so + // floor above lastChunk: nothing to produce, but report an empty freeze so // the empty-tick rate stays visible. Scans below still run. metrics.Freeze(time.Since(freezeStart)) } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_arith_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_arith_test.go index 2c71ff8fb..e386e1432 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_arith_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_arith_test.go @@ -93,26 +93,3 @@ func TestEffectiveRetentionFloor(t *testing.T) { } } -// --------------------------------------------------------------------------- -// lowestMaterializedChunk. -// --------------------------------------------------------------------------- - -func TestLowestMaterializedChunk(t *testing.T) { - t.Run("empty catalog => ok=false", func(t *testing.T) { - cat, _ := testCatalog(t) - _, ok, err := lowestMaterializedChunk(cat) - require.NoError(t, err) - require.False(t, ok) - }) - - t.Run("min over chunk artifact keys and hot keys", func(t *testing.T) { - cat, _ := testCatalog(t) - freezeKinds(t, cat, 7, geometry.KindLedgers) // chunk artifact key at 7 - require.NoError(t, cat.PutHotTransient(4)) // hot key at 4 (lower) - freezeKinds(t, cat, 9, geometry.KindEvents) - low, ok, err := lowestMaterializedChunk(cat) - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, chunk.ID(4), low) - }) -} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go index ed2a1d6fa..83db02ab5 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go @@ -15,6 +15,7 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) @@ -139,18 +140,29 @@ func (r *fatalRecorder) fatalf(format string, args ...any) { func (r *fatalRecorder) fired() bool { return r.count.Load() > 0 } -// runTickForCatalog runs one lifecycle tick the way ingestion would drive it: -// it derives the highest complete chunk from the catalog (the chunk id ingestion -// hands over at a boundary) and passes it as lastChunk. A negative result (young -// network, no complete chunk) is passed as chunk 0 — the resolve range guard -// then makes the plan empty, matching the design's young-network no-op. +// lastCompleteChunkAtID maps geometry.LastCompleteChunkAt to a chunk.ID (ok=false +// on a negative result). Was a production helper until #25 (the tick now plans +// [floor, lastChunk] without it); it lives here for the tick-mirroring helpers. +func lastCompleteChunkAtID(ledger uint32) (chunk.ID, bool) { + c := geometry.LastCompleteChunkAt(ledger) + if c < 0 { + return 0, false + } + return chunk.ID(c), true //nolint:gosec // c >= 0 +} + +// runTickForCatalog runs one lifecycle tick the way ingestion would drive it: it +// derives the highest complete chunk from the catalog (the chunk id ingestion +// hands over at a boundary) and passes it as lastChunk. On a young network with no +// complete chunk it runs no tick — mirroring production, where the boundary/seed +// guard upstream never triggers the Loop in that state. func runTickForCatalog(ctx context.Context, t *testing.T, cfg Config, cat *catalog.Catalog) { t.Helper() through, err := deriveCompleteThrough(cat) require.NoError(t, err) last, ok := lastCompleteChunkAtID(through) if !ok { - last = 0 + return } runLifecycle(ctx, cfg, cat, last) } @@ -173,12 +185,7 @@ func assertQuiescent(t *testing.T, cfg Config, cat *catalog.Catalog, through uin require.NoError(t, err) floor := EffectiveRetentionFloor(through, cfg.RetentionChunks, earliest) start := ChunkIDOfLedger(floor) - low, hasLow, err := lowestMaterializedChunk(cat) - require.NoError(t, err) - if hasLow && int64(low) > start { - start = int64(low) - } - if rangeEnd, ok := lastCompleteChunkAtID(through); ok && start >= 0 { + if rangeEnd, ok := lastCompleteChunkAtID(through); ok && start >= 0 && start <= int64(rangeEnd) { // At quiescence resolve finds an empty plan, so RunBackfill (resolve + // executePlan) is a no-op that returns nil — even with no Backend wired, // since an empty plan never reaches backfillSource. From cf6fe6236878728e339a6d6502e17eab78810da0 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 16:31:43 -0400 Subject: [PATCH 35/55] fullhistory: lifecycle returns error, joined with ingestion via errgroup (#16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runLifecycle/Loop now return their error instead of aborting via an injected Fatalf. run() joins the ingestion loop and the lifecycle loop under one errgroup on a shared child ctx: whichever returns first cancels the other, and g.Wait joins both before run returns — supervise is now the single fatal-vs-restart decision point (a canceled parent ctx classifies as clean). Deletes lifecycle.Config.Fatalf + abortTick + StartConfig.Fatalf + the fatalRecorder test double; the two lifecycle Fatalf-policy tests collapse into one FailureReturnsError (ctx-cancel-is-clean is a supervise concern, covered at the daemon level). A runOps helper dedups the identical discard/prune loops and adds a between-ops ctx check, dropping runLifecycle under the cyclop limit so the nolint comes off. --- .../fullhistory/lifecycle/lifecycle.go | 93 ++++++++-------- .../lifecycle/lifecycle_helpers_test.go | 46 +++----- .../lifecycle/lifecycle_loop_test.go | 38 +++---- .../fullhistory/lifecycle/lifecycle_test.go | 65 ++++------- .../fullhistory/lifecycle/retention_test.go | 2 +- .../internal/fullhistory/startup.go | 103 +++++++++--------- .../internal/fullhistory/startup_test.go | 13 +-- 7 files changed, 150 insertions(+), 210 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index 94440ec0a..cc5684858 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -3,7 +3,6 @@ package lifecycle import ( "context" "fmt" - "log" "sync/atomic" "time" @@ -27,56 +26,47 @@ import ( // Config bundles the tick/loop dependencies. It composes the scheduler's // ExecConfig (shared postconditions + worker pool with backfill) plus the -// retention knob and an injectable fatal sink. +// retention knob. type Config struct { backfill.ExecConfig // RetentionChunks bounds the sliding retention floor's width. 0 disables the // sliding floor (the fixed earliest-ledger floor alone applies). RetentionChunks uint32 - - // Fatalf aborts the daemon on a tick op failure. WithLifecycleDefaults fills - // log.Fatalf when unset; tests override it. - Fatalf func(format string, args ...any) } -// WithLifecycleDefaults returns a copy with ExecConfig and Fatalf defaults +// WithLifecycleDefaults returns a copy with the embedded ExecConfig defaults // applied. Called once at startup before launching the loop. func (cfg Config) WithLifecycleDefaults() Config { cfg.ExecConfig = cfg.WithDefaults() - if cfg.Fatalf == nil { - cfg.Fatalf = log.Fatalf - } return cfg } -// abortTick centralizes the tick's error policy so each stage is one line. -// nil err → false (continue). A non-nil err aborts the tick (returns true); it -// calls Fatalf only when ctx is still live — a canceled ctx is a clean -// shutdown, not a failure. "what" names the failing step. -func (cfg Config) abortTick(ctx context.Context, err error, what string) bool { - if err == nil { - return false - } - if ctx.Err() == nil { - cfg.Fatalf("lifecycle tick: %s: %v", what, err) +// runOps runs each op in order, returning the first error. It checks ctx between +// ops so a shutdown mid-scan stops promptly without starting the next storage op; +// the ctx error is surfaced up through Loop for supervise to classify as clean. +func runOps(ctx context.Context, ops []func() error) error { + for _, op := range ops { + if err := ctx.Err(); err != nil { + return err + } + if err := op(); err != nil { + return err + } } - return true + return nil } - // runLifecycle runs one tick over the three stages for just-completed chunk // lastChunk. through = lastChunk.LastLedger() is the single snapshot every stage // shares, so a boundary committing mid-tick can't make stages contradict (it's // next tick's work). Plan range is [floor, lastChunk] (start raised to storage); // discard/prune key off through. // -// CLEAN-SHUTDOWN (binding): on an op error with ctx canceled, return WITHOUT -// Fatalf — cancellation is a shutdown, not a failure. Only a genuine failure -// (ctx still live) aborts via Fatalf. -// -//nolint:cyclop // linear 3-stage pipeline; the branch count is uniform abortTick guards, not real complexity -func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChunk chunk.ID) { +// It returns the first stage error WITHOUT classifying it: Loop propagates it and +// supervise is the single fatal-vs-restart decision point (a canceled ctx surfaces +// as a ctx error supervise treats as a clean shutdown). No os.Exit, no Fatalf. +func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChunk chunk.ID) error { metrics := observability.MetricsOrNop(cfg.Metrics) logger := cfg.Logger @@ -84,8 +74,8 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu through := lastChunk.LastLedger() earliest, _, err := cat.EarliestLedger() - if cfg.abortTick(ctx, err, "read earliest ledger") { - return + if err != nil { + return fmt.Errorf("read earliest ledger: %w", err) } floor := EffectiveRetentionFloor(through, cfg.RetentionChunks, earliest) @@ -99,8 +89,8 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu // Stage 1 — plan-and-execute (freeze + index fold) over [floor, lastChunk], via // the same entry point backfill uses (resolve → executePlan → Freeze metric, - // recorded internally). A canceled ctx makes RunBackfill return ctx.Err(), - // which abortTick treats as a clean shutdown (no Fatalf). + // recorded internally). A canceled ctx makes RunBackfill return ctx.Err(), which + // propagates up for supervise to treat as a clean shutdown. // // No rangeEnd clamp to the highest-complete chunk and no floor raise to // lowestMaterializedChunk (both traced dead, #25): the Loop only ever fires for @@ -111,9 +101,8 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu freezeStart := time.Now() start := ChunkIDOfLedger(floor) if start >= 0 && start <= int64(lastChunk) { - eerr := backfill.RunBackfill(ctx, cfg.ExecConfig, chunk.ID(start), lastChunk) //nolint:gosec // start in [0, lastChunk] - if cfg.abortTick(ctx, eerr, fmt.Sprintf("run backfill [%d,%s]", start, lastChunk)) { - return + if eerr := backfill.RunBackfill(ctx, cfg.ExecConfig, chunk.ID(start), lastChunk); eerr != nil { //nolint:gosec // start in [0, lastChunk] + return fmt.Errorf("run backfill [%d,%s]: %w", start, lastChunk, eerr) } } else { // floor above lastChunk: nothing to produce, but report an empty freeze so @@ -124,13 +113,11 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu // Stage 2 — discard scan. discardStart := time.Now() discardOps, err := eligibleDiscardOps(cfg, cat, through) - if cfg.abortTick(ctx, err, "eligible discard ops") { - return + if err != nil { + return fmt.Errorf("eligible discard ops: %w", err) } - for _, op := range discardOps { - if cfg.abortTick(ctx, op(), "discard op") { - return - } + if err := runOps(ctx, discardOps); err != nil { + return fmt.Errorf("discard op: %w", err) } metrics.Discard(len(discardOps), time.Since(discardStart)) if logger != nil && len(discardOps) > 0 { @@ -145,18 +132,17 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu // Stage 3 — prune scan. pruneStart := time.Now() pruneOps, prunedArtifacts, err := eligiblePruneOps(cfg, cat, through) - if cfg.abortTick(ctx, err, "eligible prune ops") { - return + if err != nil { + return fmt.Errorf("eligible prune ops: %w", err) } - for _, op := range pruneOps { - if cfg.abortTick(ctx, op(), "prune op") { - return - } + if err := runOps(ctx, pruneOps); err != nil { + return fmt.Errorf("prune op: %w", err) } metrics.Prune(prunedArtifacts, time.Since(pruneStart)) if logger != nil && prunedArtifacts > 0 { logger.WithField("pruned", prunedArtifacts).Info("streaming: lifecycle prune stage complete") } + return nil } // BoundarySignal couples ingestion (the producer) to the lifecycle Loop (the @@ -202,17 +188,24 @@ func (s *BoundarySignal) take() (chunk.ID, bool) { // wake, reads the latest completed chunk id, and runs one tick over // [floor, lastChunk] (which subsumes every boundary skipped while it was busy). It // selects on ctx.Done() too, so it never blocks past shutdown. -func Loop(ctx context.Context, cfg Config, cat *catalog.Catalog, sig *BoundarySignal) { +// +// It returns the first tick error to its caller (run() joins it with ingestion in +// an errgroup, so supervise is the single fatal-vs-restart point). A ctx +// cancellation returns nil — cancellation is a shutdown, and the sibling goroutine +// (ingestion, or an already-returned failing tick) carries any real cause. +func Loop(ctx context.Context, cfg Config, cat *catalog.Catalog, sig *BoundarySignal) error { for { select { case <-ctx.Done(): - return + return nil case <-sig.wake: lastChunk, ok := sig.take() if !ok { continue } - runLifecycle(ctx, cfg, cat, lastChunk) + if err := runLifecycle(ctx, cfg, cat, lastChunk); err != nil { + return err + } } } } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go index 83db02ab5..0bd376b3f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go @@ -2,8 +2,6 @@ package lifecycle import ( "context" - "fmt" - "sync/atomic" "testing" "github.com/stretchr/testify/assert" @@ -105,15 +103,14 @@ func ingestFullHotChunk(t *testing.T, cat *catalog.Catalog, c chunk.ID) { require.NoError(t, db.Close()) // release the write handle (boundary handoff) } -// lifecycleTestConfig wires a Config over the real production primitives plus a -// fatal recorder so a tick abort is observable instead of killing the test -// process. The freeze reads the hot tier by opening the chunk's real on-disk DB -// (created by ingestFullHotChunk) straight from its Layout path — the same open -// production does after #22. -func lifecycleTestConfig(t *testing.T, cat *catalog.Catalog, retentionChunks uint32) (Config, *fatalRecorder) { +// lifecycleTestConfig wires a Config over the real production primitives. The +// freeze reads the hot tier by opening the chunk's real on-disk DB (created by +// ingestFullHotChunk) straight from its Layout path — the same open production +// does after #22. A tick failure now surfaces as runLifecycle's returned error +// (no Fatalf), so tests assert on that error rather than a recorder. +func lifecycleTestConfig(t *testing.T, cat *catalog.Catalog, retentionChunks uint32) Config { t.Helper() - rec := &fatalRecorder{} - cfg := Config{ + return Config{ ExecConfig: backfill.ExecConfig{ Catalog: cat, Logger: silentLogger(), @@ -121,25 +118,9 @@ func lifecycleTestConfig(t *testing.T, cat *catalog.Catalog, retentionChunks uin Process: backfill.ProcessConfig{}, }, RetentionChunks: retentionChunks, - Fatalf: rec.fatalf, } - return cfg, rec } -// fatalRecorder captures Fatalf calls so a test can assert a tick did (or did -// NOT) abort the daemon. -type fatalRecorder struct { - count atomic.Int32 - last atomic.Value // string -} - -func (r *fatalRecorder) fatalf(format string, args ...any) { - r.count.Add(1) - r.last.Store(fmt.Sprintf(format, args...)) -} - -func (r *fatalRecorder) fired() bool { return r.count.Load() > 0 } - // lastCompleteChunkAtID maps geometry.LastCompleteChunkAt to a chunk.ID (ok=false // on a negative result). Was a production helper until #25 (the tick now plans // [floor, lastChunk] without it); it lives here for the tick-mirroring helpers. @@ -153,18 +134,19 @@ func lastCompleteChunkAtID(ledger uint32) (chunk.ID, bool) { // runTickForCatalog runs one lifecycle tick the way ingestion would drive it: it // derives the highest complete chunk from the catalog (the chunk id ingestion -// hands over at a boundary) and passes it as lastChunk. On a young network with no -// complete chunk it runs no tick — mirroring production, where the boundary/seed -// guard upstream never triggers the Loop in that state. -func runTickForCatalog(ctx context.Context, t *testing.T, cfg Config, cat *catalog.Catalog) { +// hands over at a boundary) and passes it as lastChunk, returning the tick's +// error. On a young network with no complete chunk it runs no tick (returns nil) — +// mirroring production, where the boundary/seed guard upstream never triggers the +// Loop in that state. +func runTickForCatalog(ctx context.Context, t *testing.T, cfg Config, cat *catalog.Catalog) error { t.Helper() through, err := deriveCompleteThrough(cat) require.NoError(t, err) last, ok := lastCompleteChunkAtID(through) if !ok { - return + return nil } - runLifecycle(ctx, cfg, cat, last) + return runLifecycle(ctx, cfg, cat, last) } // makeReadyHotDirNoData opens and closes a real (empty) hot DB for c so its dir diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go index 326cebad3..d8702a11e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_loop_test.go @@ -21,7 +21,7 @@ import ( // blocks forever and never fatals on shutdown. func TestLifecycleLoop_RunsTickPerNotifyThenStopsOnCtx(t *testing.T) { cat, _ := smallTxHashIndexCatalog(t, 1) - cfg, rec := lifecycleTestConfig(t, cat, 0) + cfg := lifecycleTestConfig(t, cat, 0) // Make the tick observable WITHOUT a slow full ingest: chunk 0 is already // fully frozen and folded into its (terminal, cpi=1) window, with a leftover @@ -35,22 +35,19 @@ func TestLifecycleLoop_RunsTickPerNotifyThenStopsOnCtx(t *testing.T) { sig := NewBoundarySignal() ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - Loop(ctx, cfg, cat, sig) - close(done) - }() + done := make(chan error, 1) + go func() { done <- Loop(ctx, cfg, cat, sig) }() sig.Publish(chunk.ID(0)) // ingestion hands over the just-completed chunk 0 require.Eventually(t, func() bool { has, err := hotKeyExists(cat, 0) return err == nil && !has }, 10*time.Second, 20*time.Millisecond, "the signal ran a tick that discarded chunk 0") - require.False(t, rec.fired()) cancel() select { - case <-done: + case err := <-done: + require.NoError(t, err, "a ctx-canceled Loop is a clean return") case <-time.After(5 * time.Second): t.Fatal("the loop did not return on ctx cancellation") } @@ -62,7 +59,7 @@ func TestLifecycleLoop_RunsTickPerNotifyThenStopsOnCtx(t *testing.T) { // 2, both are discarded (whether that takes one coalesced tick or two). func TestLifecycleLoop_DrainsToMostRecent(t *testing.T) { cat, _ := smallTxHashIndexCatalog(t, 1) - cfg, rec := lifecycleTestConfig(t, cat, 0) + cfg := lifecycleTestConfig(t, cat, 0) for c := chunk.ID(0); c <= 1; c++ { freezeKinds(t, cat, c, geometry.KindLedgers, geometry.KindEvents, geometry.KindTxHash) @@ -75,11 +72,8 @@ func TestLifecycleLoop_DrainsToMostRecent(t *testing.T) { sig := NewBoundarySignal() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - done := make(chan struct{}) - go func() { - Loop(ctx, cfg, cat, sig) - close(done) - }() + done := make(chan error, 1) + go func() { done <- Loop(ctx, cfg, cat, sig) }() sig.Publish(chunk.ID(0)) sig.Publish(chunk.ID(1)) // latest-cell coalesces: a tick over [floor, 1] discards both @@ -88,11 +82,11 @@ func TestLifecycleLoop_DrainsToMostRecent(t *testing.T) { h1, e1 := hotKeyExists(cat, 1) return e0 == nil && e1 == nil && !h0 && !h1 }, 10*time.Second, 20*time.Millisecond, "one drained tick discarded both completed chunks") - require.False(t, rec.fired()) cancel() select { - case <-done: + case err := <-done: + require.NoError(t, err, "a ctx-canceled Loop is a clean return") case <-time.After(5 * time.Second): t.Fatal("the loop did not return on ctx cancellation") } @@ -103,19 +97,17 @@ func TestLifecycleLoop_DrainsToMostRecent(t *testing.T) { // channel forever). func TestLifecycleLoop_ReturnsImmediatelyOnAlreadyCancelledCtx(t *testing.T) { cat, _ := smallTxHashIndexCatalog(t, 1) - cfg, _ := lifecycleTestConfig(t, cat, 0) + cfg := lifecycleTestConfig(t, cat, 0) ctx, cancel := context.WithCancel(context.Background()) cancel() sig := NewBoundarySignal() // never published to - done := make(chan struct{}) - go func() { - Loop(ctx, cfg, cat, sig) - close(done) - }() + done := make(chan error, 1) + go func() { done <- Loop(ctx, cfg, cat, sig) }() select { - case <-done: + case err := <-done: + require.NoError(t, err, "an already-canceled ctx is a clean return") case <-time.After(5 * time.Second): t.Fatal("the loop blocked instead of observing the canceled ctx") } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go index 253d3d340..3f6787265 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) @@ -31,15 +30,14 @@ func TestRunLifecycleTick_BoundaryFreezesFoldsDiscards(t *testing.T) { // tests to fit the gate's go-test timeout. t.Parallel() cat, _ := smallTxHashIndexCatalog(t, 1) // window w == chunk w; a one-chunk window finalizes immediately - cfg, rec := lifecycleTestConfig(t, cat, 0) + cfg := lifecycleTestConfig(t, cat, 0) // Chunk 0: just-closed, full hot DB on disk. Chunk 1: the new live chunk. ingestFullHotChunk(t, cat, 0) live := openLiveHotDB(t, cat, 1) // the live chunk's hot DB (held open by "ingestion") t.Cleanup(func() { _ = live.Close() }) - runTickForCatalog(context.Background(), t, cfg, cat) - require.False(t, rec.fired(), "a healthy tick never aborts: %v", rec.last.Load()) + require.NoError(t, runTickForCatalog(context.Background(), t, cfg, cat), "a healthy tick never fails") // Chunk 0's cold artifacts are all frozen. for _, kind := range []geometry.Kind{geometry.KindLedgers, geometry.KindEvents} { @@ -81,7 +79,7 @@ func TestRunLifecycleTick_BoundaryFreezesFoldsDiscards(t *testing.T) { // the discard fire. cpi=2 so a single chunk does NOT finalize the window. func TestRunLifecycleTick_DiscardGatedOnIndexCoverage(t *testing.T) { cat, _ := smallTxHashIndexCatalog(t, 2) // window 0 = chunks [0,1] - cfg, _ := lifecycleTestConfig(t, cat, 0) + cfg := lifecycleTestConfig(t, cat, 0) // Pre-freeze chunk 0's ledgers+events+txhash directly (no hot dependence), and // leave it with a "ready" hot DB on disk. The window is NOT finalized (cpi=2, @@ -120,7 +118,7 @@ func TestRunLifecycleTick_DiscardGatedOnIndexCoverage(t *testing.T) { // retention floor has its artifact files and hot DB swept, regardless of state. func TestRunLifecycleTick_PastFloorPrune(t *testing.T) { cat, _ := smallTxHashIndexCatalog(t, 1) - cfg, rec := lifecycleTestConfig(t, cat, 2) // retain ~2 chunks + cfg := lifecycleTestConfig(t, cat, 2) // retain ~2 chunks // CompleteThrough will be chunk 5's last ledger (positional: live chunk 6). // floor = geometry.LastCompleteChunkAt(through)-retention+1 = 5-2+1 = chunk 4's first @@ -141,8 +139,7 @@ func TestRunLifecycleTick_PastFloorPrune(t *testing.T) { floor := EffectiveRetentionFloor(through, cfg.RetentionChunks, 0) require.Equal(t, chunk.ID(4).FirstLedger(), floor, "floor anchors 2 chunks back") - runTickForCatalog(context.Background(), t, cfg, cat) - require.False(t, rec.fired(), "prune tick never aborts: %v", rec.last.Load()) + require.NoError(t, runTickForCatalog(context.Background(), t, cfg, cat), "prune tick never fails") // Chunks 0..3 (wholly below the floor) are gone: keys and files. for c := chunk.ID(0); c <= 3; c++ { @@ -168,7 +165,7 @@ func TestRunLifecycleTick_PastFloorPrune(t *testing.T) { // crashed build attempt) is swept regardless of window, even within retention. func TestRunLifecycleTick_PrunesTransientIndexDebris(t *testing.T) { cat, _ := smallTxHashIndexCatalog(t, 2) - cfg, rec := lifecycleTestConfig(t, cat, 0) + cfg := lifecycleTestConfig(t, cat, 0) // A crashed build left a "freezing" coverage key (no commit). _, err := cat.MarkTxHashIndexFreezing(0, 0, 0) @@ -181,7 +178,6 @@ func TestRunLifecycleTick_PrunesTransientIndexDebris(t *testing.T) { require.Len(t, ops, 1, "the freezing debris is swept") require.Equal(t, 1, artifacts, "one index artifact swept") require.NoError(t, ops[0]()) - require.False(t, rec.fired()) covs, err := cat.AllTxHashIndexKeys() require.NoError(t, err) @@ -189,44 +185,25 @@ func TestRunLifecycleTick_PrunesTransientIndexDebris(t *testing.T) { } // --------------------------------------------------------------------------- -// CLEAN SHUTDOWN: a ctx canceled mid-tick returns WITHOUT fatal. +// ERROR PLUMBING: a failing tick RETURNS its error (no Fatalf / os.Exit). +// supervise — not the tick — classifies ctx-cancel-is-clean vs restart (tested at +// the daemon level: TestRunDaemon_LoadValidateWireStartCleanShutdown, TestSupervise_*). // --------------------------------------------------------------------------- -// genuineFailureTickSetup wires a catalog whose chunk-0 build is GENUINELY -// unproducible: chunk 0 sits below a READY live chunk 1 (so it counts as complete -// and the plan range [0,0] is non-empty), has no frozen artifacts, and its hot key -// is "transient" (not a ready read source). With no bulk Backend configured (the -// lifecycleTestConfig default), backfillSource has no source for chunk 0 and -// RunBackfill fails with a non-cancellation error. MaxRetries defaults to 0, so it -// fails fast. Returns the config and the fatal recorder. -func genuineFailureTickSetup(t *testing.T) (Config, *fatalRecorder, *catalog.Catalog) { - t.Helper() +// TestRunLifecycleTick_FailureReturnsError: when a plan op fails, runLifecycle +// returns the wrapped error rather than aborting the process — so Loop can +// propagate it up through the errgroup to supervise. The chunk-0 build is +// GENUINELY unproducible: chunk 0 sits below a READY live chunk 1 (so it counts as +// complete and the plan range [0,0] is non-empty), has no frozen artifacts, and +// its hot key is "transient" (not a ready read source). With no bulk Backend +// configured, backfillSource has no source for chunk 0 and RunBackfill fails; +// MaxRetries defaults to 0, so it fails fast. +func TestRunLifecycleTick_FailureReturnsError(t *testing.T) { cat, _ := smallTxHashIndexCatalog(t, 1) - cfg, rec := lifecycleTestConfig(t, cat, 0) // hot tier read by path, no Backend + cfg := lifecycleTestConfig(t, cat, 0) // hot tier read by path, no Backend readyHot(t, cat, 1) // ready live chunk => through = chunk 0 last ledger require.NoError(t, cat.PutHotTransient(0)) // chunk 0 below live, no frozen artifacts, not a ready source - return cfg, rec, cat -} - -// TestRunLifecycleTick_CleanShutdownNoFatal: when RunBackfill returns an error AND -// ctx was canceled, the tick must NOT call Fatalf — cancellation is a shutdown, -// never an op failure. The chunk-0 build is genuinely unproducible (no source), but -// the canceled ctx takes precedence per the clean-shutdown policy. -func TestRunLifecycleTick_CleanShutdownNoFatal(t *testing.T) { - cfg, rec, cat := genuineFailureTickSetup(t) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() // shutdown requested before the tick runs - - runLifecycle(ctx, cfg, cat, 0) // lastChunk 0: plan range [0,0], build fails under a canceled ctx - require.False(t, rec.fired(), "a canceled ctx is a clean shutdown, NOT an op failure — no Fatalf") -} - -// TestRunLifecycleTick_GenuineFailureAborts: when a plan op fails for a real -// reason (NOT ctx cancellation), the tick aborts via Fatalf per the error policy. -func TestRunLifecycleTick_GenuineFailureAborts(t *testing.T) { - cfg, rec, cat := genuineFailureTickSetup(t) - runLifecycle(context.Background(), cfg, cat, 0) // lastChunk 0: plan range [0,0], the failing build - require.True(t, rec.fired(), "a genuine op failure aborts the daemon") + err := runLifecycle(context.Background(), cfg, cat, 0) // plan range [0,0], the failing build + require.Error(t, err, "a genuine op failure surfaces up the call stack") } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go index fec1a2c03..c43012ae4 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go @@ -135,7 +135,7 @@ func TestReaderRetention_WindowStraddlingFloorServesInRangeNotBelow(t *testing.T // chunk artifacts (chunks 0,1) are pruned. assert.False(t, floor.Excludes(wins.LastChunk(0)), "a straddling window is not wholly below the floor — its .idx is kept") - cfg, _ := lifecycleTestConfig(t, cat, 2) + cfg := lifecycleTestConfig(t, cat, 2) pops, _, err := eligiblePruneOps(cfg, cat, through) require.NoError(t, err) for _, op := range pops { diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index fe845fe56..ad8d238f3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -4,10 +4,10 @@ import ( "context" "errors" "fmt" - "sync" "time" "github.com/cenkalti/backoff/v4" + "golang.org/x/sync/errgroup" "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" @@ -20,12 +20,14 @@ import ( // run is the daemon's startup, in two steps: (1) BACKFILL to the // tip, then (2) SERVE + INGEST — open the resume chunk's hot DB, start captive -// core (injected), launch the lifecycle goroutine on a doorbell, begin serving -// reads (injected), and run the live ingestion loop. Returns nil only on a clean -// shutdown (ctx canceled mid-run, or the ingestion loop's clean stop); any other -// return is a restartable error the supervisor warns on and retries with backoff -// (a first start with no reachable backend, a backfill/ingest failure, or a -// "ready" hot DB that won't open — none are auto-healed, all are re-attempted). +// core (injected), begin serving reads (injected), then run the live ingestion +// loop and the lifecycle loop as a joined errgroup pair (whichever returns first +// tears down the other; g.Wait surfaces the first error). Returns nil only on a +// clean shutdown (ctx canceled mid-run, or the ingestion loop's clean stop); any +// other return is a restartable error the supervisor warns on and retries with +// backoff (a first start with no reachable backend, a backfill/ingest/lifecycle +// failure, or a "ready" hot DB that won't open — none are auto-healed, all are +// re-attempted). func run(ctx context.Context, cfg StartConfig) error { if err := cfg.validate(); err != nil { return err @@ -109,55 +111,56 @@ func run(ctx context.Context, cfg StartConfig) error { boundary.Publish(chunk.ID(seed)) //nolint:gosec // seed >= 0 } - // The lifecycle goroutine is tied to a PER-ITERATION child ctx (not the daemon - // ctx) and is canceled + JOINED before run returns for ANY reason — restoring - // the single-lifecycle-goroutine invariant across supervisor restarts (a - // daemon-ctx-tied loop would survive a restartable return and run a tick - // concurrently with the next iteration's lifecycle + ingestion: two backfill - // passes truncating the same .pack/.idx). Loop checks ctx at every step, so - // the join cannot block past the current step. - lifecycleCtx, cancelLifecycle := context.WithCancel(ctx) - // Assemble the lifecycle config from the SAME Exec wiring backfill uses, so - // the two share one catalog/pool by construction. WithLifecycleDefaults fills - // Fatalf when unset (Exec is already defaulted, so its re-default is a no-op). + // Assemble the lifecycle config from the SAME Exec wiring backfill uses, so the + // two share one catalog/pool by construction (Exec is already defaulted, so + // WithLifecycleDefaults' re-default is a no-op). lifecycleCfg := lifecycle.Config{ ExecConfig: cfg.Exec, RetentionChunks: cfg.RetentionChunks, - Fatalf: cfg.Fatalf, }.WithLifecycleDefaults() - var lifecycleWG sync.WaitGroup - lifecycleWG.Go(func() { - lifecycle.Loop(lifecycleCtx, lifecycleCfg, cat, boundary) - }) - // The two return paths registered after this defer (the ingestion-loop return - // and the ServeReads error path) have no live producer on the boundary signal — - // ingestion is a same-goroutine call whose inline Publish has stopped, and the - // serve path never starts it — so no Publish can race the cancel. - defer func() { - cancelLifecycle() - lifecycleWG.Wait() - }() - - // Begin serving reads (injected). It must return promptly (launch, not block). + + // Begin serving reads (injected) BEFORE launching the loops. It must return + // promptly (launch, not block); a serve failure just closes hotDB and returns, + // with no running goroutines to tear down. if err := cfg.ServeReads(ctx); err != nil { _ = hotDB.Close() return fmt.Errorf("startup serve reads: %w", err) } - // The ingestion loop is the hot-tier owner: it owns hotDB for the rest of its - // life (closes it on any exit, reopens at each boundary) and consumes the stream - // from the resume ledger. Returns the stream/boundary error; the daemon top level - // classifies a ctx-canceled return as a clean shutdown. - return runIngestionLoop(ctx, ingestionLoopConfig{ - Stream: stream, - Resume: resumeLedger, - HotDB: hotDB, - Catalog: cat, - Boundary: boundary, - Logger: logger, - Metrics: metrics, - Sink: cfg.Exec.Process.Sink, + // Ingestion and the lifecycle run as a JOINED pair under one per-iteration child + // ctx: whichever returns first cancels the other, and g.Wait joins BOTH before + // run returns for ANY reason — restoring the single-lifecycle-goroutine invariant + // across supervisor restarts (a surviving loop would run a tick concurrently with + // the next iteration's lifecycle + ingestion: two backfill passes truncating the + // same .pack/.idx). runLifecycle returns its error up through the group, so + // supervise is the ONE fatal-vs-restart decision point — no os.Exit in the tick. + // A parent-ctx cancel makes ingestion return a ctx error the group surfaces; + // supervise classifies a canceled parent as a clean shutdown. Loop checks ctx at + // every step, so the join cannot block past the current step. + loopCtx, cancelLoops := context.WithCancel(ctx) + defer cancelLoops() + var g errgroup.Group + g.Go(func() error { + defer cancelLoops() // ingestion stopping tears down the lifecycle loop + // The ingestion loop is the hot-tier owner: it owns hotDB for the rest of its + // life (closes it on any exit, reopens at each boundary) and consumes the + // stream from the resume ledger. + return runIngestionLoop(loopCtx, ingestionLoopConfig{ + Stream: stream, + Resume: resumeLedger, + HotDB: hotDB, + Catalog: cat, + Boundary: boundary, + Logger: logger, + Metrics: metrics, + Sink: cfg.Exec.Process.Sink, + }) }) + g.Go(func() error { + defer cancelLoops() // a tick error tears down ingestion + return lifecycle.Loop(loopCtx, lifecycleCfg, cat, boundary) + }) + return g.Wait() } // backfillToTip runs the backfill loop, returning lastCommitted as backfill makes @@ -288,10 +291,6 @@ type StartConfig struct { // diverge on the catalog/pool (the invariant is structural, not by comment). RetentionChunks uint32 - // Fatalf aborts the daemon on a lifecycle tick op failure; nil ⇒ the - // lifecycle default (log.Fatalf). Tests override it. - Fatalf func(format string, args ...any) - // NetworkTip samples the bulk backend's tip during backfill. Required. NetworkTip NetworkTipBackend @@ -316,8 +315,8 @@ const ( ) // withDefaults fills the tip-backoff defaults and the embedded Exec defaults -// (Workers -> GOMAXPROCS). The lifecycle.Config (including its Fatalf default) is -// assembled from Exec + RetentionChunks + Fatalf in run(). +// (Workers -> GOMAXPROCS). The lifecycle.Config is assembled from Exec + +// RetentionChunks in run(). func (cfg StartConfig) withDefaults() StartConfig { cfg.Exec = cfg.Exec.WithDefaults() if cfg.TipBackoff <= 0 { diff --git a/cmd/stellar-rpc/internal/fullhistory/startup_test.go b/cmd/stellar-rpc/internal/fullhistory/startup_test.go index 79b4ddfc2..4c83d7e2d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup_test.go @@ -95,14 +95,11 @@ func startTestConfig( cfg := StartConfig{ Exec: exec, RetentionChunks: 0, - // A tick op failure should fail the test loudly, not kill the process; the - // loop goroutine is joined before run() returns, so t.Errorf is safe here. - Fatalf: func(format string, args ...any) { t.Errorf("unexpected lifecycle fatal: "+format, args...) }, - NetworkTip: tip, - Core: core, - ServeReads: func(context.Context) error { return nil }, - TipBackoff: time.Millisecond, - TipMaxAttempts: 3, + NetworkTip: tip, + Core: core, + ServeReads: func(context.Context) error { return nil }, + TipBackoff: time.Millisecond, + TipMaxAttempts: 3, } if recordPlan != nil { cfg.runBackfill = func(_ context.Context, _ backfill.ExecConfig, lo, hi chunk.ID) error { From c6d018c66198be22cf9ad3d895b235a9d9fc9b8e Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 16:43:42 -0400 Subject: [PATCH 36/55] fullhistory/ingest: emit per-ingester ColdIngest off the Close path (#39-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cold ingester now emits its single per-chunk ColdIngest only on a terminal step: a Finalize (success or error) or its own Ingest error. Close no longer emits, so an ingester that was built but never ingested or finalized — a constructor-rollback sibling, or one abandoned mid-chunk by a failing sibling — produces NO phantom-success sample. Deletes the errColdBuildAborted sentinel, the coldAborter interface, the three abortMetric methods, and coldMetrics.recordErr; closeColdAll just closes and joins. Rewrites the two rollback tests and the sibling-failure test to assert zero per-ingester emit; the own-Ingest-failure test now sees its error sample emitted from Ingest rather than Close. Confirms tamirms 3515678800 (go emit-nothing; assert zero per-ingester samples). --- .../internal/fullhistory/ingest/driver.go | 29 ++---- .../internal/fullhistory/ingest/events.go | 30 +++---- .../fullhistory/ingest/ingest_test.go | 89 ++++++++----------- .../internal/fullhistory/ingest/ledgers.go | 19 ++-- .../internal/fullhistory/ingest/metrics.go | 28 +++--- .../internal/fullhistory/ingest/service.go | 12 +-- .../internal/fullhistory/ingest/txhash.go | 12 +-- 7 files changed, 83 insertions(+), 136 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go index 099a10bee..417769d08 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go @@ -13,30 +13,13 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) -// errColdBuildAborted is recorded against an already-built cold ingester when a -// LATER constructor fails and the build rolls back — without it, closing a -// fully-built ingester emits a clean ColdIngest, a phantom "success" for a chunk -// that ingested nothing. -var errColdBuildAborted = errors.New("ingest: cold ingester build aborted (sibling constructor failed)") - -// coldAborter is implemented by the concrete cold ingesters so the -// constructor-rollback path can mark their per-chunk metric as aborted before -// Close emits it, turning what would be a phantom success into a recorded -// abort. Optional: an ingester that does not implement it just gets its normal -// Close emission. -type coldAborter interface { - abortMetric(err error) -} - // closeColdAll closes every cold ingester built so far, joining each Close error -// into err. Used when a LATER constructor fails mid-build: the already-built -// ingesters never ingested anything, so each one's metric is first marked -// aborted (so the deferred Close emit is not a phantom success). +// into err. Used when a LATER constructor fails mid-build. The already-built +// ingesters never ingested or finalized, and Close no longer emits a per-ingester +// ColdIngest, so a rolled-back build produces no phantom-success sample — no +// abort bookkeeping needed here. func closeColdAll(ings []ColdIngester, err error) error { for _, ing := range ings { - if a, ok := ing.(coldAborter); ok { - a.abortMetric(errColdBuildAborted) - } if cerr := ing.Close(); cerr != nil { err = errors.Join(err, fmt.Errorf("close: %w", cerr)) } @@ -196,8 +179,8 @@ func WriteColdChunk( ings, berr := buildColdIngesters(dirs, chunkID, sink, cfg) if berr != nil { - // A constructor failure is still a chunk attempt - // (closeColdAll only emitted the per-ingester aborts). + // A constructor failure is still a chunk attempt: emit the aggregate + // (closeColdAll rolled back the built ingesters with no per-ingester emit). sink.ColdChunkTotal(time.Since(start)) return berr } diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/events.go b/cmd/stellar-rpc/internal/fullhistory/ingest/events.go index f072e90fe..bb51e890b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/events.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/events.go @@ -70,11 +70,13 @@ func NewEventsColdIngester(coldDir string, chunkID chunk.ID, sink MetricSink) (C func (e *eventsCold) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { start := time.Now() n, ierr := e.ingestSeq(seq, lcm) + e.metrics.observe(time.Since(start), n, ierr) if ierr != nil { e.failed = true + e.metrics.emit(0, nil) // an Ingest error abandons the chunk; meter it now (Close no longer emits) + return ierr } - e.metrics.observe(time.Since(start), n, ierr) - return ierr + return nil } // Finalize writes the events.pack trailer (Finish) + materializes the cold @@ -88,9 +90,9 @@ func (e *eventsCold) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMe func (e *eventsCold) Finalize(ctx context.Context) error { start := time.Now() if e.failed { - err := fmt.Errorf("events cold ingester for chunk %s: Finalize after failed Ingest", e.chunkID) - e.metrics.emit(time.Since(start), err) - return err + // Ingest already metered and latched this failure; refuse to finalize a + // chunk whose mirror/pack may be ahead of the offsets commit point. + return fmt.Errorf("events cold ingester for chunk %s: Finalize after failed Ingest", e.chunkID) } if err := e.writer.Finish(e.offsets); err != nil { err = fmt.Errorf("events ColdWriter.Finish: %w", err) @@ -111,16 +113,12 @@ func (e *eventsCold) Finalize(ctx context.Context) error { return nil } -// Close drops the partial events.pack when Finalize never ran, and emits the -// cold metrics if Finalize did not already (the failure path). The writer.Close -// error is folded into the emitted metric so a close-time failure (e.g. ENOSPC -// on the partial-drop) is counted in errors_total. emit is a no-op after a -// successful Finalize. Error propagation is unchanged: the writer.Close error is -// still returned. +// Close drops the partial events.pack when Finalize never ran. It does NOT emit +// the cold metric: a terminal Ingest error or Finalize already emitted it, and an +// ingester that never got that far (a rolled-back build) must produce no phantom +// sample. The writer.Close error is returned unchanged. func (e *eventsCold) Close() error { - cerr := e.writer.Close() - e.metrics.emit(0, cerr) - return cerr + return e.writer.Close() } // ingestSeq writes one ledger's events and returns the count written. The @@ -187,7 +185,3 @@ func (e *eventsCold) ingestSeq(seq uint32, lcm xdr.LedgerCloseMetaView) (int, er } return len(payloads), nil } - -// abortMetric records a synthetic abort error so a subsequent Close emit does -// not look like a clean success. Used by the constructor-rollback path. -func (e *eventsCold) abortMetric(err error) { e.metrics.recordErr(err) } diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go index 3e60b0baa..ea96b0f43 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go @@ -717,19 +717,18 @@ func (f *failingCold) Ingest(context.Context, uint32, xdr.LedgerCloseMetaView) e func (f *failingCold) Finalize(context.Context) error { f.finalized = true; return nil } func (f *failingCold) Close() error { f.closed = true; return nil } -// TestColdService_FailurePath_NoArtifact uses a real ledger cold ingester plus a +// TestColdService_FailurePath_NoArtifact uses two real cold ingesters plus a // failing sibling: ColdService.Ingest returns the sibling's error, Finalize is // not called, the deferred Close drops the partial ledger pack, and no finalized -// artifact remains. It also asserts the cold metrics still fire on this failure -// path: each real ingester emits exactly one ColdIngest and the service emits one -// aggregate ColdChunkTotal — driven from Close, since Finalize never ran. +// artifact remains. It asserts the aggregate ColdChunkTotal still fires for the +// attempt, but the two real ingesters emit NO per-ingester ColdIngest: each +// ingested cleanly (no terminal error of its own) and never finalized, and Close +// no longer emits — so a chunk abandoned by a sibling leaves no phantom sample. func TestColdService_FailurePath_NoArtifact(t *testing.T) { chunkID := chunk.ID(0) coldDir := t.TempDir() sink := &testSink{} - // Two real cold ingesters (ledger + events) plus a failing sibling, so we can - // assert each real ingester emits its per-chunk ColdIngest from Close. realLedger, err := NewLedgerColdIngester(filepath.Join(coldDir, dataTypeLedgers), chunkID, sink) require.NoError(t, err) realEvents, err := NewEventsColdIngester(filepath.Join(coldDir, dataTypeEvents), chunkID, sink) @@ -743,19 +742,16 @@ func TestColdService_FailurePath_NoArtifact(t *testing.T) { require.ErrorIs(t, err, errFailingCold) require.False(t, failing.finalized, "Finalize must not run on the failure path") - // Before Close, no cold metric has fired (emission is deferred to Close on the - // failure path). - require.Empty(t, sink.coldDataTypes(), "no ColdIngest before Close on failure path") - require.Zero(t, sink.coldChunkTotals, "no ColdChunkTotal before Close on failure path") + // Nothing has emitted: the real ingesters ingested cleanly (no terminal error) + // and never finalized; the mock sibling records nothing. + require.Empty(t, sink.coldDataTypes(), "no per-ingester ColdIngest on the sibling-failure path") + require.Zero(t, sink.coldChunkTotals, "no ColdChunkTotal before Close") - // Close drops partials and drives the deferred metric emissions. + // Close drops partials and emits the aggregate only. require.NoError(t, service.Close()) require.True(t, failing.closed) - // Each real ingester emitted exactly one ColdIngest; the aggregate fired once. - cdt := sink.coldDataTypes() - require.Equal(t, 1, cdt[dataTypeLedgers], "ledger cold ingester emits once on failure path") - require.Equal(t, 1, cdt[dataTypeEvents], "events cold ingester emits once on failure path") + require.Empty(t, sink.coldDataTypes(), "a chunk abandoned by a sibling emits no per-ingester ColdIngest") require.Equal(t, 1, sink.coldChunkTotals, "exactly one aggregate ColdChunkTotal") // No finalized ledger pack must exist. @@ -768,8 +764,9 @@ func TestColdService_FailurePath_NoArtifact(t *testing.T) { // so its OWN Ingest fails (recording firstErr), then Close. The failure is an // out-of-order seq: the per-chunk ColdWriter expects the chunk's first ledger, // so AppendLedger rejects a later one. Per #765 a failed cold chunk must record -// a per-ingester error count and an aggregate duration sample. Emission happens -// exactly once (from Close), with the accumulated error carried. +// a per-ingester error count and an aggregate duration sample. A terminal Ingest +// error emits the single per-ingester ColdIngest right there (Close no longer +// emits), so the error-carrying sample is present after Ingest returns. func TestColdIngester_Failure_RecordsErrorMetric(t *testing.T) { chunkID := chunk.ID(0) coldDir := t.TempDir() @@ -780,12 +777,14 @@ func TestColdIngester_Failure_RecordsErrorMetric(t *testing.T) { service := NewColdService([]ColdIngester{realLedger}, sink) // An out-of-order seq makes the writer's own AppendLedger fail inside the - // ingester's Ingest, so it records its firstErr. (drain would never feed - // this — the test targets the ingester's metric path directly.) + // ingester's Ingest, so it records its firstErr and emits the error-carrying + // ColdIngest. (drain would never feed this — the test targets the ingester's + // metric path directly.) wrongSeq := chunkID.FirstLedger() + 5 require.Error(t, service.Ingest(context.Background(), wrongSeq, viewOf(t, wrongSeq))) + require.Equal(t, 1, sink.coldDataTypes()[dataTypeLedgers], "the failed Ingest emits its ColdIngest immediately") - // Finalize is skipped on this path; Close drives the single emission. + // Finalize is skipped on this path; Close emits nothing more. require.NoError(t, service.Close()) // Exactly one ColdIngest for ledgers, carrying the error, plus one aggregate. @@ -1133,13 +1132,13 @@ func countCleanColdIngests(s *testSink) int { return n } -// TestBuildColdIngesters_RollbackNoPhantomMetric makes a LATER constructor -// (txhash) fail by planting a regular file at the txhash per-type directory, -// so the constructor's own MkdirAll fails. The earlier-built ledger ingester -// is rolled back via closeColdAll, which must NOT emit a phantom success -// ColdIngest — the recorded ledger metric (if any) must carry the abort -// error, never a clean (nil-err, 0-items) success. -func TestBuildColdIngesters_RollbackNoPhantomMetric(t *testing.T) { +// TestBuildColdIngesters_RollbackOneBuilt makes a LATER constructor (txhash) fail +// by planting a regular file at the txhash per-type directory, so the +// constructor's own MkdirAll fails. The earlier-built ledger ingester is rolled +// back via closeColdAll — which only closes it. Since Close no longer emits a +// per-ingester ColdIngest, a rolled-back ingester (built, never ingested or +// finalized) produces NO sample at all: no phantom success, no synthetic abort. +func TestBuildColdIngesters_RollbackOneBuilt(t *testing.T) { chunkID := chunk.ID(0) coldDir := t.TempDir() sink := &testSink{} @@ -1152,24 +1151,16 @@ func TestBuildColdIngesters_RollbackNoPhantomMetric(t *testing.T) { _, err := buildColdIngesters(coldDirsAt(coldDir), chunkID, sink, Config{Ledgers: true, Txhash: true}) require.Error(t, err, "txhash constructor must fail on the planted file") - // The ledger ingester was built then rolled back. No phantom SUCCESS metric: - // any recorded ledger ColdIngest must carry an error. - cdt := sink.coldDataTypes() - if cdt[dataTypeLedgers] > 0 { - require.Equal(t, cdt[dataTypeLedgers], sink.coldErrorTypes()[dataTypeLedgers], - "rolled-back ledger ingester must not emit a phantom success ColdIngest") - } - // And the success-only assertion: there must be zero clean (nil-err) cold - // ingest signals recorded. - require.Zero(t, countCleanColdIngests(sink), "no clean ColdIngest on the rollback path") + // The ledger ingester was built then rolled back with no Ingest/Finalize, so + // it emits nothing. + require.Empty(t, sink.coldDataTypes(), "a rolled-back ingester emits no per-ingester ColdIngest") } -// TestBuildColdIngesters_RollbackLaterFailure_TxhashAborts makes the LAST -// constructor (events) fail AFTER both the ledger AND txhash ingesters were -// already built, so closeColdAll rolls back two ingesters. It asserts the txhash -// ingester (which DOES implement abortMetric) emits an error-carrying — not a -// clean-success — ColdIngest, complementing the ledger-only abort coverage above. -func TestBuildColdIngesters_RollbackLaterFailure_TxhashAborts(t *testing.T) { +// TestBuildColdIngesters_RollbackTwoBuilt makes the LAST constructor (events) +// fail AFTER both the ledger AND txhash ingesters were already built, so +// closeColdAll rolls back two ingesters. Same invariant at greater rollback +// depth: neither rolled-back ingester emits a per-ingester ColdIngest. +func TestBuildColdIngesters_RollbackTwoBuilt(t *testing.T) { chunkID := chunk.ID(0) coldDir := t.TempDir() sink := &testSink{} @@ -1184,15 +1175,9 @@ func TestBuildColdIngesters_RollbackLaterFailure_TxhashAborts(t *testing.T) { Config{Ledgers: true, Txhash: true, Events: true}) require.Error(t, err, "events constructor must fail on the planted directory") - // The txhash ingester was built then rolled back: its recorded ColdIngest must - // carry the abort error, never a clean success. - cdt := sink.coldDataTypes() - require.Equal(t, 1, cdt[dataTypeTxhash], "rolled-back txhash ingester emits one ColdIngest") - require.Equal(t, 1, sink.coldErrorTypes()[dataTypeTxhash], - "the rolled-back txhash ColdIngest must carry the abort error") - - // No phantom clean success on the rollback path for any ingester. - require.Zero(t, countCleanColdIngests(sink), "no clean ColdIngest on the rollback path") + // Both the ledger and txhash ingesters were built then rolled back with no + // Ingest/Finalize, so neither emits a per-ingester ColdIngest. + require.Empty(t, sink.coldDataTypes(), "rolled-back ingesters emit no per-ingester ColdIngest") } // TestWriteColdChunk_ConstructorFailure_EmitsAggregate drives a constructor failure diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go index 75192ea89..8c1cefe50 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go @@ -47,6 +47,7 @@ func (c *ledgerCold) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMe start := time.Now() if err := c.writer.AppendLedger(seq, []byte(lcm)); err != nil { c.metrics.observe(time.Since(start), 0, err) + c.metrics.emit(0, nil) // an Ingest error abandons the chunk; meter it now (Close no longer emits) return fmt.Errorf("AppendLedger(seq=%d): %w", seq, err) } c.metrics.sink.IngestStage(dataTypeLedgers, tierCold, stageWrite, time.Since(start), 1) @@ -74,18 +75,10 @@ func (c *ledgerCold) Finalize(_ context.Context) error { return nil } -// Close drops the partial pack when Finalize never ran, and emits the cold -// metrics if Finalize did not already (the failure path). The writer.Close -// error is folded into the emitted metric so a close-time failure is counted in -// errors_total. emit is a no-op after a successful Finalize, so this never -// double-counts. Error propagation is unchanged: the writer.Close error is -// still returned. +// Close drops the partial pack when Finalize never ran. It does NOT emit the cold +// metric: a terminal Ingest error or Finalize already emitted it, and an ingester +// that never got that far (a rolled-back build) must produce no phantom sample. +// The writer.Close error is returned unchanged. func (c *ledgerCold) Close() error { - cerr := c.writer.Close() - c.metrics.emit(0, cerr) - return cerr + return c.writer.Close() } - -// abortMetric records a synthetic abort error so a subsequent Close emit does -// not look like a clean success. Used by the constructor-rollback path. -func (c *ledgerCold) abortMetric(err error) { c.metrics.recordErr(err) } diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go index 30d3adc87..b49d8cbe1 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go @@ -91,13 +91,15 @@ func orNop(sink MetricSink) MetricSink { // coldMetrics is the per-chunk metric accumulator shared by all three cold // ingesters. Each ingester accumulates Ingest wall-clock (accum), item count // (items), and the FIRST error it saw (firstErr) across the chunk, then emits a -// single ColdIngest signal — in Finalize if reached, otherwise in Close (the -// failure path). The emitted flag guards against a double-emit: a successful -// Finalize emits and sets emitted=true so the deferred Close is a no-op, while a -// chunk that errors before Finalize emits exactly once from Close. +// single ColdIngest signal on a TERMINAL step only: Finalize (success or error), +// or an Ingest error (which abandons the chunk). Close NEVER emits — an ingester +// that was built but never ingested/finalized (e.g. a sibling constructor failed +// and the build rolled back) produces NO phantom sample. The emitted flag guards +// against a double-emit so the guarantee holds even if a defensive caller drives +// the terminal steps redundantly. // -// This guarantees: failed chunk → one ColdIngest with the error recorded; -// success → exactly one ColdIngest per ingester; never both. +// This guarantees: a chunk that ingested and then failed/finalized → exactly one +// ColdIngest (error recorded on failure); a rolled-back ingester → none. type coldMetrics struct { sink MetricSink dataType string @@ -111,15 +113,6 @@ func newColdMetrics(sink MetricSink, dataType string) coldMetrics { return coldMetrics{sink: orNop(sink), dataType: dataType} } -// recordErr folds err into firstErr WITHOUT emitting. Used on the -// constructor-rollback path so the subsequent Close emit carries the abort -// error instead of looking like a clean (nil-err, 0-items) success. -func (m *coldMetrics) recordErr(err error) { - if err != nil { - m.firstErr = errOrFirst(m.firstErr, err) - } -} - // observe records one Ingest's wall-clock and (on error) the first error. func (m *coldMetrics) observe(d time.Duration, items int, err error) { m.accum += d @@ -132,8 +125,9 @@ func (m *coldMetrics) observe(d time.Duration, items int, err error) { // emit reports the single ColdIngest signal for this ingester, adding extra to // the accumulated Ingest time (e.g. the Finalize wall-clock) and folding err // (if non-nil) into firstErr before reporting. It is a no-op after the first -// call, so calling it from both Finalize (success) and Close (deferred cleanup) -// emits exactly once. Pass a nil err when there is no stage error to record. +// call, so a redundant terminal-step call emits exactly once. Pass a nil err +// when the error is already recorded (an Ingest failure observes it) or there is +// none. func (m *coldMetrics) emit(extra time.Duration, err error) { if err != nil { m.firstErr = errOrFirst(m.firstErr, err) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go index a1eb0ed8c..f3e06c9ee 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go @@ -126,11 +126,13 @@ func (s *ColdService) Finalize(ctx context.Context) error { } // Close closes every cold ingester, joining each Close error, and emits the -// aggregate ColdChunkTotal if Finalize never reached it (the failure path). Each -// ingester's own Close in turn emits that ingester's per-chunk ColdIngest if its -// Finalize never ran, so a failed chunk still produces one per-ingester signal -// and one aggregate. Idempotent: on the failure path a writer's Close drops its -// partial file; after a successful Finalize all emissions are no-ops. +// aggregate ColdChunkTotal if Finalize never reached it (the failure path). The +// per-ingester ColdIngest is emitted on a terminal Ingest error or in Finalize, +// never from an ingester's Close — so a chunk that failed after ingesting still +// produced one per-ingester signal, while one rolled back before any work +// produces none (only the aggregate here). Idempotent: on the failure path a +// writer's Close drops its partial file; after a successful Finalize this is a +// no-op for the aggregate. func (s *ColdService) Close() error { var err error for _, ing := range s.ingesters { diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go b/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go index dfd667452..6fcb42ba0 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go @@ -61,6 +61,7 @@ func (t *txhashCold) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMe hashes, err := sdkingest.ExtractTxHashes(lcm) if err != nil { t.metrics.observe(time.Since(start), 0, err) + t.metrics.emit(0, nil) // an Ingest error abandons the chunk; meter it now (Close no longer emits) return fmt.Errorf("ExtractTxHashes seq %d: %w", seq, err) } for i := range hashes { @@ -100,14 +101,9 @@ func (t *txhashCold) Finalize(_ context.Context) error { return err } -// Close emits the cold metrics if Finalize never ran (the failure path); emit is -// a no-op after Finalize. There is no open file handle to release (the .bin is -// written in Finalize). +// Close is a no-op: there is no open file handle to release (the .bin is written +// in Finalize), and the cold metric is emitted on a terminal Ingest error or in +// Finalize — never here, so a rolled-back build produces no phantom sample. func (t *txhashCold) Close() error { - t.metrics.emit(0, nil) return nil } - -// abortMetric records a synthetic abort error so a subsequent Close emit does -// not look like a clean success. Used by the constructor-rollback path. -func (t *txhashCold) abortMetric(err error) { t.metrics.recordErr(err) } From 4ff5e5be972914358b697c7eb7f2d40d59e90f0f Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 16:49:54 -0400 Subject: [PATCH 37/55] fullhistory,events: halve hot per-ledger extraction to one walk (#18 part 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hotchunk.IngestLedger ran two full TxProcessing walks per ledger: ExtractTxHashes for the txhash CF, then LCMViewToPayloads (which internally runs ExtractLedgerEvents) for the events CF. ExtractLedgerEvents already yields, per transaction in apply order, the tx hash AND its contract events — so one walk feeds both. Factors PayloadsFromLedgerEvents out of events.LCMViewToPayloads (the 3-pass cursor-ordered shaping, minus the SDK walk); LCMViewToPayloads stays as the thin view-reading wrapper the cold path still uses. IngestLedger now calls ExtractLedgerEvents once, builds txhash entries from each element's Hash, and shapes events from the same slice. The shaping order is byte-identical, so hot event-ID assignment is unchanged (events cursor-order tests + non-short E2E green). Defers per-type metric attribution (the other half of the #18 thread) — after merging to one shared walk there is no per-type extraction boundary left to time, so the attribution shape needs a decision (see follow-up). --- cmd/stellar-rpc/internal/events/extract.go | 20 ++++++++++-- .../pkg/stores/hotchunk/hotchunk.go | 31 ++++++++++++++----- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/cmd/stellar-rpc/internal/events/extract.go b/cmd/stellar-rpc/internal/events/extract.go index b7055c4de..1c1c1b865 100644 --- a/cmd/stellar-rpc/internal/events/extract.go +++ b/cmd/stellar-rpc/internal/events/extract.go @@ -34,7 +34,8 @@ import ( // (ingest.ExtractLedgerEvents — one TxProcessing walk yields hash + events // together). This function adds only the RPC-specific Payload shape, the // Stage→(TxIdx, OpIdx) cursor-sentinel mapping, EventIdx, and the cursor -// ordering. +// ordering — all in PayloadsFromLedgerEvents, over which this is the thin +// view-reading wrapper. func LCMViewToPayloads(lcm xdr.LedgerCloseMetaView) ([]Payload, error) { ledgerSeq, err := lcm.LedgerSequence() if err != nil { @@ -44,11 +45,26 @@ func LCMViewToPayloads(lcm xdr.LedgerCloseMetaView) ([]Payload, error) { if err != nil { return nil, err } - txEvents, err := ingest.ExtractLedgerEvents(lcm) if err != nil { return nil, err } + return PayloadsFromLedgerEvents(txEvents, ledgerSeq, ledgerClosedAt) +} + +// PayloadsFromLedgerEvents shapes an already-extracted per-transaction event +// slice (ingest.ExtractLedgerEvents output) into cursor-ordered Payloads. It is +// the body of LCMViewToPayloads minus the SDK walk, so a caller that already +// holds the txEvents — the hot ingest path, which also needs the paired tx +// hashes (txEvents[i].Hash) — can feed BOTH txhash and events from ONE +// ExtractLedgerEvents call instead of walking TxProcessing twice. ledgerSeq and +// ledgerClosedAt are the view's header values (cheap reads, not a walk). The +// cursor ordering and EventIdx assignment are IDENTICAL to what LCMViewToPayloads +// produced inline, so event IDs are unchanged across the refactor. +func PayloadsFromLedgerEvents( + txEvents []ingest.LedgerTransactionEvents, ledgerSeq uint32, ledgerClosedAt int64, +) ([]Payload, error) { + var err error at := func(i int) (uint32, xdr.Hash) { return uint32(i) + 1, xdr.Hash(txEvents[i].Hash) //nolint:gosec // 1-based, matching ingest reader's tx.Index } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index c7653777d..485b7c30d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -173,20 +173,35 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts // Pre-extract anything that can fail BEFORE opening the batch, so a decode // error rejects the ledger without a half-built batch. - hashes, err := sdkingest.ExtractTxHashes(lcm) + // + // ONE TxProcessing walk feeds BOTH hot data types: ExtractLedgerEvents yields, + // per transaction in apply order, the tx hash AND its contract events. txhash + // reads each element's Hash and events shapes the same slice + // (PayloadsFromLedgerEvents), so the two share one walk instead of the two + // (ExtractTxHashes + LCMViewToPayloads-internal ExtractLedgerEvents) they used + // to each run — halving per-ledger extraction. Shaping the already-extracted + // slice (not re-walking) keeps the event-ID assignment order identical to + // LCMViewToPayloads. The atomic batch below serializes only the commit; the + // extractors are independent and could run concurrently into the same batch if + // catch-up profiling ever demands it — sequential is right at live cadence. + txEvents, err := sdkingest.ExtractLedgerEvents(lcm) if err != nil { - return counts, fmt.Errorf("extract tx hashes seq %d: %w", seq, err) + return counts, fmt.Errorf("extract ledger events seq %d: %w", seq, err) } - txEntries := make([]txhash.Entry, len(hashes)) - for i, h := range hashes { - txEntries[i] = txhash.Entry{Hash: [32]byte(h), LedgerSeq: seq} + txEntries := make([]txhash.Entry, len(txEvents)) + for i := range txEvents { + txEntries[i] = txhash.Entry{Hash: txEvents[i].Hash, LedgerSeq: seq} } - counts.Txhash = len(hashes) + counts.Txhash = len(txEntries) + closedAt, err := lcm.LedgerCloseTime() + if err != nil { + return counts, fmt.Errorf("ledger close time seq %d: %w", seq, err) + } // A pre-Soroban ledger yields zero payloads, no error. - payloads, err := events.LCMViewToPayloads(lcm) + payloads, err := events.PayloadsFromLedgerEvents(txEvents, seq, closedAt) if err != nil { - return counts, fmt.Errorf("LCMViewToPayloads seq %d: %w", seq, err) + return counts, fmt.Errorf("shape events seq %d: %w", seq, err) } counts.Events = len(payloads) counts.Ledgers = 1 From f360ad485cac73467adb0e93c0a226f50dc66337 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 16:59:55 -0400 Subject: [PATCH 38/55] fullhistory: unify the frozen-index coverage predicate (#37) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three call sites answered "is this chunk/range covered by a frozen index?" differently: progress's frozenCoverageContains scanned all windows and tolerated duplicate frozen coverages silently; eligibility's indexCovers and resolve's coverageRange.covers went through FrozenTxHashIndex, which errors on two frozen coverages per window. On an INV-2 violation progress would derive a resume point while the first tick's eligibility aborted — three predicates disagreeing about one snapshot. Adds catalog.FrozenIndexCoversRange (built on FrozenTxHashIndex, so INV-2 is asserted on every read) plus the per-chunk FrozenIndexCovers convenience. All three sites route through it: deletes frozenCoverageContains and indexCovers, and resolve's coverageRange.covers. New catalog test plants two frozen coverages and asserts both predicate forms surface the uniqueness violation. Confirms tamirms 3514888224. --- .../internal/fullhistory/backfill/resolve.go | 10 +---- .../internal/fullhistory/catalog/catalog.go | 23 ++++++++++ .../catalog/catalog_protocol_test.go | 23 ++++++++++ .../fullhistory/lifecycle/eligibility.go | 14 +----- .../fullhistory/lifecycle/lifecycle_test.go | 4 +- .../fullhistory/lifecycle/progress.go | 43 ++++++------------- 6 files changed, 64 insertions(+), 53 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/resolve.go b/cmd/stellar-rpc/internal/fullhistory/backfill/resolve.go index 8ea961990..6814e66c0 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/resolve.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/resolve.go @@ -30,11 +30,6 @@ type coverageRange struct { Lo, Hi chunk.ID } -// covers reports whether this range fully contains other (other ⊆ this). -func (r coverageRange) covers(other coverageRange) bool { - return r.Lo <= other.Lo && r.Hi >= other.Hi -} - // resolve diffs the desired state (every artifact of [rangeStart, rangeEnd] durable) // against the catalog, emitting a Plan. A pure read — recomputes from durable keys // every run, so a restart re-plans cleanly. @@ -98,12 +93,11 @@ func resolveTxHashIndex( Hi: min(txLayout.LastChunk(w), rangeEnd), // capped by range end } - frozen, hasFrozen, err := cat.FrozenTxHashIndex(w) + covered, err := cat.FrozenIndexCoversRange(w, desired.Lo, desired.Hi) if err != nil { return IndexBuild{}, false, err } - stored := coverageRange{Lo: frozen.Lo, Hi: frozen.Hi} - if hasFrozen && stored.covers(desired) { + if covered { // Frozen coverage already spans desired, so no rebuild is due — steady state, a // risen floor, or a finalized window. Any non-frozen leftover a crashed build // stranded (a superseded "pruning"/"freezing" coverage or a demoted .bin) is the diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go index a43f360c5..bbdf9f27f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog.go @@ -146,6 +146,29 @@ func (c *Catalog) FrozenTxHashIndex(w geometry.TxHashIndexID) (geometry.TxHashIn return frozen, found, nil } +// FrozenIndexCoversRange reports whether index w's UNIQUE frozen coverage spans +// the whole inclusive [lo, hi] chunk range. It reads through FrozenTxHashIndex, +// so INV-2 (at most one frozen coverage per index) is asserted on every call. +// This is the single "covered by a frozen index" predicate the resolve diff +// (backfill), the discard eligibility scan, and the watermark derivation all +// share, so they can never disagree about the same catalog snapshot. Reports +// false (no error) when the index has no frozen coverage yet. +func (c *Catalog) FrozenIndexCoversRange(w geometry.TxHashIndexID, lo, hi chunk.ID) (bool, error) { + frozen, ok, err := c.FrozenTxHashIndex(w) + if err != nil { + return false, err + } + return ok && frozen.Lo <= lo && hi <= frozen.Hi, nil +} + +// FrozenIndexCovers reports whether chunk ch's OWN index window has a frozen +// coverage containing it. A chunk belongs to exactly one window, so its own +// window is the only one that can cover it — the degenerate single-chunk case of +// FrozenIndexCoversRange. +func (c *Catalog) FrozenIndexCovers(ch chunk.ID) (bool, error) { + return c.FrozenIndexCoversRange(c.txhashIndex.TxHashIndexID(ch), ch, ch) +} + // --------------------------------------------------------------------------- // Config pins. Written once on first start, immutable thereafter. // --------------------------------------------------------------------------- diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol_test.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol_test.go index b83590657..f70e9e536 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_protocol_test.go @@ -43,6 +43,29 @@ func TestCommitIndexPromoteAndDemote(t *testing.T) { require.Equal(t, geometry.StateFrozen, states[geometry.TxHashIndexKey(5, 5100, 5350)]) } +// TestFrozenIndexCoversRange_AssertsUniqueness pins that the shared "covered by a +// frozen index" predicate (#37) propagates the INV-2 assertion FrozenTxHashIndex +// makes: two frozen coverages in one window must make EVERY read error, so +// watermark derivation (progress), discard eligibility, and the resolve diff can +// never disagree — one silently tolerating the duplicate while another aborts. +func TestFrozenIndexCoversRange_AssertsUniqueness(t *testing.T) { + cat, _ := testCatalog(t) + + // Plant two frozen coverages in window 5, bypassing the promote/demote commit + // path (which never leaves two frozen) to stage the corrupt snapshot directly. + require.NoError(t, cat.store.Put(geometry.TxHashIndexKey(5, 5100, 5349), string(geometry.StateFrozen))) + require.NoError(t, cat.store.Put(geometry.TxHashIndexKey(5, 5100, 5350), string(geometry.StateFrozen))) + + _, rangeErr := cat.FrozenIndexCoversRange(5, 5100, 5349) + require.Error(t, rangeErr, "the range predicate must surface the uniqueness violation") + require.Contains(t, rangeErr.Error(), "two frozen coverages") + + // The per-chunk convenience form resolves a chunk to its window and inherits + // the same assertion. + _, chunkErr := cat.FrozenIndexCovers(5100) + require.Error(t, chunkErr, "the per-chunk predicate inherits the uniqueness assertion") +} + func TestCommitIndexTerminalDemotesTxhashKeys(t *testing.T) { cat, _ := testCatalog(t) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go index 1972f4970..8ef4ef987 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go @@ -42,7 +42,7 @@ func eligibleDiscardOps(cfg Config, cat *catalog.Catalog, through uint32) ([]fun if perr != nil { return nil, perr } - covers, cerr := indexCovers(c, cat) + covers, cerr := cat.FrozenIndexCovers(c) if cerr != nil { return nil, cerr } @@ -76,7 +76,7 @@ func pendingArtifacts(c chunk.ID, cat *catalog.Catalog) (catalog.ArtifactSet, er return need, err } if txState != geometry.StateFrozen { - covers, cerr := indexCovers(c, cat) + covers, cerr := cat.FrozenIndexCovers(c) if cerr != nil { return need, cerr } @@ -87,16 +87,6 @@ func pendingArtifacts(c chunk.ID, cat *catalog.Catalog) (catalog.ArtifactSet, er return need, nil } -// indexCovers reports whether the durable .idx for chunk's window already hashes -// it — the frozen coverage's [Lo, Hi] contains c. -func indexCovers(c chunk.ID, cat *catalog.Catalog) (bool, error) { - fk, ok, err := cat.FrozenTxHashIndex(cat.TxHashIndexLayout().TxHashIndexID(c)) - if err != nil { - return false, err - } - return ok && fk.Lo <= c && c <= fk.Hi, nil -} - // eligiblePruneOps is the system's only file-deleter, key-driven, covering both // key families. It returns sweep closures (SweepTxHashIndexKey per index key, one // batched SweepChunkArtifacts for the chunk family). "Below the floor" is the diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go index 3f6787265..1e3f0679d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go @@ -46,7 +46,7 @@ func TestRunLifecycleTick_BoundaryFreezesFoldsDiscards(t *testing.T) { assert.Equal(t, geometry.StateFrozen, state, "chunk 0 %s frozen", kind) } // The window's index is terminal and covers chunk 0. - covered, err := indexCovers(0, cat) + covered, err := cat.FrozenIndexCovers(0) require.NoError(t, err) assert.True(t, covered, "the window index folded chunk 0 in") fk, ok, err := cat.FrozenTxHashIndex(cat.TxHashIndexLayout().TxHashIndexID(0)) @@ -100,7 +100,7 @@ func TestRunLifecycleTick_DiscardGatedOnIndexCoverage(t *testing.T) { // Now finalize the window's index so it covers chunk 0 (terminal needs chunk // 1's .bin too; build a non-terminal-but-covering frozen coverage [0,0]). freezeCoverage(t, cat, 0, 0, 0) - covered, err := indexCovers(0, cat) + covered, err := cat.FrozenIndexCovers(0) require.NoError(t, err) require.True(t, covered) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go index e3d6b8a07..04595aad5 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go @@ -150,19 +150,23 @@ func highestDurableChunk(cat *catalog.Catalog) (int64, error) { } } - // A frozen index coverage satisfies txhash even after the .bin was demoted. - covered, err := frozenCoverageContains(cat) - if err != nil { - return 0, err - } - highest := int64(-1) for c, k := range frozen { if !k.ledgers || !k.events { continue } - if !k.txhash && !covered(c) { - continue + // A frozen index coverage satisfies txhash even after the .bin was demoted. + // The shared catalog predicate asserts INV-2 (one frozen coverage per window) + // on every read, so watermark derivation, discard eligibility, and resolve + // can never disagree about the same snapshot. + if !k.txhash { + covered, err := cat.FrozenIndexCovers(c) + if err != nil { + return 0, err + } + if !covered { + continue + } } if id := int64(c); id > highest { highest = id @@ -171,29 +175,6 @@ func highestDurableChunk(cat *catalog.Catalog) (int64, error) { return highest, nil } -// frozenCoverageContains returns a predicate reporting whether a chunk falls in -// some frozen index coverage [Lo, Hi]; coverages are read once up front. -func frozenCoverageContains(cat *catalog.Catalog) (func(chunk.ID) bool, error) { - covs, err := cat.AllTxHashIndexKeys() - if err != nil { - return nil, err - } - var frozen []geometry.TxHashIndexCoverage - for _, cov := range covs { - if cov.State == geometry.StateFrozen { - frozen = append(frozen, cov) - } - } - return func(c chunk.ID) bool { - for _, cov := range frozen { - if cov.Lo <= c && c <= cov.Hi { - return true - } - } - return false - }, nil -} - // ChunkIDOfLedger maps a ledger to its chunk, signed so a sub-genesis ledger // yields -1 instead of panicking. func ChunkIDOfLedger(ledger uint32) int64 { From 049db0e8631eb5142553870e1c276eebf1125b3d Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 17:11:26 -0400 Subject: [PATCH 39/55] fullhistory: make geometry.Layout the only cold-path derivation (#36) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three cold ingesters each re-derived filepath.Join(root, BucketID, leaf) from bare per-type roots, while the freeze barrier and the sweeps resolved the same paths through geometry.Layout — two copies of one formula that must agree forever, or the freeze fsyncs/flips 'frozen' on a path the writer didn't write and the sweep orphans a file no key names. Now processChunk resolves each chunk's destination from Layout (LedgerPackPath / TxHashBinPath / new EventsBucketDir) and passes the resolved paths in ingest.ColdDirs; the ingester constructors take the resolved path directly and no longer re-derive it. Layout owns the formula alone. Also folds the second copy of the default tree: Config.ResolvePaths now builds its defaults from geometry.NewLayout's accessors (production and every package's test helpers spell the tree once), so a rename can't leave them disagreeing. Whole tree build+vet+ingest/backfill/geometry/root tests + non-short E2E green. Confirms tamirms 3514888137. --- .../internal/fullhistory/backfill/process.go | 6 +- .../fullhistory/backfill/process_test.go | 2 +- .../internal/fullhistory/config.go | 23 ++++--- .../internal/fullhistory/geometry/paths.go | 10 ++- .../internal/fullhistory/ingest/driver.go | 34 +++++----- .../internal/fullhistory/ingest/events.go | 11 ++- .../fullhistory/ingest/ingest_test.go | 67 ++++++++++--------- .../internal/fullhistory/ingest/ledgers.go | 23 +++---- .../internal/fullhistory/ingest/txhash.go | 15 ++--- 9 files changed, 101 insertions(+), 90 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go index b3f2473e3..135625e27 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go @@ -103,9 +103,9 @@ func processChunk(ctx context.Context, chunkID chunk.ID, artifacts catalog.Artif // one-write:create — materialize this chunk's cold artifacts from the resolved // source's raw ledger iterator. WriteColdChunk is source-blind. dirs := ingest.ColdDirs{ - Ledgers: layout.LedgersRoot(), - Txhash: layout.TxHashRawRoot(), - Events: layout.EventsRoot(), + LedgerPack: layout.LedgerPackPath(chunkID), + TxhashBin: layout.TxHashBinPath(chunkID), + EventsDir: layout.EventsBucketDir(chunkID), } raw := src.RawLedgers(ctx, ledgerbackend.BoundedRange(chunkID.FirstLedger(), chunkID.LastLedger())) if rerr := ingest.WriteColdChunk( diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/process_test.go b/cmd/stellar-rpc/internal/fullhistory/backfill/process_test.go index b1905a198..5c917c005 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/process_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/process_test.go @@ -455,7 +455,7 @@ func writeRealPack(t *testing.T, cat *catalog.Catalog, chunkID chunk.ID) { stream := &fullChunkStream{t: t, gen: zeroTxLCMBytes} raw := stream.RawLedgers(context.Background(), ledgerbackend.BoundedRange(chunkID.FirstLedger(), chunkID.LastLedger())) - dirs := ingest.ColdDirs{Ledgers: cat.Layout().LedgersRoot()} + dirs := ingest.ColdDirs{LedgerPack: cat.Layout().LedgerPackPath(chunkID)} require.NoError(t, ingest.WriteColdChunk( context.Background(), silentLogger(), chunkID, raw, dirs, ingest.NopSink{}, ingest.Config{Ledgers: true})) diff --git a/cmd/stellar-rpc/internal/fullhistory/config.go b/cmd/stellar-rpc/internal/fullhistory/config.go index d557f5a8c..3f30a5842 100644 --- a/cmd/stellar-rpc/internal/fullhistory/config.go +++ b/cmd/stellar-rpc/internal/fullhistory/config.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "os" - "path/filepath" "runtime" "github.com/pelletier/go-toml" @@ -204,23 +203,27 @@ type Paths struct { // ResolvePaths fills every storage path, defaulting under default_data_dir. // Relative overrides are kept relative (resolved against the caller's working -// dir); only the defaults are joined to the data dir. +// dir); only the defaults are joined to the data dir. The default tree is spelled +// ONCE, by geometry.NewLayout — production flows through here and every package's +// test helpers through NewLayout, so a rename to the tree can't leave the two +// disagreeing. func (cfg Config) ResolvePaths() Paths { dataDir := cfg.Service.DefaultDataDir - pick := func(override, def string) string { + def := geometry.NewLayout(dataDir) + pick := func(override, defPath string) string { if override != "" { return override } - return def + return defPath } return Paths{ DataDir: dataDir, - Catalog: pick(cfg.Storage.Catalog, filepath.Join(dataDir, "catalog", "rocksdb")), - Ledgers: pick(cfg.Storage.Ledgers, filepath.Join(dataDir, "ledgers")), - Events: pick(cfg.Storage.Events, filepath.Join(dataDir, "events")), - TxhashRaw: pick(cfg.Storage.TxhashRaw, filepath.Join(dataDir, "txhash", "raw")), - TxhashIndex: pick(cfg.Storage.TxhashIndex, filepath.Join(dataDir, "txhash", "index")), - HotStorage: pick(cfg.Storage.Hot, filepath.Join(dataDir, "hot")), + Catalog: pick(cfg.Storage.Catalog, def.CatalogPath()), + Ledgers: pick(cfg.Storage.Ledgers, def.LedgersRoot()), + Events: pick(cfg.Storage.Events, def.EventsRoot()), + TxhashRaw: pick(cfg.Storage.TxhashRaw, def.TxHashRawRoot()), + TxhashIndex: pick(cfg.Storage.TxhashIndex, def.TxHashIndexRoot()), + HotStorage: pick(cfg.Storage.Hot, def.HotRoot()), } } diff --git a/cmd/stellar-rpc/internal/fullhistory/geometry/paths.go b/cmd/stellar-rpc/internal/fullhistory/geometry/paths.go index 58eb6752b..4f3331dd6 100644 --- a/cmd/stellar-rpc/internal/fullhistory/geometry/paths.go +++ b/cmd/stellar-rpc/internal/fullhistory/geometry/paths.go @@ -82,10 +82,18 @@ func (l Layout) LedgerPackPath(c chunk.ID) string { return filepath.Join(l.ledgersRoot, c.BucketID(), ledger.PackName(c)) } +// EventsBucketDir is a chunk's events cold-segment directory — the bucket dir the +// three events files (pack, index-pack, index-hash) live under, and the single +// path the cold events ingester writes into. Sharing it with EventsPaths keeps +// the events tree's shape defined once. +func (l Layout) EventsBucketDir(c chunk.ID) string { + return filepath.Join(l.eventsRoot, c.BucketID()) +} + // EventsPaths are a chunk's three events cold-segment files. Leaves owned by // eventstore.*. func (l Layout) EventsPaths(c chunk.ID) []string { - dir := filepath.Join(l.eventsRoot, c.BucketID()) + dir := l.EventsBucketDir(c) return []string{ filepath.Join(dir, eventstore.EventsPackName(c)), filepath.Join(dir, eventstore.IndexPackName(c)), diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go index 417769d08..3f50cc908 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go @@ -101,36 +101,40 @@ func drain(ctx context.Context, ledgers iter.Seq2[[]byte, error], chunkID chunk. return nil } -// ColdDirs is the per-type output root for one chunk's cold artifacts. An empty -// field for an enabled type is a config error. +// ColdDirs holds ONE chunk's RESOLVED cold-artifact destinations, derived by the +// caller from geometry.Layout so the ingesters write exactly where the freeze +// barrier and the sweeps resolve — the path formula lives in Layout alone, never +// re-derived here. LedgerPack and TxhashBin are the chunk's full file paths; +// EventsDir is its events bucket dir. An empty field for an enabled type is a +// config error. type ColdDirs struct { - Ledgers string - Txhash string - Events string + LedgerPack string + TxhashBin string + EventsDir string } -// buildColdIngesters opens one ColdIngester per enabled type under its dirs field. +// buildColdIngesters opens one ColdIngester per enabled type at its resolved path. // Single definition site of the ctor table, order, and rollback. func buildColdIngesters(dirs ColdDirs, chunkID chunk.ID, sink MetricSink, cfg Config) ([]ColdIngester, error) { ctors := []struct { enabled bool dataType string - dir string + path string open func(string, chunk.ID, MetricSink) (ColdIngester, error) }{ - {cfg.Ledgers, dataTypeLedgers, dirs.Ledgers, NewLedgerColdIngester}, - {cfg.Txhash, dataTypeTxhash, dirs.Txhash, NewTxhashColdIngester}, - {cfg.Events, dataTypeEvents, dirs.Events, NewEventsColdIngester}, + {cfg.Ledgers, dataTypeLedgers, dirs.LedgerPack, NewLedgerColdIngester}, + {cfg.Txhash, dataTypeTxhash, dirs.TxhashBin, NewTxhashColdIngester}, + {cfg.Events, dataTypeEvents, dirs.EventsDir, NewEventsColdIngester}, } ings := make([]ColdIngester, 0, len(ctors)) for _, c := range ctors { if !c.enabled { continue } - if c.dir == "" { - return nil, closeColdAll(ings, fmt.Errorf("ingest: %s enabled but ColdDirs.%s is empty", c.dataType, c.dataType)) + if c.path == "" { + return nil, closeColdAll(ings, fmt.Errorf("ingest: %s enabled but its ColdDirs path is empty", c.dataType)) } - ing, err := c.open(c.dir, chunkID, sink) + ing, err := c.open(c.path, chunkID, sink) if err != nil { return nil, closeColdAll(ings, fmt.Errorf("open %s cold ingester: %w", c.dataType, err)) } @@ -139,8 +143,8 @@ func buildColdIngesters(dirs ColdDirs, chunkID chunk.ID, sink MetricSink, cfg Co return ings, nil } -// WriteColdChunk materializes ONE chunk's cold artifacts into the roots named by -// dirs, in a single pass, from the already-opened raw ledger iterator. It is +// WriteColdChunk materializes ONE chunk's cold artifacts at the resolved paths +// named by dirs, in a single pass, from the already-opened raw ledger iterator. It is // SOURCE-BLIND: the caller (backfill) resolves the chunk's ledger source — the // local frozen .pack or the bulk backend — and hands its RawLedgers iterator here, // so the cold materializer never learns where the bytes came from and is faked in diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/events.go b/cmd/stellar-rpc/internal/fullhistory/ingest/events.go index bb51e890b..c39e5e4f0 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/events.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/events.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math" - "path/filepath" "time" "github.com/stellar/go-stellar-sdk/xdr" @@ -48,11 +47,11 @@ type eventsCold struct { failed bool } -// NewEventsColdIngester opens a per-chunk events.pack cold writer under coldDir -// and returns a ColdIngester that owns it. The writer uses its zero-value -// options; driver-level tuning is a follow-up via Config. -func NewEventsColdIngester(coldDir string, chunkID chunk.ID, sink MetricSink) (ColdIngester, error) { - bucketDir := filepath.Join(coldDir, chunkID.BucketID()) +// NewEventsColdIngester opens a per-chunk events.pack cold writer in bucketDir — +// the caller's geometry.Layout.EventsBucketDir(chunkID), so the write path is +// Layout's single derivation — and returns a ColdIngester that owns it. The +// writer uses its zero-value options; driver-level tuning is a follow-up via Config. +func NewEventsColdIngester(bucketDir string, chunkID chunk.ID, sink MetricSink) (ColdIngester, error) { w, err := eventstore.NewColdWriter(chunkID, bucketDir, eventstore.ColdWriterOptions{}) if err != nil { return nil, fmt.Errorf("eventstore.NewColdWriter: %w", err) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go index ea96b0f43..9136aaaf2 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go @@ -191,13 +191,14 @@ func packPath(ledgersRoot string, c chunk.ID) string { return filepath.Join(ledgersRoot, c.BucketID(), ledger.PackName(c)) } -// coldDirsAt derives the three per-type cold roots under one dir — the fixed -// layout the removed RunCold used, convenient for single-tmpdir tests. -func coldDirsAt(dir string) ColdDirs { +// coldDirsAt resolves chunk c's three cold-artifact paths under one dir's per-type +// roots — mirroring what geometry.Layout derives in production, so the readback +// helpers (packPath/txhashBinPath) find what the ingesters wrote. +func coldDirsAt(dir string, c chunk.ID) ColdDirs { return ColdDirs{ - Ledgers: filepath.Join(dir, dataTypeLedgers), - Txhash: filepath.Join(dir, dataTypeTxhash), - Events: filepath.Join(dir, dataTypeEvents), + LedgerPack: packPath(filepath.Join(dir, dataTypeLedgers), c), + TxhashBin: txhashBinPath(filepath.Join(dir, dataTypeTxhash)), + EventsDir: filepath.Join(dir, dataTypeEvents, c.BucketID()), } } @@ -444,7 +445,7 @@ func TestLedgerColdIngester_Readback(t *testing.T) { raw := marshalLCM(t, seq) coldDir := t.TempDir() - ing, err := NewLedgerColdIngester(coldDir, chunkID, nil) + ing, err := NewLedgerColdIngester(packPath(coldDir, chunkID), chunkID, nil) require.NoError(t, err) defer func() { require.NoError(t, ing.Close()) }() @@ -473,7 +474,7 @@ func TestTxhashColdIngester_Bin(t *testing.T) { first := chunkID.FirstLedger() coldDir := t.TempDir() - ing, err := NewTxhashColdIngester(coldDir, chunkID, nil) + ing, err := NewTxhashColdIngester(txhashBinPath(coldDir), chunkID, nil) require.NoError(t, err) defer func() { require.NoError(t, ing.Close()) }() @@ -495,7 +496,7 @@ func TestEventsColdIngester_Readback(t *testing.T) { first := chunkID.FirstLedger() coldDir := t.TempDir() - ing, err := NewEventsColdIngester(coldDir, chunkID, nil) + ing, err := NewEventsColdIngester(filepath.Join(coldDir, chunkID.BucketID()), chunkID, nil) require.NoError(t, err) defer func() { require.NoError(t, ing.Close()) }() @@ -531,7 +532,7 @@ func TestEventsColdIngester_V0KeepsOffsetsContiguous(t *testing.T) { first := chunkID.FirstLedger() coldDir := t.TempDir() - ing, err := NewEventsColdIngester(coldDir, chunkID, nil) + ing, err := NewEventsColdIngester(filepath.Join(coldDir, chunkID.BucketID()), chunkID, nil) require.NoError(t, err) defer func() { require.NoError(t, ing.Close()) }() @@ -590,7 +591,7 @@ func TestWriteColdChunk_EventlessChunk_FullyReadable(t *testing.T) { // Every ledger in the chunk is a V0 (pre-Soroban) ledger → zero events. require.NoError(t, WriteColdChunk( context.Background(), logger, chunkID, rawChunk(fullStream(t, chunkID, marshalV0LCM), chunkID), - coldDirsAt(coldDir), sink, Config{Events: true}, + coldDirsAt(coldDir, chunkID), sink, Config{Events: true}, )) bucketDir := filepath.Join(coldDir, dataTypeEvents, chunkID.BucketID()) @@ -632,7 +633,7 @@ func TestColdService_Success(t *testing.T) { coldDir := t.TempDir() sink := &testSink{} - ings, err := buildColdIngesters(coldDirsAt(coldDir), chunkID, sink, Config{Ledgers: true, Txhash: true, Events: true}) + ings, err := buildColdIngesters(coldDirsAt(coldDir, chunkID), chunkID, sink, Config{Ledgers: true, Txhash: true, Events: true}) require.NoError(t, err) service := NewColdService(ings, sink) defer func() { require.NoError(t, service.Close()) }() @@ -729,9 +730,9 @@ func TestColdService_FailurePath_NoArtifact(t *testing.T) { coldDir := t.TempDir() sink := &testSink{} - realLedger, err := NewLedgerColdIngester(filepath.Join(coldDir, dataTypeLedgers), chunkID, sink) + realLedger, err := NewLedgerColdIngester(packPath(filepath.Join(coldDir, dataTypeLedgers), chunkID), chunkID, sink) require.NoError(t, err) - realEvents, err := NewEventsColdIngester(filepath.Join(coldDir, dataTypeEvents), chunkID, sink) + realEvents, err := NewEventsColdIngester(filepath.Join(coldDir, dataTypeEvents, chunkID.BucketID()), chunkID, sink) require.NoError(t, err) failing := &failingCold{} service := NewColdService([]ColdIngester{realLedger, realEvents, failing}, sink) @@ -772,7 +773,7 @@ func TestColdIngester_Failure_RecordsErrorMetric(t *testing.T) { coldDir := t.TempDir() sink := &testSink{} - realLedger, err := NewLedgerColdIngester(filepath.Join(coldDir, dataTypeLedgers), chunkID, sink) + realLedger, err := NewLedgerColdIngester(packPath(filepath.Join(coldDir, dataTypeLedgers), chunkID), chunkID, sink) require.NoError(t, err) service := NewColdService([]ColdIngester{realLedger}, sink) @@ -827,7 +828,7 @@ func TestWriteColdChunk_RoundTrip(t *testing.T) { sink := &testSink{} require.NoError(t, WriteColdChunk( - context.Background(), logger, chunkID, rawChunk(stream, chunkID), coldDirsAt(coldDir), sink, Config{Ledgers: true}, + context.Background(), logger, chunkID, rawChunk(stream, chunkID), coldDirsAt(coldDir, chunkID), sink, Config{Ledgers: true}, )) path := packPath(filepath.Join(coldDir, "ledgers"), chunkID) @@ -856,7 +857,7 @@ func TestWriteColdChunk_ShortStream_NoArtifact(t *testing.T) { short := &fakeStream{t: t, count: 3} err := WriteColdChunk( - context.Background(), logger, chunkID, rawChunk(short, chunkID), coldDirsAt(coldDir), nil, Config{Ledgers: true}, + context.Background(), logger, chunkID, rawChunk(short, chunkID), coldDirsAt(coldDir, chunkID), nil, Config{Ledgers: true}, ) require.Error(t, err) require.Contains(t, err.Error(), "ended at") @@ -885,7 +886,7 @@ func TestWriteColdChunk_TxhashCold_Bin(t *testing.T) { require.NoError(t, WriteColdChunk( context.Background(), logger, chunkID, rawChunk(fullStream(t, chunkID, gen), chunkID), - coldDirsAt(coldDir), nil, Config{Txhash: true}, + coldDirsAt(coldDir, chunkID), nil, Config{Txhash: true}, )) entries, err := txhash.ReadColdBin(txhashBinPath(filepath.Join(coldDir, dataTypeTxhash))) @@ -914,7 +915,7 @@ func TestWriteColdChunk_EventsCold_Readback(t *testing.T) { require.NoError(t, WriteColdChunk( context.Background(), logger, chunkID, rawChunk(fullStream(t, chunkID, gen), chunkID), - coldDirsAt(coldDir), nil, Config{Events: true}, + coldDirsAt(coldDir, chunkID), nil, Config{Events: true}, )) bucketDir := filepath.Join(coldDir, "events", chunkID.BucketID()) @@ -955,7 +956,7 @@ func TestWriteColdChunk_OutOfOrderSeq_NoArtifact(t *testing.T) { stream := &seqStream{t: t, seqs: seqs} err := WriteColdChunk( - context.Background(), logger, chunkID, rawChunk(stream, chunkID), coldDirsAt(coldDir), nil, Config{Ledgers: true}, + context.Background(), logger, chunkID, rawChunk(stream, chunkID), coldDirsAt(coldDir, chunkID), nil, Config{Ledgers: true}, ) require.Error(t, err) require.Contains(t, err.Error(), "yielded ledger") @@ -988,7 +989,7 @@ func TestDrain_TxhashSeqGuard(t *testing.T) { err := WriteColdChunk( context.Background(), logger, chunkID, rawChunk(&seqStream{t: t, seqs: seqs}, chunkID), - coldDirsAt(coldDir), nil, Config{Txhash: true}, + coldDirsAt(coldDir, chunkID), nil, Config{Txhash: true}, ) require.Error(t, err) require.Contains(t, err.Error(), "yielded ledger") @@ -1014,7 +1015,7 @@ func TestWriteColdChunk_DrainStreamError_NoArtifact(t *testing.T) { stream := &errAtSeqStream{t: t, errAtSeq: failAt, err: wantErr} err := WriteColdChunk( - context.Background(), logger, chunkID, rawChunk(stream, chunkID), coldDirsAt(coldDir), nil, Config{Ledgers: true}, + context.Background(), logger, chunkID, rawChunk(stream, chunkID), coldDirsAt(coldDir, chunkID), nil, Config{Ledgers: true}, ) require.Error(t, err) require.ErrorIs(t, err, wantErr, "the backend error must propagate") @@ -1047,7 +1048,7 @@ func TestTxhashColdIngester_BinContent(t *testing.T) { first := chunkID.FirstLedger() coldDir := t.TempDir() - ing, err := NewTxhashColdIngester(coldDir, chunkID, nil) + ing, err := NewTxhashColdIngester(txhashBinPath(coldDir), chunkID, nil) require.NoError(t, err) defer func() { require.NoError(t, ing.Close()) }() @@ -1096,7 +1097,7 @@ func TestWriteColdChunk_CanceledContext(t *testing.T) { cancel() rerr := WriteColdChunk( ctx, logger, chunkID, rawChunk(fullStream(t, chunkID, nil), chunkID), - coldDirsAt(coldDir), sink, Config{Ledgers: true}, + coldDirsAt(coldDir, chunkID), sink, Config{Ledgers: true}, ) require.ErrorIs(t, rerr, context.Canceled) require.Equal(t, 1, sink.coldChunkTotals, "a canceled chunk attempt still emits one ColdChunkTotal") @@ -1112,7 +1113,7 @@ func TestWriteColdChunk_ConfigGuards(t *testing.T) { chunkID := chunk.ID(0) err := WriteColdChunk(context.Background(), logger, chunkID, - rawChunk(fullStream(t, chunkID, nil), chunkID), coldDirsAt(t.TempDir()), nil, Config{}) + rawChunk(fullStream(t, chunkID, nil), chunkID), coldDirsAt(t.TempDir(), chunkID), nil, Config{}) require.Error(t, err) require.Contains(t, err.Error(), "enables no data types") } @@ -1148,7 +1149,7 @@ func TestBuildColdIngesters_RollbackOneBuilt(t *testing.T) { // fails its bucket-dir MkdirAll. require.NoError(t, os.WriteFile(filepath.Join(coldDir, dataTypeTxhash), []byte("not a dir"), 0o644)) - _, err := buildColdIngesters(coldDirsAt(coldDir), chunkID, sink, Config{Ledgers: true, Txhash: true}) + _, err := buildColdIngesters(coldDirsAt(coldDir, chunkID), chunkID, sink, Config{Ledgers: true, Txhash: true}) require.Error(t, err, "txhash constructor must fail on the planted file") // The ledger ingester was built then rolled back with no Ingest/Finalize, so @@ -1171,7 +1172,7 @@ func TestBuildColdIngesters_RollbackTwoBuilt(t *testing.T) { packPath := filepath.Join(coldDir, dataTypeEvents, chunkID.BucketID(), eventstore.EventsPackName(chunkID)) require.NoError(t, os.MkdirAll(packPath, 0o755)) - _, err := buildColdIngesters(coldDirsAt(coldDir), chunkID, sink, + _, err := buildColdIngesters(coldDirsAt(coldDir, chunkID), chunkID, sink, Config{Ledgers: true, Txhash: true, Events: true}) require.Error(t, err, "events constructor must fail on the planted directory") @@ -1195,7 +1196,7 @@ func TestWriteColdChunk_ConstructorFailure_EmitsAggregate(t *testing.T) { err := WriteColdChunk( context.Background(), logger, chunkID, rawChunk(fullStream(t, chunkID, nil), chunkID), - coldDirsAt(coldDir), sink, Config{Ledgers: true}, + coldDirsAt(coldDir, chunkID), sink, Config{Ledgers: true}, ) require.Error(t, err) require.Equal(t, 1, sink.coldChunkTotals, @@ -1216,7 +1217,7 @@ func TestEventsCold_FinishThenIndexFails_LeavesInertPack(t *testing.T) { first := chunkID.FirstLedger() coldDir := t.TempDir() - ing, err := NewEventsColdIngester(coldDir, chunkID, nil) + ing, err := NewEventsColdIngester(filepath.Join(coldDir, chunkID.BucketID()), chunkID, nil) require.NoError(t, err) // Ingest one event-bearing ledger so the mirror is non-empty (an empty @@ -1253,7 +1254,7 @@ func TestEventsCold_FinalizeAfterFailedIngest_Refuses(t *testing.T) { chunkID := chunk.ID(0) coldDir := t.TempDir() - ing, err := NewEventsColdIngester(coldDir, chunkID, nil) + ing, err := NewEventsColdIngester(filepath.Join(coldDir, chunkID.BucketID()), chunkID, nil) require.NoError(t, err) defer func() { require.NoError(t, ing.Close()) }() @@ -1374,7 +1375,7 @@ func TestWriteColdChunk_LazySourceFirstReadError(t *testing.T) { wantErr := errors.New("induced lazy-source failure (bad config / missing object)") err := WriteColdChunk( context.Background(), logger, chunkID, rawChunk(lazyErrStream{err: wantErr}, chunkID), - coldDirsAt(coldDir), sink, Config{Ledgers: true}, + coldDirsAt(coldDir, chunkID), sink, Config{Ledgers: true}, ) require.Error(t, err) require.ErrorIs(t, err, wantErr) @@ -1398,7 +1399,7 @@ func TestWriteColdChunk_EmptyStream(t *testing.T) { err := WriteColdChunk( context.Background(), logger, chunkID, rawChunk(&fakeStream{t: t, count: 0}, chunkID), - coldDirsAt(coldDir), sink, Config{Ledgers: true}, + coldDirsAt(coldDir, chunkID), sink, Config{Ledgers: true}, ) require.Error(t, err) require.Contains(t, err.Error(), "ended at", "the completeness check rejects the empty stream") @@ -1420,7 +1421,7 @@ func TestColdService_FinalizeAbort_KeepsEarlierArtifact(t *testing.T) { coldDir := t.TempDir() sink := &testSink{} - realLedger, err := NewLedgerColdIngester(filepath.Join(coldDir, dataTypeLedgers), chunkID, sink) + realLedger, err := NewLedgerColdIngester(packPath(filepath.Join(coldDir, dataTypeLedgers), chunkID), chunkID, sink) require.NoError(t, err) failErr := errors.New("induced finalize failure") failing := &finalizeErrCold{err: failErr} diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go index 8c1cefe50..b303efe6e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go @@ -25,22 +25,19 @@ type ledgerCold struct { appended bool } -// NewLedgerColdIngester opens a per-chunk cold ledger writer under coldDir and -// returns a ColdIngester that owns it. The writer uses its zero-value options; -// driver-level tuning is a follow-up via Config. -func NewLedgerColdIngester(coldDir string, chunkID chunk.ID, sink MetricSink) (ColdIngester, error) { - // The chunk's pack lives under its %05d bucket subdirectory; ledger.PackName - // owns the per-chunk filename so the naming convention has a single owner - // shared with the cold-ledger read path (ledger.NewPackStream). - path := filepath.Join(coldDir, chunkID.BucketID(), ledger.PackName(chunkID)) - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return nil, fmt.Errorf("mkdir %s: %w", filepath.Dir(path), err) +// NewLedgerColdIngester opens a per-chunk cold ledger writer at packPath — the +// caller's geometry.Layout.LedgerPackPath(chunkID), so the write path is Layout's +// single derivation, not a second copy — and returns a ColdIngester that owns it. +// The writer uses its zero-value options; driver-level tuning is a follow-up via Config. +func NewLedgerColdIngester(packPath string, chunkID chunk.ID, sink MetricSink) (ColdIngester, error) { + if err := os.MkdirAll(filepath.Dir(packPath), 0o755); err != nil { + return nil, fmt.Errorf("mkdir %s: %w", filepath.Dir(packPath), err) } - w, err := ledger.NewColdWriter(path, chunkID.FirstLedger(), ledger.ColdWriterOptions{}) + w, err := ledger.NewColdWriter(packPath, chunkID.FirstLedger(), ledger.ColdWriterOptions{}) if err != nil { - return nil, fmt.Errorf("ledger.NewColdWriter %s: %w", path, err) + return nil, fmt.Errorf("ledger.NewColdWriter %s: %w", packPath, err) } - return &ledgerCold{path: path, writer: w, metrics: newColdMetrics(sink, dataTypeLedgers)}, nil + return &ledgerCold{path: packPath, writer: w, metrics: newColdMetrics(sink, dataTypeLedgers)}, nil } func (c *ledgerCold) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go b/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go index 6fcb42ba0..dd9ce305f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go @@ -33,19 +33,18 @@ type txhashCold struct { } // NewTxhashColdIngester returns a ColdIngester that accumulates a per-chunk -// sorted .bin under coldDir's bucket subdirectory, written at Finalize -// (overwriting any prior attempt's file — see the package doc's artifact -// model). -func NewTxhashColdIngester(coldDir string, chunkID chunk.ID, sink MetricSink) (ColdIngester, error) { - bucketDir := filepath.Join(coldDir, chunkID.BucketID()) - if err := os.MkdirAll(bucketDir, 0o755); err != nil { - return nil, fmt.Errorf("mkdir %s: %w", bucketDir, err) +// sorted .bin at binPath — the caller's geometry.Layout.TxHashBinPath(chunkID), +// so the write path is Layout's single derivation — written at Finalize +// (overwriting any prior attempt's file — see the package doc's artifact model). +func NewTxhashColdIngester(binPath string, chunkID chunk.ID, sink MetricSink) (ColdIngester, error) { + if err := os.MkdirAll(filepath.Dir(binPath), 0o755); err != nil { + return nil, fmt.Errorf("mkdir %s: %w", filepath.Dir(binPath), err) } // The initial cap (64Ki entries, ~1.3 MB) deliberately starts well below a // typical pubnet chunk's tx count (~3M): empty/sparse chunks stay cheap, // and a busy chunk just pays a few amortized growths. return &txhashCold{ - binPath: filepath.Join(bucketDir, txhash.ColdBinName(chunkID)), + binPath: binPath, chunkID: chunkID, entries: make([]txhash.ColdEntry, 0, 1<<16), metrics: newColdMetrics(sink, dataTypeTxhash), From f981593d87c4e858924c3a23047cec500d9901ff Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 17:20:54 -0400 Subject: [PATCH 40/55] fullhistory/ingest: batch-scope hot metric attribution (#18 part 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HotService.emit reported the same whole-batch duration AND error against all three data types: summing per-type hot durations over-counted wall-clock 3x, and one commit/extraction failure incremented the ledgers, txhash, and events error counters alike. After #18 part 1 there is no honest per-type split anyway — one atomic synced batch (one fsync) commits all CFs, and one shared ExtractLedgerEvents walk feeds both txhash and events. So attribution moves to the batch: HotLedgerTotal(d, err) is the one per-ledger signal (wall-clock + commit outcome), and per-type HotItems reports VOLUME only, on the success path. Drops HotIngest, the emit helper, and itemsOnSuccess. The PrometheusSink loses the per-type hot duration histogram and per-type hot error counter, gains hot_commit_errors_total (batch-scoped), and keeps the per-type hot items counter. Confirms Simon's Option A. ingest tests + non-short E2E green. --- .../fullhistory/ingest/ingest_test.go | 22 ++--- .../internal/fullhistory/ingest/metrics.go | 97 +++++++++++-------- .../internal/fullhistory/ingest/service.go | 34 +++---- 3 files changed, 77 insertions(+), 76 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go index 9136aaaf2..8b525dc64 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go @@ -35,10 +35,9 @@ const testPassphrase = "Public Global Stellar Network ; September 2015" // ───────────────────────── test metric sink ───────────────────────── -type hotCall struct { +type hotItemCall struct { dataType string items int - err error } type coldCall struct { @@ -55,20 +54,20 @@ type stageCall struct { } // testSink records every MetricSink call for assertions. Safe for concurrent -// use (HotIngest fires from the per-ledger fan-out goroutines). +// use (the hot methods fire from the per-ledger ingestion goroutine). type testSink struct { mu sync.Mutex - hotIngests []hotCall + hotItems []hotItemCall coldIngests []coldCall stages []stageCall hotLedgerTotals int coldChunkTotals int } -func (s *testSink) HotIngest(dataType string, _ time.Duration, items int, err error) { +func (s *testSink) HotItems(dataType string, items int) { s.mu.Lock() defer s.mu.Unlock() - s.hotIngests = append(s.hotIngests, hotCall{dataType, items, err}) + s.hotItems = append(s.hotItems, hotItemCall{dataType, items}) } func (s *testSink) ColdIngest(dataType string, _ time.Duration, items int, err error) { @@ -77,7 +76,7 @@ func (s *testSink) ColdIngest(dataType string, _ time.Duration, items int, err e s.coldIngests = append(s.coldIngests, coldCall{dataType, items, err}) } -func (s *testSink) HotLedgerTotal(time.Duration) { +func (s *testSink) HotLedgerTotal(_ time.Duration, _ error) { s.mu.Lock() defer s.mu.Unlock() s.hotLedgerTotals++ @@ -110,7 +109,7 @@ func (s *testSink) hotDataTypes() map[string]int { s.mu.Lock() defer s.mu.Unlock() m := map[string]int{} - for _, c := range s.hotIngests { + for _, c := range s.hotItems { m[c.dataType]++ } return m @@ -802,10 +801,11 @@ func TestPrometheusSink_Smoke(t *testing.T) { reg := prometheus.NewRegistry() require.NotPanics(t, func() { sink := NewPrometheusSink(reg, "test") - sink.HotIngest(dataTypeLedgers, time.Millisecond, 1, nil) - sink.HotIngest(dataTypeEvents, time.Millisecond, 3, errFailingCold) + sink.HotItems(dataTypeLedgers, 1) + sink.HotItems(dataTypeEvents, 3) sink.ColdIngest(dataTypeTxhash, time.Second, 100, nil) - sink.HotLedgerTotal(time.Millisecond) + sink.HotLedgerTotal(time.Millisecond, nil) + sink.HotLedgerTotal(time.Millisecond, errFailingCold) // exercise the commit-error counter sink.ColdChunkTotal(time.Second) sink.IngestStage(dataTypeEvents, tierHot, stageExtract, time.Millisecond, 3) sink.IngestStage(dataTypeEvents, tierCold, stageFinalize, time.Second, 0) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go index b49d8cbe1..941dce20d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go @@ -38,30 +38,35 @@ const ( // a CSV recorder in benchmarks, or a test recorder — interchangeably. // // Implementations must be safe for concurrent use across ALL methods: the live -// hot ingestion loop reports HotIngest/HotLedgerTotal from its own goroutine +// hot ingestion loop reports HotItems/HotLedgerTotal from its own goroutine // while the lifecycle may freeze several chunks concurrently (each its own // WriteColdChunk), so the cold methods (ColdIngest, ColdChunkTotal) can likewise // be called from several goroutines at once. type MetricSink interface { - // HotIngest reports one hot ingester's per-ledger Ingest: dataType is the - // data-type label, d the wall-clock, items the number of items written - // (events, txhashes, or 1 for a ledger), err the Ingest error (nil on - // success). - HotIngest(dataType string, d time.Duration, items int, err error) + // HotItems reports the per-type volume of one HotService.Ingest: how many items + // (events, txhashes, or 1 ledger) that type contributed to the ledger's atomic + // batch. Emitted on the success path only — a failed atomic batch wrote nothing + // durably. There is deliberately no per-type hot DURATION or ERROR: the whole + // ledger commits as ONE synced batch (one fsync), and post-#18 a single shared + // ExtractLedgerEvents walk feeds both txhash and events, so neither timing nor + // an extraction failure is attributable per type — both live on HotLedgerTotal. + HotItems(dataType string, items int) // ColdIngest reports one cold ingester's per-chunk total: the summed Ingest // wall-clock plus its Finalize, items the total items written for the chunk, // err the first error (nil on success). ColdIngest(dataType string, d time.Duration, items int, err error) - // HotLedgerTotal reports the per-ledger wall-clock of one HotService.Ingest - // (the single atomic synced WriteBatch across all CFs). - HotLedgerTotal(d time.Duration) + // HotLedgerTotal is the ONE batch-level signal per hot ledger: d is the + // wall-clock of the single atomic synced WriteBatch across all CFs, err its + // commit outcome (nil on success). It carries the hot tier's only honest + // per-ledger duration and error — attribution is batch-scoped, not per type. + HotLedgerTotal(d time.Duration, err error) // ColdChunkTotal reports the per-chunk wall-clock across all cold ingesters' // ingests plus their Finalizes (the ColdService lifetime). ColdChunkTotal(d time.Duration) // IngestStage reports one ingester's per-stage wall-clock INSIDE an // Ingest/Finalize call: stage is one of the stage* constants (extract, // term_index, write, finalize), tier "hot" or "cold", items the stage's - // natural item count (0 where none applies). The whole-call HotIngest / + // natural item count (0 where none applies). The whole-call HotLedgerTotal / // ColdIngest signals above cannot be decomposed by a sink after the // fact, so the per-stage granularity the bench reports need is exposed // as its own signal — a sink that doesn't want it (production @@ -73,9 +78,9 @@ type MetricSink interface { // caller passes a nil sink to a service or ingester. type NopSink struct{} -func (NopSink) HotIngest(string, time.Duration, int, error) {} +func (NopSink) HotItems(string, int) {} func (NopSink) ColdIngest(string, time.Duration, int, error) {} -func (NopSink) HotLedgerTotal(time.Duration) {} +func (NopSink) HotLedgerTotal(time.Duration, error) {} func (NopSink) ColdChunkTotal(time.Duration) {} func (NopSink) IngestStage(string, string, string, time.Duration, int) {} @@ -196,18 +201,23 @@ func (c ingestCollectors) observe(d time.Duration, items int, err error) { // passing it into the ingest drivers) is a follow-up — there is no full-history // ingest daemon startup path yet. This type only provides the registerable sink. type PrometheusSink struct { - // Pre-resolved per-ingester children, keyed by data type, one map per - // tier (the duration histograms have per-tier buckets). Every producer - // draws its data_type/stage from the same unexported constant sets these - // maps are built from, so a lookup can never miss — the maps are indexed - // directly, with no on-the-fly vector fallback. - hot map[string]ingestCollectors + // Per-type hot volume counters (HotItems), keyed by data type. The hot tier has + // no per-type duration or error — one atomic batch, one shared extraction walk — + // so unlike cold it needs only an item counter per type. + hotItems map[string]prometheus.Counter + // hotCommitErrors counts failed hot batch commits (HotLedgerTotal's err); it is + // batch-scoped (not per data type) because one fsync commits all CFs together. + hotCommitErrors prometheus.Counter + // Pre-resolved per-cold-ingester children, keyed by data type (duration + // histogram, items counter, errors counter). Producers draw their data_type + // from the same unexported constant set the map is built from, so a lookup can + // never miss — indexed directly, with no on-the-fly vector fallback. cold map[string]ingestCollectors // Per-stage durations (IngestStage), pre-resolved per // (data_type, stage) with per-tier buckets, keyed "dataType/stage". hotStage map[string]prometheus.Observer coldStage map[string]prometheus.Observer - // Aggregate per-tier wall-clock: hot per-ledger Ingest, cold per-chunk + // Aggregate per-tier wall-clock: hot per-ledger batch, cold per-chunk // service lifetime. Separate histograms so each tier gets fitting buckets. hotLedgerTotal prometheus.Observer coldChunkTotal prometheus.Observer @@ -217,13 +227,6 @@ type PrometheusSink struct { // registry under namespace + the fullhistory_ingest subsystem. namespace is the // daemon convention value (interfaces.PrometheusNamespace). func NewPrometheusSink(registry *prometheus.Registry, namespace string) *PrometheusSink { - hotDuration := prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, Subsystem: metricsSubsystem, - Name: "hot_ingest_duration_seconds", - Help: "per-ingester hot Ingest wall-clock (per ledger)", - Buckets: hotBuckets, - }, []string{"data_type"}) - coldDuration := prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: metricsSubsystem, Name: "cold_ingest_duration_seconds", @@ -250,6 +253,12 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh Buckets: hotBuckets, }) + hotCommitErrors := prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, Subsystem: metricsSubsystem, + Name: "hot_commit_errors_total", + Help: "failed hot batch commits (batch-scoped: one fsync commits all CFs)", + }) + coldChunkTotal := prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: metricsSubsystem, Name: "cold_chunk_duration_seconds", @@ -272,19 +281,15 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh Buckets: coldStageBuckets, }, []string{"data_type", "stage"}) - registry.MustRegister(hotDuration, coldDuration, ingestItems, ingestErrors, - hotLedgerTotal, coldChunkTotal, hotStageVec, coldStageVec) + registry.MustRegister(coldDuration, ingestItems, ingestErrors, + hotLedgerTotal, hotCommitErrors, coldChunkTotal, hotStageVec, coldStageVec) - hot := make(map[string]ingestCollectors, 3) + hotItems := make(map[string]prometheus.Counter, 3) cold := make(map[string]ingestCollectors, 3) hotStage := make(map[string]prometheus.Observer, 3*len(ingestStages)) coldStage := make(map[string]prometheus.Observer, 3*len(ingestStages)) for _, dataType := range []string{dataTypeLedgers, dataTypeTxhash, dataTypeEvents} { - hot[dataType] = ingestCollectors{ - duration: hotDuration.WithLabelValues(dataType), - items: ingestItems.WithLabelValues(dataType, tierHot), - errors: ingestErrors.WithLabelValues(dataType, tierHot), - } + hotItems[dataType] = ingestItems.WithLabelValues(dataType, tierHot) cold[dataType] = ingestCollectors{ duration: coldDuration.WithLabelValues(dataType), items: ingestItems.WithLabelValues(dataType, tierCold), @@ -297,25 +302,31 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh } return &PrometheusSink{ - hot: hot, - cold: cold, - hotStage: hotStage, - coldStage: coldStage, - hotLedgerTotal: hotLedgerTotal, - coldChunkTotal: coldChunkTotal, + hotItems: hotItems, + hotCommitErrors: hotCommitErrors, + cold: cold, + hotStage: hotStage, + coldStage: coldStage, + hotLedgerTotal: hotLedgerTotal, + coldChunkTotal: coldChunkTotal, } } -func (p *PrometheusSink) HotIngest(dataType string, d time.Duration, items int, err error) { - p.hot[dataType].observe(d, items, err) +func (p *PrometheusSink) HotItems(dataType string, items int) { + if items > 0 { + p.hotItems[dataType].Add(float64(items)) + } } func (p *PrometheusSink) ColdIngest(dataType string, d time.Duration, items int, err error) { p.cold[dataType].observe(d, items, err) } -func (p *PrometheusSink) HotLedgerTotal(d time.Duration) { +func (p *PrometheusSink) HotLedgerTotal(d time.Duration, err error) { p.hotLedgerTotal.Observe(d.Seconds()) + if err != nil { + p.hotCommitErrors.Inc() + } } func (p *PrometheusSink) ColdChunkTotal(d time.Duration) { diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go index f3e06c9ee..d16531687 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go @@ -40,32 +40,22 @@ func NewHotService(db *hotchunk.DB, sink MetricSink) *HotService { } // Ingest commits lcm to the shared hot DB in one atomic synced WriteBatch -// (decision (a)). HotLedgerTotal is emitted regardless of success; on success, -// one HotIngest per hot data type reports its item count. +// (decision (a)) and emits the ledger's metrics. Attribution is batch-scoped, not +// per type: HotLedgerTotal carries the whole-batch wall-clock and the commit +// outcome (one fsync commits all CFs; post-#18 one shared ExtractLedgerEvents walk +// feeds both txhash and events, so neither timing nor an extraction failure is +// attributable per type). Per-type HotItems reports only VOLUME, on success — a +// failed atomic batch wrote nothing durably. func (s *HotService) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { start := time.Now() counts, err := s.db.IngestLedger(seq, lcm) - d := time.Since(start) - s.emit(counts, d, err) - s.sink.HotLedgerTotal(d) - return err -} - -// emit reports one HotIngest per hot data type. On error, counts are 0 with the -// error attached (a failed atomic commit wrote nothing durably). -func (s *HotService) emit(counts hotchunk.LedgerCounts, d time.Duration, err error) { - s.sink.HotIngest(dataTypeLedgers, d, itemsOnSuccess(counts.Ledgers, err), err) - s.sink.HotIngest(dataTypeTxhash, d, itemsOnSuccess(counts.Txhash, err), err) - s.sink.HotIngest(dataTypeEvents, d, itemsOnSuccess(counts.Events, err), err) -} - -// itemsOnSuccess returns n on success and 0 on error — a failed atomic batch -// commits nothing, so no items were written. -func itemsOnSuccess(n int, err error) int { - if err != nil { - return 0 + s.sink.HotLedgerTotal(time.Since(start), err) + if err == nil { + s.sink.HotItems(dataTypeLedgers, counts.Ledgers) + s.sink.HotItems(dataTypeTxhash, counts.Txhash) + s.sink.HotItems(dataTypeEvents, counts.Events) } - return n + return err } // ColdService drives a set of ColdIngesters for one chunk: sequential per-ledger From ec7697ede2d9dc32f72aa40ab819e8b30ea99e4b Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 17:38:07 -0400 Subject: [PATCH 41/55] fullhistory: errgroup.WithContext + ingestion owns its first chunk (#16 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-2 review on the #16 landing (tamirms): - errgroup.WithContext replaces the manual WithCancel + per-closure cancelLoops. WithContext records the returning goroutine's error BEFORE cancelling, so g.Wait surfaces the real cause instead of the sibling's induced context-canceled (the warn log no longer blames cancellation for a failure). - The ingestion loop returns an error on a clean stream end instead of nil: a nil return classified as clean shutdown and silently stopped ingestion while ctx was live. (Only reachable via fake/custom streams, but the safe default.) With the loop never returning nil-while-live, WithContext can't hang the lifecycle sibling on a clean ingestion stop. - The loop opens the resume chunk's hot DB ITSELF (open + deferred close adjacent in one function) instead of run() opening it and handing the handle across a call boundary. Deletes the nextIngestLedger startup assertion, whose two returns ran before the closing defer was registered and leaked the DB + its RocksDB LOCK on a transient read error (every restart then failed to reopen). The values were provably equal — a dead guard, like the #25 clamps. Tests keep an impliedResume helper for the restart-watermark assertion; seedWatermark now closes its handle; deletes the dead drainLifecycle helper. - Comment cleanup: supervise is clean-vs-restart (two-way), not three-way; drop the No-os.Exit/Fatalf tombstones and the two-writers/EMFILE counterfactuals. Whole tree build+vet, root -short, and non-short E2E green. --- .../internal/fullhistory/daemon.go | 13 +-- .../internal/fullhistory/hotloop.go | 59 +++++------ .../internal/fullhistory/hotloop_test.go | 97 +++++++++---------- .../fullhistory/lifecycle/lifecycle.go | 12 +-- .../internal/fullhistory/startup.go | 86 ++++++---------- 5 files changed, 108 insertions(+), 159 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index a3be66c88..42abb08a7 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -216,13 +216,10 @@ func buildSinks(opts daemonOptions, registry *prometheus.Registry) (observabilit return metrics, sink } -// supervise restarts run after a backoff on ANY non-clean return ("startup is the -// recovery path"): nil means a clean shutdown, a ctx cancel means a clean shutdown, -// everything else is warned and retried. Loss can't be distinguished from a -// transient inside the process (an unmounted volume looks identical to a destroyed -// one, and EMFILE / a lingering RocksDB LOCK are recoverable), so there is no -// fatal-and-exit class — genuine loss presents as a crash-loop with a clear warn -// line, the same page an operator would get from a one-shot exit. The +// supervise is the daemon's clean-vs-restart decision point ("startup is the +// recovery path"): nil or a ctx cancel is a clean shutdown, everything else is +// warned and retried after a backoff. There is deliberately no fatal-and-exit +// class — genuine loss presents as a crash-loop with a clear warn line. The // never-auto-heal guarantee lives in the must-exist open (openHotDBForChunk), not here. func supervise( ctx context.Context, start StartConfig, logger *supportlog.Entry, backoff time.Duration, @@ -243,7 +240,7 @@ func supervise( } // sleepCtx blocks for d or until ctx is canceled, returning ctx.Err() if canceled -// first and nil otherwise. supervise's three-way clean/fatal/restart loop can't be +// first and nil otherwise. supervise's clean-vs-restart loop can't be // a backoff.Retry, so it keeps a hand-rolled sleep — but shares this one helper // rather than re-rolling the timer/select (and its easy-to-forget timer.Stop). func sleepCtx(ctx context.Context, d time.Duration) error { diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index 3cb7cb9ee..def67a4e2 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -2,6 +2,7 @@ package fullhistory import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -88,12 +89,12 @@ type boundaryPublisher interface { Publish(c chunk.ID) } -// ingestionLoopConfig bundles the ingestion loop's dependencies (previously eight -// positional params). +// ingestionLoopConfig bundles the ingestion loop's dependencies. The loop opens +// the resume chunk's hot DB itself from Catalog + Resume, so there is no hot-DB +// handle to thread in (and no cross-call ownership gap to leak through). type ingestionLoopConfig struct { Stream ledgerbackend.LedgerStream Resume uint32 - HotDB *hotchunk.DB Catalog *catalog.Catalog Boundary boundaryPublisher Logger *supportlog.Entry @@ -124,23 +125,19 @@ type ingestionLoopConfig struct { // here in the producer by construction, not a lock the readers rely on. func runIngestionLoop(ctx context.Context, cfg ingestionLoopConfig) (err error) { metrics := observability.MetricsOrNop(cfg.Metrics) - hotDB := cfg.HotDB - - // Startup assertion: the resume passed in must equal what the live hot DB - // implies. run() derives resume (lastCommitted+1) and opens this DB; the two - // always agree, but only by case analysis — so assert it and fail loudly on a - // disagreement (a bug), rather than silently trusting one over the other. - if implied, ierr := nextIngestLedger(hotDB); ierr != nil { - return fmt.Errorf("derive resume assertion: %w", ierr) - } else if implied != cfg.Resume { - return fmt.Errorf("resume ledger %d disagrees with hot DB %s implied resume %d", - cfg.Resume, hotDB.ChunkID(), implied) - } - // The loop is hotDB's single writer and reopens it at every boundary. On any - // exit, close the live handle so the rocksdb instance does not leak (the - // boundary handoff already closed every prior chunk's DB); no writer races this + // Open the resume chunk's hot DB HERE, so the open and its deferred close are + // adjacent in one function — no cross-call ownership gap for a transient open + // failure to leak the handle (and its RocksDB LOCK) through. The loop trusts the + // resume point passed in (run() derived it from the same durable state); there is + // nothing to re-derive or assert. The loop is this DB's single writer and reopens + // it at every boundary; the defer closes whatever handle is live on any exit (the + // boundary handoff already closed every prior chunk's DB), and no writer races the // close (the loop has stopped on every exit path). + hotDB, err := openHotDBForChunk(cfg.Catalog, chunk.IDFromLedger(cfg.Resume), cfg.Logger) + if err != nil { + return fmt.Errorf("open resume hot tier for ledger %d: %w", cfg.Resume, err) + } defer func() { if hotDB != nil { if cerr := hotDB.Close(); cerr != nil && err == nil { @@ -201,23 +198,11 @@ func runIngestionLoop(ctx context.Context, cfg ingestionLoopConfig) (err error) WithField("last_ledger", vl.Seq). Info("streaming: ingestion chunk boundary — handed off to lifecycle") } - // The unbounded stream only ends on ctx cancellation or a source error, both - // surfaced as the cursor's error element above; a nil return here means the - // source stopped cleanly (no more ledgers, no error). - return nil -} - -// nextIngestLedger is the resume point a live hot DB implies: one past its -// authoritative last-committed ledger, or the bound chunk's first ledger on an -// empty DB. run() derives the same value independently (lastCommitted+1); -// runIngestionLoop asserts the two agree. -func nextIngestLedger(db *hotchunk.DB) (uint32, error) { - maxSeq, ok, err := db.MaxCommittedSeq() - if err != nil { - return 0, err - } - if !ok { - return db.ChunkID().FirstLedger(), nil - } - return maxSeq + 1, nil + // The unbounded production stream ends only on ctx cancellation or a source + // error, both surfaced as the cursor's error element above. Falling through here + // means the source stopped WITHOUT an error while the daemon ctx is still live — + // unexpected for captive core; surface it as a restartable error rather than a + // nil return, which supervise would read as a clean shutdown and silently stop + // ingesting. + return errors.New("ingestion stream ended unexpectedly (source stopped with no error)") } diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index 991eeca0d..7c3b69b0a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -114,23 +114,35 @@ func (r *recordingBoundary) list() []chunk.ID { return append([]chunk.ID(nil), r.ids...) } -// loopConfig builds an ingestionLoopConfig for a test: the stream + hot DB + a -// recording boundary, with Resume derived from the DB (the value the loop asserts). -func loopConfig(t *testing.T, stream ledgerbackend.LedgerStream, db *hotchunk.DB, cat *catalog.Catalog) (ingestionLoopConfig, *recordingBoundary) { - t.Helper() - resume, err := nextIngestLedger(db) - require.NoError(t, err) +// loopConfig builds an ingestionLoopConfig for a test: the stream + resume point + +// a recording boundary. The loop opens the resume chunk's hot DB itself, so no DB +// handle is passed — and the test must hold none on that dir while the loop runs (a +// second read-write open would contend the RocksDB LOCK). +func loopConfig(stream ledgerbackend.LedgerStream, cat *catalog.Catalog, resume uint32) (ingestionLoopConfig, *recordingBoundary) { rec := &recordingBoundary{} return ingestionLoopConfig{ Stream: stream, Resume: resume, - HotDB: db, Catalog: cat, Boundary: rec, Logger: silentLogger(), }, rec } +// impliedResume is the resume point a hot DB's durable watermark implies — one past +// its last committed ledger, or the chunk's first ledger when empty. Production no +// longer derives this in the loop (it trusts the resume run() passes it), but tests +// still assert that a restart's durable watermark matches what startup would derive. +func impliedResume(t *testing.T, db *hotchunk.DB) uint32 { + t.Helper() + maxSeq, ok, err := db.MaxCommittedSeq() + require.NoError(t, err) + if !ok { + return db.ChunkID().FirstLedger() + } + return maxSeq + 1 +} + // openLiveHotDB opens (and brackets ready) the live hot DB for a chunk via the // production opener, returning the handle and the catalog it lives under. func openLiveHotDB(t *testing.T, cat *catalog.Catalog, c chunk.ID) *hotchunk.DB { @@ -140,15 +152,14 @@ func openLiveHotDB(t *testing.T, cat *catalog.Catalog, c chunk.ID) *hotchunk.DB return db } -// seedWatermark advances a chunk's hot DB to a last-committed ledger of seq so -// the indexed poll resumes at seq+1, letting a boundary test drive the loop over -// only the last ledger or two of a chunk. It ingests a real zero-tx LCM for -// every ledger up to seq through the production IngestLedger path (the events -// CF requires strict ledger contiguity from the chunk's first ledger). The -// returned DB is the (re-opened, ready) live handle the loop then owns. Seeding -// a near-full chunk costs one synced commit per ledger, so its callers run +// seedWatermark commits real zero-tx LCMs for [FirstLedger, seq] into chunk c's +// hot DB through the production IngestLedger path (the events CF requires strict +// ledger contiguity from the chunk's first ledger), then CLOSES the handle — +// leaving the chunk "ready" on disk with NO open handle, so the loop can open it +// itself. Returns the resume point (seq+1) a boundary test drives the loop from. +// Seeding a near-full chunk costs one synced commit per ledger, so its callers run // t.Parallel(). -func seedWatermark(t *testing.T, cat *catalog.Catalog, c chunk.ID, seq uint32) *hotchunk.DB { +func seedWatermark(t *testing.T, cat *catalog.Catalog, c chunk.ID, seq uint32) uint32 { t.Helper() db := openLiveHotDB(t, cat, c) for s := c.FirstLedger(); s <= seq; s++ { @@ -156,23 +167,7 @@ func seedWatermark(t *testing.T, cat *catalog.Catalog, c chunk.ID, seq uint32) * require.NoError(t, err) } require.NoError(t, db.Close()) - reopened, err := openHotDBForChunk(cat, c, silentLogger()) - require.NoError(t, err) - return reopened -} - -// drainLifecycle counts how many chunk ids the buffered lifecycle channel -// delivered after the loop returned (the loop is done, so no send races this). -func drainLifecycle(ch chan chunk.ID) []chunk.ID { - var got []chunk.ID - for { - select { - case c := <-ch: - got = append(got, c) - default: - return got - } - } + return seq + 1 } // --------------------------------------------------------------------------- @@ -196,9 +191,7 @@ func TestOpenHotTier_CreatesBracketAndDir(t *testing.T) { _, statErr := os.Stat(cat.Layout().HotChunkPath(c)) require.NoError(t, statErr, "the dir exists") - resume, err := nextIngestLedger(db) - require.NoError(t, err) - assert.Equal(t, c.FirstLedger(), resume, "an empty resume DB resumes at the chunk's first ledger") + assert.Equal(t, c.FirstLedger(), impliedResume(t, db), "an empty resume DB resumes at the chunk's first ledger") } // TestOpenHotTier_ReadyButDirMissingFailsOpen: a "ready" key whose DB is gone @@ -242,13 +235,13 @@ func TestRunIngestionLoop_LedgerLandsAcrossAllCFs(t *testing.T) { cat, _ := testCatalog(t) c := chunk.ID(0) first := c.FirstLedger() - db := openLiveHotDB(t, cat, c) // A short contiguous prefix from the chunk's first ledger (events require - // strict contiguity from FirstLedger), then the stream runs dry and errs. + // strict contiguity from FirstLedger), then the stream runs dry and errs. The + // loop opens the empty chunk 0 itself and resumes at its first ledger. stream := streamForSeqs(t, first, first+2) stream.endErr = errors.New("backend crashed") - cfg, _ := loopConfig(t, stream, db, cat) + cfg, _ := loopConfig(stream, cat, first) err := runIngestionLoop(context.Background(), cfg) require.Error(t, err, "stream ran past the prefix and errored") @@ -281,13 +274,13 @@ func TestRunIngestionLoop_BoundaryNotifiesCompletedChunk(t *testing.T) { cat, _ := testCatalog(t) c := chunk.ID(0) c1 := c + 1 - db := seedWatermark(t, cat, c, c.LastLedger()-1) + resume := seedWatermark(t, cat, c, c.LastLedger()-1) // == c.LastLedger() stream := &fakeCoreStream{frames: map[uint32][]byte{ c.LastLedger(): zeroTxLCMBytes(t, c.LastLedger()), // boundary 0->1 c1.FirstLedger(): zeroTxLCMBytes(t, c1.FirstLedger()), // a ledger in chunk 1 }, endErr: errors.New("end")} - cfg, rec := loopConfig(t, stream, db, cat) + cfg, rec := loopConfig(stream, cat, resume) done := make(chan error, 1) go func() { @@ -316,11 +309,10 @@ func TestRunIngestionLoop_CtxCancelReturnsCtxErr(t *testing.T) { cat, _ := testCatalog(t) c := chunk.ID(0) first := c.FirstLedger() - db := openLiveHotDB(t, cat, c) stream := streamForSeqs(t, first, first+1) stream.blockOnCtx = true // after the frames, behave like a live tip stream - cfg, _ := loopConfig(t, stream, db, cat) + cfg, _ := loopConfig(stream, cat, first) ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) @@ -348,13 +340,12 @@ func TestRunIngestionLoop_StreamErrorReturnsError(t *testing.T) { cat, _ := testCatalog(t) c := chunk.ID(0) first := c.FirstLedger() - db := openLiveHotDB(t, cat, c) boom := errors.New("backend exploded") stream := streamForSeqs(t, first, first) stream.yieldErrAt = first + 1 stream.errAt = boom - cfg, _ := loopConfig(t, stream, db, cat) + cfg, _ := loopConfig(stream, cat, first) err := runIngestionLoop(context.Background(), cfg) require.Error(t, err) @@ -375,27 +366,27 @@ func TestRunIngestionLoop_RestartResumesFromWatermark(t *testing.T) { c := chunk.ID(0) first := c.FirstLedger() - // First run: commit [first, first+2], then the stream errs. - db1 := openLiveHotDB(t, cat, c) + // First run: the loop opens empty chunk 0 itself (resumes at first), commits + // [first, first+2], then the stream errs. stream1 := streamForSeqs(t, first, first+2) stream1.endErr = errors.New("end") - cfg1, _ := loopConfig(t, stream1, db1, cat) + cfg1, _ := loopConfig(stream1, cat, first) err := runIngestionLoop(context.Background(), cfg1) require.Error(t, err) assert.Equal(t, first, stream1.firstSeen.Load(), "first run resumed at the chunk's first ledger") - // Restart: re-open the live DB the way startup would. The resume point must - // be watermark+1. + // The durable watermark now implies resume first+3 — exactly what startup would + // derive on restart. Close the handle before the loop reopens the dir. db2, err := openHotDBForChunk(cat, c, silentLogger()) require.NoError(t, err) - resume, err := nextIngestLedger(db2) - require.NoError(t, err) + resume := impliedResume(t, db2) assert.Equal(t, first+3, resume, "restart resumes one past the durable watermark") + require.NoError(t, db2.Close()) - // Second run resumes at watermark+1 and commits two more ledgers. + // Second run resumes at the derived watermark and commits two more ledgers. stream2 := streamForSeqs(t, first+3, first+5) stream2.endErr = errors.New("end") - cfg2, _ := loopConfig(t, stream2, db2, cat) + cfg2, _ := loopConfig(stream2, cat, resume) err = runIngestionLoop(context.Background(), cfg2) require.Error(t, err) assert.Equal(t, first+3, stream2.firstSeen.Load(), "second run resumed at watermark+1") diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index cc5684858..05ad85ae4 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -63,9 +63,9 @@ func runOps(ctx context.Context, ops []func() error) error { // next tick's work). Plan range is [floor, lastChunk] (start raised to storage); // discard/prune key off through. // -// It returns the first stage error WITHOUT classifying it: Loop propagates it and -// supervise is the single fatal-vs-restart decision point (a canceled ctx surfaces -// as a ctx error supervise treats as a clean shutdown). No os.Exit, no Fatalf. +// It returns the first stage error WITHOUT classifying it: Loop propagates it to +// run's errgroup and supervise decides clean-vs-restart (a canceled ctx surfaces +// as a ctx error supervise treats as a clean shutdown). func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChunk chunk.ID) error { metrics := observability.MetricsOrNop(cfg.Metrics) logger := cfg.Logger @@ -190,9 +190,9 @@ func (s *BoundarySignal) take() (chunk.ID, bool) { // selects on ctx.Done() too, so it never blocks past shutdown. // // It returns the first tick error to its caller (run() joins it with ingestion in -// an errgroup, so supervise is the single fatal-vs-restart point). A ctx -// cancellation returns nil — cancellation is a shutdown, and the sibling goroutine -// (ingestion, or an already-returned failing tick) carries any real cause. +// an errgroup, so supervise decides clean-vs-restart). A ctx cancellation returns +// nil — cancellation is a shutdown, and the sibling goroutine (ingestion, or an +// already-returned failing tick) carries any real cause. func Loop(ctx context.Context, cfg Config, cat *catalog.Catalog, sig *BoundarySignal) error { for { select { diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index ad8d238f3..830f19a94 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -18,16 +18,15 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) -// run is the daemon's startup, in two steps: (1) BACKFILL to the -// tip, then (2) SERVE + INGEST — open the resume chunk's hot DB, start captive -// core (injected), begin serving reads (injected), then run the live ingestion -// loop and the lifecycle loop as a joined errgroup pair (whichever returns first -// tears down the other; g.Wait surfaces the first error). Returns nil only on a -// clean shutdown (ctx canceled mid-run, or the ingestion loop's clean stop); any -// other return is a restartable error the supervisor warns on and retries with -// backoff (a first start with no reachable backend, a backfill/ingest/lifecycle -// failure, or a "ready" hot DB that won't open — none are auto-healed, all are -// re-attempted). +// run is the daemon's startup, in two steps: (1) BACKFILL to the tip, then +// (2) SERVE + INGEST — start captive core (injected), begin serving reads +// (injected), then run the live ingestion loop (which opens the resume chunk's hot +// DB itself) and the lifecycle loop as a joined errgroup pair (whichever returns +// first cancels the other; g.Wait surfaces the first error). Returns nil only on a +// clean shutdown (ctx canceled mid-run); any other return is a restartable error +// the supervisor warns on and retries with backoff (a first start with no +// reachable backend, a backfill/ingest/lifecycle failure, or a "ready" hot DB that +// won't open — none are auto-healed, all are re-attempted). func run(ctx context.Context, cfg StartConfig) error { if err := cfg.validate(); err != nil { return err @@ -75,16 +74,11 @@ func run(ctx context.Context, cfg StartConfig) error { WithField("resume_chunk", chunk.IDFromLedger(lastCommitted+1).String()). Info("backfill complete — opening resume hot tier and ingesting") - // Step 2: serve + ingest. resumeLedger is one past the last-committed ledger — the live - // chunk's next un-committed ledger; runIngestionLoop re-derives the exact resume - // point from durable state, so a mid-chunk and a boundary last-committed ledger both resume right. + // Step 2: serve + ingest. resumeLedger is one past the last-committed ledger — + // the live chunk's next un-committed ledger. The ingestion loop opens that + // chunk's hot DB itself (open and deferred close in one function, no cross-call + // ownership gap) and consumes the stream from there. resumeLedger := lastCommitted + 1 - resumeChunk := chunk.IDFromLedger(resumeLedger) - - hotDB, err := openHotDBForChunk(cat, resumeChunk, logger) - if err != nil { - return fmt.Errorf("startup open resume hot tier chunk %s: %w", resumeChunk, err) - } // The live ingestion stream. It owns the captive-core process (started on the // loop's first pull, torn down when the loop exits), so there is no eager @@ -93,62 +87,45 @@ func run(ctx context.Context, cfg StartConfig) error { // first stream error for the daemon to classify (and restart). stream, err := cfg.Core.OpenCore(ctx) if err != nil { - _ = hotDB.Close() return fmt.Errorf("startup open ingestion stream: %w", err) } - // The lifecycle goroutine runs one tick per boundary signal. Ingestion Publishes - // the just-completed chunk id into a latest-cell (a slow lifecycle can't fall - // behind — no bounded buffer, no fatal). It shares NO in-memory state with - // ingestion — all derived from durable keys. + // The lifecycle goroutine runs one tick per boundary signal; ingestion Publishes + // the just-completed chunk id into a latest-cell. It shares NO in-memory state + // with ingestion — all derived from durable keys. boundary := lifecycle.NewBoundarySignal() // Seed the first tick with the last complete chunk at the resume point so it - // fires at once — clearing crash/downtime leftovers concurrently with serving. - // Skipped on a young network where no chunk is complete (the first real boundary - // triggers the first tick). + // fires at once. Skipped on a young network where no chunk is complete. if seed := geometry.LastCompleteChunkAt(lastCommitted); seed >= 0 { boundary.Publish(chunk.ID(seed)) //nolint:gosec // seed >= 0 } - // Assemble the lifecycle config from the SAME Exec wiring backfill uses, so the - // two share one catalog/pool by construction (Exec is already defaulted, so - // WithLifecycleDefaults' re-default is a no-op). + // The lifecycle config draws on the SAME Exec wiring backfill uses, so the two + // share one catalog/pool by construction. lifecycleCfg := lifecycle.Config{ ExecConfig: cfg.Exec, RetentionChunks: cfg.RetentionChunks, }.WithLifecycleDefaults() - // Begin serving reads (injected) BEFORE launching the loops. It must return - // promptly (launch, not block); a serve failure just closes hotDB and returns, - // with no running goroutines to tear down. + // Begin serving reads (injected) BEFORE launching the loops; it must return + // promptly (launch, not block). if err := cfg.ServeReads(ctx); err != nil { - _ = hotDB.Close() return fmt.Errorf("startup serve reads: %w", err) } - // Ingestion and the lifecycle run as a JOINED pair under one per-iteration child - // ctx: whichever returns first cancels the other, and g.Wait joins BOTH before - // run returns for ANY reason — restoring the single-lifecycle-goroutine invariant - // across supervisor restarts (a surviving loop would run a tick concurrently with - // the next iteration's lifecycle + ingestion: two backfill passes truncating the - // same .pack/.idx). runLifecycle returns its error up through the group, so - // supervise is the ONE fatal-vs-restart decision point — no os.Exit in the tick. - // A parent-ctx cancel makes ingestion return a ctx error the group surfaces; - // supervise classifies a canceled parent as a clean shutdown. Loop checks ctx at - // every step, so the join cannot block past the current step. - loopCtx, cancelLoops := context.WithCancel(ctx) - defer cancelLoops() - var g errgroup.Group + // Ingestion and the lifecycle run as a joined pair under errgroup.WithContext: + // gctx cancels as soon as EITHER returns — and WithContext records the returning + // goroutine's error BEFORE cancelling, so g.Wait surfaces the real cause, not the + // sibling's induced context-canceled. g.Wait joins both before run returns, + // restoring the single-lifecycle-goroutine invariant across supervisor restarts. + // supervise is the one clean-vs-restart decision point; a canceled parent ctx + // classifies as clean. + g, gctx := errgroup.WithContext(ctx) g.Go(func() error { - defer cancelLoops() // ingestion stopping tears down the lifecycle loop - // The ingestion loop is the hot-tier owner: it owns hotDB for the rest of its - // life (closes it on any exit, reopens at each boundary) and consumes the - // stream from the resume ledger. - return runIngestionLoop(loopCtx, ingestionLoopConfig{ + return runIngestionLoop(gctx, ingestionLoopConfig{ Stream: stream, Resume: resumeLedger, - HotDB: hotDB, Catalog: cat, Boundary: boundary, Logger: logger, @@ -157,8 +134,7 @@ func run(ctx context.Context, cfg StartConfig) error { }) }) g.Go(func() error { - defer cancelLoops() // a tick error tears down ingestion - return lifecycle.Loop(loopCtx, lifecycleCfg, cat, boundary) + return lifecycle.Loop(gctx, lifecycleCfg, cat, boundary) }) return g.Wait() } From 6df6ad8bdeed0e493e1cfd2d6979479a6712e4df Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 17:47:28 -0400 Subject: [PATCH 42/55] fullhistory/ingest: hot per-ledger phase timings (#18 amendment) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tamirms amended #18 after the Option-A confirmation: per-PHASE timings are a different axis from the per-type extraction the shared walk made meaningless, and the hot IngestStage plumbing should gain producers rather than be deleted. IngestLedger now returns LedgerPhases alongside LedgerCounts: Extract (the shared ExtractLedgerEvents walk + shaping, pre-batch), Ledgers/Txhash/Events (each facade's queue-into-batch step, stamped inside the batch callback), and Commit (the whole Batch minus those three = the RocksDB write: WAL append + fsync + memtable — the fsync wait pprof can't see). The three queue steps are strictly nested in the Batch call, so Commit is a non-negative remainder; the phases partition the per-ledger wall-clock. HotService reports them via IngestStage on the hot tier: extract and commit under a 'batch' pseudo-type (ledger-scoped, not per-type), the three queue steps under each type's stageWrite. PrometheusSink pre-resolves exactly these five hot-phase keys (not the cold cross-product) — so the hot stage histogram now has real producers instead of being dead weight. Whole tree build+vet, ingest+hotchunk, lifecycle -short, non-short E2E green. --- .../internal/fullhistory/hotloop_test.go | 2 +- .../fullhistory/ingest/ingest_test.go | 7 ++- .../internal/fullhistory/ingest/metrics.go | 34 ++++++++++-- .../internal/fullhistory/ingest/service.go | 27 +++++++--- .../lifecycle/lifecycle_helpers_test.go | 2 +- .../pkg/stores/hotchunk/hotchunk.go | 54 ++++++++++++++++--- .../pkg/stores/hotchunk/hotchunk_test.go | 20 +++---- 7 files changed, 113 insertions(+), 33 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index 7c3b69b0a..45ebfb3f1 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -163,7 +163,7 @@ func seedWatermark(t *testing.T, cat *catalog.Catalog, c chunk.ID, seq uint32) u t.Helper() db := openLiveHotDB(t, cat, c) for s := c.FirstLedger(); s <= seq; s++ { - _, err := db.IngestLedger(s, zeroTxLCMBytes(t, s)) + _, _, err := db.IngestLedger(s, zeroTxLCMBytes(t, s)) require.NoError(t, err) } require.NoError(t, db.Close()) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go index 8b525dc64..68709a917 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go @@ -807,7 +807,12 @@ func TestPrometheusSink_Smoke(t *testing.T) { sink.HotLedgerTotal(time.Millisecond, nil) sink.HotLedgerTotal(time.Millisecond, errFailingCold) // exercise the commit-error counter sink.ColdChunkTotal(time.Second) - sink.IngestStage(dataTypeEvents, tierHot, stageExtract, time.Millisecond, 3) + // The five hot per-ledger phases (batch-scoped extract/commit + per-type write). + sink.IngestStage(dataTypeBatch, tierHot, stageExtract, time.Millisecond, 0) + sink.IngestStage(dataTypeLedgers, tierHot, stageWrite, time.Millisecond, 0) + sink.IngestStage(dataTypeTxhash, tierHot, stageWrite, time.Millisecond, 0) + sink.IngestStage(dataTypeEvents, tierHot, stageWrite, time.Millisecond, 0) + sink.IngestStage(dataTypeBatch, tierHot, stageCommit, time.Millisecond, 0) sink.IngestStage(dataTypeEvents, tierCold, stageFinalize, time.Second, 0) }) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go index 941dce20d..7586e9abc 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go @@ -12,6 +12,10 @@ const ( dataTypeLedgers = "ledgers" dataTypeTxhash = "txhash" dataTypeEvents = "events" + // dataTypeBatch labels the hot per-ledger phases that are ledger-scoped rather + // than per data type — the shared extract walk and the batch commit — so they + // share one axis instead of being triple-counted across the three types. + dataTypeBatch = "batch" ) // Tier labels reported to a MetricSink. @@ -27,8 +31,9 @@ const ( const ( stageExtract = "extract" // view → payloads / hashes derivation stageTermIndex = "term_index" // per-event term derivation + mirror update (events cold) - stageWrite = "write" // store write / pack append + stageWrite = "write" // store write / pack append (cold) / queue-into-batch (hot) stageFinalize = "finalize" // per-chunk commit (pack trailer, index build, .bin write) + stageCommit = "commit" // hot: the RocksDB batch write (WAL append + fsync + memtable) ) // MetricSink receives ingest timing and volume signals. Ingesters report their @@ -173,6 +178,21 @@ var ( //nolint:gochecknoglobals // fixed label set, read-only var ingestStages = []string{stageExtract, stageTermIndex, stageWrite, stageFinalize} +// hotPhaseKeys is the fixed set of per-ledger phase children the hot ingest path +// reports via IngestStage(dataType, tierHot, stage): the shared extract walk and +// the batch commit under the batch pseudo-type, plus per-type queue-into-batch +// timings under stageWrite. Not the cold cross-product — the hot path has its own +// phase taxonomy — so the sink pre-resolves exactly these keys. +// +//nolint:gochecknoglobals // fixed label set, read-only +var hotPhaseKeys = []struct{ dataType, stage string }{ + {dataTypeBatch, stageExtract}, + {dataTypeLedgers, stageWrite}, + {dataTypeTxhash, stageWrite}, + {dataTypeEvents, stageWrite}, + {dataTypeBatch, stageCommit}, +} + // ingestCollectors bundles the pre-resolved per-(data_type, tier) children. // The label space is fixed at construction (three data types × two tiers), so // resolving the children once removes the per-emit label-map allocation and @@ -268,8 +288,9 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh hotStageVec := prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: metricsSubsystem, - Name: "hot_stage_duration_seconds", - Help: "per-stage wall-clock inside a hot Ingest (extract, write; ledgers emits write only)", + Name: "hot_stage_duration_seconds", + Help: "per-ledger phase wall-clock (batch/extract, {ledgers,txhash,events}/write, " + + "batch/commit; the phases sum to the per-ledger total)", Buckets: hotBuckets, }, []string{"data_type", "stage"}) @@ -286,7 +307,6 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh hotItems := make(map[string]prometheus.Counter, 3) cold := make(map[string]ingestCollectors, 3) - hotStage := make(map[string]prometheus.Observer, 3*len(ingestStages)) coldStage := make(map[string]prometheus.Observer, 3*len(ingestStages)) for _, dataType := range []string{dataTypeLedgers, dataTypeTxhash, dataTypeEvents} { hotItems[dataType] = ingestItems.WithLabelValues(dataType, tierHot) @@ -296,10 +316,14 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh errors: ingestErrors.WithLabelValues(dataType, tierCold), } for _, stage := range ingestStages { - hotStage[dataType+"/"+stage] = hotStageVec.WithLabelValues(dataType, stage) coldStage[dataType+"/"+stage] = coldStageVec.WithLabelValues(dataType, stage) } } + // Hot phases are a fixed 5-key set (not the cold cross-product). + hotStage := make(map[string]prometheus.Observer, len(hotPhaseKeys)) + for _, k := range hotPhaseKeys { + hotStage[k.dataType+"/"+k.stage] = hotStageVec.WithLabelValues(k.dataType, k.stage) + } return &PrometheusSink{ hotItems: hotItems, diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go index d16531687..796535f8a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go @@ -40,24 +40,37 @@ func NewHotService(db *hotchunk.DB, sink MetricSink) *HotService { } // Ingest commits lcm to the shared hot DB in one atomic synced WriteBatch -// (decision (a)) and emits the ledger's metrics. Attribution is batch-scoped, not -// per type: HotLedgerTotal carries the whole-batch wall-clock and the commit -// outcome (one fsync commits all CFs; post-#18 one shared ExtractLedgerEvents walk -// feeds both txhash and events, so neither timing nor an extraction failure is -// attributable per type). Per-type HotItems reports only VOLUME, on success — a -// failed atomic batch wrote nothing durably. +// (decision (a)) and emits the ledger's metrics. The batch OUTCOME is batch-scoped +// — HotLedgerTotal carries the whole-batch wall-clock and the commit error (one +// fsync commits all CFs, so there is no per-type commit error). Per-type HotItems +// reports VOLUME, on success. Per-PHASE timing is a separate axis: extract (the +// shared walk + shaping) and commit (the RocksDB write) are batch-scoped, the three +// queue steps per type — together they partition the per-ledger wall-clock, and +// commit surfaces the fsync-wait split that CPU profiles can't. func (s *HotService) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { start := time.Now() - counts, err := s.db.IngestLedger(seq, lcm) + counts, phases, err := s.db.IngestLedger(seq, lcm) s.sink.HotLedgerTotal(time.Since(start), err) if err == nil { s.sink.HotItems(dataTypeLedgers, counts.Ledgers) s.sink.HotItems(dataTypeTxhash, counts.Txhash) s.sink.HotItems(dataTypeEvents, counts.Events) + s.reportPhases(phases) } return err } +// reportPhases emits the per-ledger phase timings via the hot IngestStage plumbing: +// the shared extract + commit under the batch pseudo-type, the three queue steps +// under each type's stageWrite. Item counts are 0 — HotItems already carries volume. +func (s *HotService) reportPhases(p hotchunk.LedgerPhases) { + s.sink.IngestStage(dataTypeBatch, tierHot, stageExtract, p.Extract, 0) + s.sink.IngestStage(dataTypeLedgers, tierHot, stageWrite, p.Ledgers, 0) + s.sink.IngestStage(dataTypeTxhash, tierHot, stageWrite, p.Txhash, 0) + s.sink.IngestStage(dataTypeEvents, tierHot, stageWrite, p.Events, 0) + s.sink.IngestStage(dataTypeBatch, tierHot, stageCommit, p.Commit, 0) +} + // ColdService drives a set of ColdIngesters for one chunk: sequential per-ledger // Ingest, then Finalize on each. It times from the first Ingest (or, if none ran, // from the Finalize/Close call) and emits the aggregate ColdChunkTotal exactly diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go index 0bd376b3f..705951e32 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go @@ -97,7 +97,7 @@ func ingestFullHotChunk(t *testing.T, cat *catalog.Catalog, c chunk.ID) { } else { raw = zeroTxLCMBytes(t, seq) } - _, err := db.IngestLedger(seq, xdr.LedgerCloseMetaView(raw)) + _, _, err := db.IngestLedger(seq, xdr.LedgerCloseMetaView(raw)) require.NoError(t, err) } require.NoError(t, db.Close()) // release the write handle (boundary handoff) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index 485b7c30d..7ade848c3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -13,6 +13,7 @@ import ( "fmt" "iter" "slices" + "time" sdkingest "github.com/stellar/go-stellar-sdk/ingest" "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" @@ -160,6 +161,24 @@ type LedgerCounts struct { Events int } +// LedgerPhases reports the per-phase wall-clock of one IngestLedger call so the +// caller can attribute hot per-ledger CPU vs IO without re-instrumenting: +// - Extract: the shared ExtractLedgerEvents walk + txhash-entry build + event +// shaping (all pre-batch); +// - Ledgers/Txhash/Events: each facade's queue-into-batch step; +// - Commit: the RocksDB batch write (WAL append + fsync + memtable) = the whole +// Batch call minus the three queue steps — the fsync wait pprof can't see. +// +// The phases sum to ~the whole call (minus the tiny post-commit mirror apply); +// fields are zero for a phase that an error return preempted. +type LedgerPhases struct { + Extract time.Duration + Ledgers time.Duration + Txhash time.Duration + Events time.Duration + Commit time.Duration +} + // IngestLedger commits ONE ledger as a SINGLE atomic synced WriteBatch across all // hot CFs (decision (a)): queue ledgers, txhash, and events rows into one // BatchWriter, commit once, and only then apply the events in-memory mirror/offsets @@ -168,8 +187,11 @@ type LedgerCounts struct { // lcm is a borrowed zero-copy view; every extractor copies what it retains, so // the view need not outlive this call. Store.Batch's lifecycle RLock + checkOpen // is the authoritative closed-store guard, so there is no separate pre-check here. -func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts, error) { - var counts LedgerCounts +func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts, LedgerPhases, error) { + var ( + counts LedgerCounts + phases LedgerPhases + ) // Pre-extract anything that can fail BEFORE opening the batch, so a decode // error rejects the ledger without a half-built batch. @@ -184,9 +206,10 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts // LCMViewToPayloads. The atomic batch below serializes only the commit; the // extractors are independent and could run concurrently into the same batch if // catch-up profiling ever demands it — sequential is right at live cadence. + extractStart := time.Now() txEvents, err := sdkingest.ExtractLedgerEvents(lcm) if err != nil { - return counts, fmt.Errorf("extract ledger events seq %d: %w", seq, err) + return counts, phases, fmt.Errorf("extract ledger events seq %d: %w", seq, err) } txEntries := make([]txhash.Entry, len(txEvents)) for i := range txEvents { @@ -196,44 +219,59 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts closedAt, err := lcm.LedgerCloseTime() if err != nil { - return counts, fmt.Errorf("ledger close time seq %d: %w", seq, err) + return counts, phases, fmt.Errorf("ledger close time seq %d: %w", seq, err) } // A pre-Soroban ledger yields zero payloads, no error. payloads, err := events.PayloadsFromLedgerEvents(txEvents, seq, closedAt) if err != nil { - return counts, fmt.Errorf("shape events seq %d: %w", seq, err) + return counts, phases, fmt.Errorf("shape events seq %d: %w", seq, err) } counts.Events = len(payloads) counts.Ledgers = 1 + phases.Extract = time.Since(extractStart) // The events facade validates + marshals inside the batch callback (so a // rejected ledger never leaves committed rows) and returns the post-commit // apply hook. Under decision (a) resume is always MaxCommittedSeq+1, so seq is - // never a duplicate — the hook is always non-nil on success. + // never a duplicate — the hook is always non-nil on success. Each facade's queue + // step is timed individually; Commit (below) is the whole Batch minus those — + // the RocksDB write (WAL append + fsync + memtable). var applyEvents func() + batchStart := time.Now() cerr := d.store.Batch(func(b *rocksdb.BatchWriter) error { + ls := time.Now() if err := d.ledger.AddLedgerToBatch(b, ledger.Entry{Seq: seq, Bytes: []byte(lcm)}); err != nil { return fmt.Errorf("queue ledger seq %d: %w", seq, err) } + phases.Ledgers = time.Since(ls) + + ts := time.Now() if len(txEntries) > 0 { if err := d.txhash.AddEntriesToBatch(b, txEntries); err != nil { return fmt.Errorf("queue tx hashes seq %d: %w", seq, err) } } + phases.Txhash = time.Since(ts) + + es := time.Now() apply, err := d.events.IngestLedgerToBatch(b, seq, payloads) if err != nil { return fmt.Errorf("queue events seq %d: %w", seq, err) } + phases.Events = time.Since(es) applyEvents = apply return nil }) if cerr != nil { - return counts, fmt.Errorf("commit ledger %d to chunk %s: %w", seq, d.chunkID, cerr) + return counts, phases, fmt.Errorf("commit ledger %d to chunk %s: %w", seq, d.chunkID, cerr) } + // The three queue steps are strictly nested inside the Batch call (monotonic + // clock), so Commit is the non-negative remainder: the RocksDB write itself. + phases.Commit = time.Since(batchStart) - phases.Ledgers - phases.Txhash - phases.Events // Batch is durable — now and only now apply the events mirror/offsets update. applyEvents() - return counts, nil + return counts, phases, nil } // hotLedgerStream is a ledgerbackend.LedgerStream over a ledger.HotStore, so the diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go index 778aa871d..b5840777a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go @@ -82,11 +82,11 @@ func TestIngestLedger_AllCFsAdvanceTogether(t *testing.T) { rawA, hashA, termA := lcmWithEvent(t, first) rawB, hashB, _ := lcmWithEvent(t, first+1) - counts, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(rawA)) + counts, _, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(rawA)) require.NoError(t, err) assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 1, Events: 1}, counts) - counts, err = db.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawB)) + counts, _, err = db.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawB)) require.NoError(t, err) assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 1, Events: 1}, counts) @@ -130,7 +130,7 @@ func TestIngestLedger_RejectedLedgerPersistsNothingAcrossAnyCF(t *testing.T) { badSeq := chunkID.LastLedger() + 1 raw, hash, term := lcmWithEvent(t, badSeq) - _, err := db.IngestLedger(badSeq, xdr.LedgerCloseMetaView(raw)) + _, _, err := db.IngestLedger(badSeq, xdr.LedgerCloseMetaView(raw)) require.Error(t, err) require.ErrorIs(t, err, eventstore.ErrLedgerOutOfRange) @@ -166,7 +166,7 @@ func TestIngestLedger_MidBatchCommitFailurePersistsNothing(t *testing.T) { // Commit one good ledger so there is a known watermark, then close the DB. rawGood, hashGood, _ := lcmWithEvent(t, first) - _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(rawGood)) + _, _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(rawGood)) require.NoError(t, err) require.NoError(t, db.Close()) @@ -184,7 +184,7 @@ func TestIngestLedger_MidBatchCommitFailurePersistsNothing(t *testing.T) { // store: the commit fails, and nothing for that ledger persists anywhere. require.NoError(t, db2.Close()) rawNext, hashNext, _ := lcmWithEvent(t, first+1) - _, err = db2.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawNext)) + _, _, err = db2.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawNext)) require.Error(t, err) // Reopen a third time: the failed ledger left NO trace in any CF, and the @@ -260,7 +260,7 @@ func TestIngestLedger_WritesEveryHotType(t *testing.T) { db := openTestDB(t) raw, hash, term := lcmWithEvent(t, first) - counts, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) + counts, _, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) require.NoError(t, err) assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 1, Events: 1}, counts) @@ -288,7 +288,7 @@ func TestReopen_RecoversEventsMirror(t *testing.T) { db, err := Open(dir, chunkID, silentLogger()) require.NoError(t, err) raw, _, _ := lcmWithEvent(t, first) - _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) + _, _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) require.NoError(t, err) require.NoError(t, db.Close()) @@ -311,7 +311,7 @@ func TestOpenReadOnly_ReadsCommittedAndRejectsWrites(t *testing.T) { db, err := Open(dir, chunkID, silentLogger()) require.NoError(t, err) for _, seq := range []uint32{first, first + 1} { - _, ierr := db.IngestLedger(seq, xdr.LedgerCloseMetaView(zeroTxLCM(t, seq))) + _, _, ierr := db.IngestLedger(seq, xdr.LedgerCloseMetaView(zeroTxLCM(t, seq))) require.NoError(t, ierr) } require.NoError(t, db.Close()) @@ -327,7 +327,7 @@ func TestOpenReadOnly_ReadsCommittedAndRejectsWrites(t *testing.T) { assert.Equal(t, first+1, seq, "read-only handle sees the committed data") // A write through the read-only handle must fail — the freeze never mutates. - _, err = ro.IngestLedger(first+2, xdr.LedgerCloseMetaView(zeroTxLCM(t, first+2))) + _, _, err = ro.IngestLedger(first+2, xdr.LedgerCloseMetaView(zeroTxLCM(t, first+2))) require.Error(t, err, "read-only DB must reject writes") } @@ -342,7 +342,7 @@ func TestIngestLedger_ClosedDBFails(t *testing.T) { require.NoError(t, db.Close()) raw := zeroTxLCM(t, chunkID.FirstLedger()) - _, err = db.IngestLedger(chunkID.FirstLedger(), xdr.LedgerCloseMetaView(raw)) + _, _, err = db.IngestLedger(chunkID.FirstLedger(), xdr.LedgerCloseMetaView(raw)) require.ErrorIs(t, err, rocksdb.ErrStoreClosed) } From 73f95dc3580a0aeaeeed2b80e633065ddeae001f Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 17:56:55 -0400 Subject: [PATCH 43/55] =?UTF-8?q?fullhistory:=20round-3=20review=20polish?= =?UTF-8?q?=20=E2=80=94=20dead=20code,=20metric=20name,=20stale=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prior-session review findings (tamirms round 3), none behavioral beyond the metric rename: - Delete rocksdb.Store.FirstKey (production-dead since ledger.HotStore.FirstSeq went); fold edgeKey into LastKey. Fix the MustExist doc: a failed must-exist open may leave a stub leaf dir (LOG) — correctness holds (retries fail on the missing CURRENT), but it's not 'never created'. - Drop BoundarySignal's 'set' flag: only Publish sends a wake, and it stores the cell first, so a received wake proves a value is present. take()->latestChunk() just loads; Loop loses the dead !ok branch. Soften Loop's ctx-cancel doc (a mid-tick cancel returns a wrapped ctx error, still clean). - Rename pruned_ops_total -> pruned_artifacts_total (both callers meter artifacts) with accurate help. - Fix inverted OpenReadOnly doc (+ process.go): the read-only open's WAL replay is DEPENDED ON by the startup watermark after an ungraceful crash — not 'no WAL to replay'. Drop cold_index.go's stale ConcurrentBitmaps.Snapshot freeze-source cite. Correct the false 'read-only opens contend the LOCK' rationale in hotsource_test / progress_realdb_test (read-only takes no LOCK; closing is hygiene) and drop the 'probe' references. - Test tidy: collapse the NextEventID->EventCount self-comparing double assertions in hot_store_test; use len(ledger.CFNames()) not a hand-stitched 1. Whole tree build+vet; rocksdb/hotchunk/eventstore/observability/lifecycle/ backfill tests green. --- .../fullhistory/backfill/hotsource_test.go | 5 ++-- .../internal/fullhistory/backfill/process.go | 6 ++-- .../fullhistory/lifecycle/lifecycle.go | 26 ++++++----------- .../lifecycle/lifecycle_arith_test.go | 1 - .../lifecycle/progress_realdb_test.go | 7 +++-- .../observability/observability.go | 2 +- .../observability/observability_test.go | 2 +- .../fullhistory/pkg/rocksdb/rocksdb.go | 28 ++++--------------- .../fullhistory/pkg/rocksdb/rocksdb_test.go | 28 ++++--------------- .../pkg/stores/eventstore/cold_index.go | 6 ++-- .../pkg/stores/eventstore/hot_store_test.go | 3 -- .../pkg/stores/hotchunk/hotchunk.go | 8 ++++-- .../pkg/stores/hotchunk/hotchunk_test.go | 2 +- 13 files changed, 42 insertions(+), 82 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/hotsource_test.go b/cmd/stellar-rpc/internal/fullhistory/backfill/hotsource_test.go index 70ab6de4a..a27f542e6 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/hotsource_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/hotsource_test.go @@ -16,8 +16,9 @@ import ( // seedReadyHotChunk brackets a "ready" hot DB for c (transient -> create -> ready) // and commits ONE ledgers-CF entry at seq `top` so MaxCommittedSeq reads back // `top`. It writes just the ledgers CF (the only CF the completeness gate reads) -// and closes the store, so tryHotSource's read-only reopen is not blocked by the -// RocksDB LOCK. The daemon opens this exact on-disk DB by its Layout path. +// and closes the store — hygiene, not a lock requirement: a read-only open takes +// no RocksDB LOCK and would succeed against a writer-held DB too. The daemon opens +// this exact on-disk DB by its Layout path. func seedReadyHotChunk(t *testing.T, cat *catalog.Catalog, c chunk.ID, top uint32) { t.Helper() require.NoError(t, cat.PutHotTransient(c)) diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go index 135625e27..3e059b688 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/process.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/process.go @@ -219,9 +219,9 @@ func resolveHotSource( func tryHotSource(chunkID chunk.ID, cfg ProcessConfig) (ledgerbackend.LedgerStream, func() error, bool, error) { dir := cfg.Catalog.Layout().HotChunkPath(chunkID) // Open the chunk's shared multi-CF DB READ-ONLY: the freeze reads its ledgers to - // re-derive the cold artifacts and must never mutate it. The freeze only targets - // chunks ingestion already released, so its data is in SST (no WAL replay). An - // absent or gutted "ready" DB fails the open — restartable, never auto-created. + // re-derive the cold artifacts and must never mutate it (the read-only open + // replays any un-synced WAL into memtables but persists nothing). An absent or + // gutted "ready" DB fails the open — restartable, never auto-created. hot, err := hotchunk.OpenReadOnly(dir, chunkID, cfg.Logger) if err != nil { return nil, nil, false, fmt.Errorf("chunk %s is ready but its hot DB won't open: %w", chunkID, err) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index 05ad85ae4..89aab47fb 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -154,7 +154,6 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu // producer and one consumer. type BoundarySignal struct { latest atomic.Uint32 - set atomic.Bool wake chan struct{} } @@ -168,20 +167,17 @@ func NewBoundarySignal() *BoundarySignal { // the newest latest when it runs), so a full buffer is dropped, never blocked on. func (s *BoundarySignal) Publish(c chunk.ID) { s.latest.Store(uint32(c)) - s.set.Store(true) select { case s.wake <- struct{}{}: default: } } -// take returns the latest published chunk id; ok=false when nothing has been -// published (chunk 0 is a valid id, so a separate flag distinguishes it). -func (s *BoundarySignal) take() (chunk.ID, bool) { - if !s.set.Load() { - return 0, false - } - return chunk.ID(s.latest.Load()), true +// latestChunk returns the most recently published completed chunk id. A wake is +// only ever sent by Publish, AFTER it stores the cell, so a received wake proves a +// value is present — no separate "was anything published" flag is needed. +func (s *BoundarySignal) latestChunk() chunk.ID { + return chunk.ID(s.latest.Load()) } // Loop is the event-driven lifecycle goroutine. It blocks on the boundary signal's @@ -190,20 +186,16 @@ func (s *BoundarySignal) take() (chunk.ID, bool) { // selects on ctx.Done() too, so it never blocks past shutdown. // // It returns the first tick error to its caller (run() joins it with ingestion in -// an errgroup, so supervise decides clean-vs-restart). A ctx cancellation returns -// nil — cancellation is a shutdown, and the sibling goroutine (ingestion, or an -// already-returned failing tick) carries any real cause. +// an errgroup, so supervise decides clean-vs-restart). A cancellation observed at +// the select returns nil; a cancellation mid-tick returns the tick's wrapped ctx +// error — both are clean, since supervise keys off the daemon ctx, not this return. func Loop(ctx context.Context, cfg Config, cat *catalog.Catalog, sig *BoundarySignal) error { for { select { case <-ctx.Done(): return nil case <-sig.wake: - lastChunk, ok := sig.take() - if !ok { - continue - } - if err := runLifecycle(ctx, cfg, cat, lastChunk); err != nil { + if err := runLifecycle(ctx, cfg, cat, sig.latestChunk()); err != nil { return err } } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_arith_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_arith_test.go index e386e1432..84d230749 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_arith_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_arith_test.go @@ -92,4 +92,3 @@ func TestEffectiveRetentionFloor(t *testing.T) { }) } } - diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go index 582c75a7e..6da718c8f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go @@ -40,7 +40,8 @@ func seedLedgersCF(t *testing.T, cat *catalog.Catalog, c chunk.ID, entries ...le // seedReadyLiveDB brackets a "ready" hot DB for chunk c (via the production // opener) and commits a single ledgers-CF entry at seq `top` so MaxCommittedSeq // reads back `top`. top==0 leaves the DB empty (present=false). It closes the DB -// so the refinement's read-only reopen is not blocked by the RocksDB LOCK. +// as hygiene — a read-only reopen takes no RocksDB LOCK, so this isn't required +// for the refinement to open, but it keeps the fixtures single-handle. func seedReadyLiveDB(t *testing.T, cat *catalog.Catalog, c chunk.ID, top uint32) { t.Helper() db := openLiveHotDB(t, cat, c) // ready key + real dir + empty DB @@ -63,8 +64,8 @@ func TestDeriveWatermark_RealHotDB_RefinementIsNotStale(t *testing.T) { // Production bracket: creates the hot dir, opens the SINGLE shared multi-CF // DB, flips the hot key "ready". This is exactly what ingestion does. db := openLiveHotDB(t, cat, live) - // Close the live writer before seeding + the probe's read-only reopen - // (RocksDB LOCK). + // Close the live writer before seeding — hygiene (the refinement's read-only + // reopen takes no RocksDB LOCK), keeping the fixture single-handle. require.NoError(t, db.Close()) // Commit two real ledgers into the ledgers CF (the CF MaxCommittedSeq reads). diff --git a/cmd/stellar-rpc/internal/fullhistory/observability/observability.go b/cmd/stellar-rpc/internal/fullhistory/observability/observability.go index dc454f5f2..708e9111d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/observability/observability.go +++ b/cmd/stellar-rpc/internal/fullhistory/observability/observability.go @@ -109,7 +109,7 @@ func NewPrometheusMetrics(registry *prometheus.Registry, namespace string) *Prom liveHotChunks: gauge("live_hot_chunks", "count of hot-chunk DBs currently on disk"), chunkBoundaries: counter("chunk_boundaries_total", "ingestion chunk-boundary handoffs"), discarded: counter("discarded_hot_chunks_total", "hot DBs retired by the discard stage"), - pruned: counter("pruned_ops_total", "artifacts swept after an index build"), + pruned: counter("pruned_artifacts_total", "artifacts swept by the prune stage (below the retention floor)"), phaseDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, Name: "phase_duration_seconds", Help: "wall-clock of a daemon phase action", diff --git a/cmd/stellar-rpc/internal/fullhistory/observability/observability_test.go b/cmd/stellar-rpc/internal/fullhistory/observability/observability_test.go index 6ebe0310a..9814ff648 100644 --- a/cmd/stellar-rpc/internal/fullhistory/observability/observability_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/observability/observability_test.go @@ -61,7 +61,7 @@ func TestPrometheusMetrics_RegistersAndRecords(t *testing.T) { assert.InDelta(t, float64(58), values["test_ns_fullhistory_streaming_last_committed_ledger"], 0) assert.InDelta(t, float64(12), values["test_ns_fullhistory_streaming_retention_floor_ledger"], 0) - assert.InDelta(t, float64(2), values["test_ns_fullhistory_streaming_pruned_ops_total"], 0) + assert.InDelta(t, float64(2), values["test_ns_fullhistory_streaming_pruned_artifacts_total"], 0) // Phase-duration histogram saw backfill_pass + freeze + rebuild + prune = 4 observations. assert.Equal(t, uint64(4), counts["test_ns_fullhistory_streaming_phase_duration_seconds"]) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go index 7f681483a..0f1b56552 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb.go @@ -67,9 +67,11 @@ type Config struct { ReadOnly bool // MustExist opens read-WRITE but with create-if-missing OFF, so opening a - // missing or gutted DB fails instead of silently fabricating a fresh empty - // one. The dir is never created. Used for the "never auto-heal" hot-DB open - // under a "ready" key — a DB the filesystem should already hold. Ignored when + // missing or gutted DB fails instead of silently fabricating a fresh empty one + // — the "never auto-heal" hot-DB open under a "ready" key, a DB the filesystem + // should already hold. (RocksDB's env layer may still leave a stub leaf dir with + // a LOG file behind on the failed open; correctness holds — every retry still + // fails on the missing CURRENT — but no usable DB is created.) Ignored when // ReadOnly is set (read-only never creates regardless). MustExist bool } @@ -306,26 +308,12 @@ func (s *Store) Iterate(cf string, prefix []byte) iter.Seq2[Entry, error] { } } -// FirstKey returns the smallest key in cf. If cf has no keys this is not -// an error: it returns (nil, false, nil), so callers detect emptiness via -// ok. (cf == "" selects the default column family; an unregistered cf name -// returns ErrCFNotFound.) -// Cheap: a single boundary seek (no scan). -func (s *Store) FirstKey(cf string) ([]byte, bool, error) { - return s.edgeKey(cf, false) -} - // LastKey returns the largest key in cf. If cf has no keys this is not an // error: it returns (nil, false, nil), so callers detect emptiness via ok. // (cf == "" selects the default column family; an unregistered cf name // returns ErrCFNotFound.) // Cheap: a single boundary seek (no scan). func (s *Store) LastKey(cf string) ([]byte, bool, error) { - return s.edgeKey(cf, true) -} - -//nolint:funcorder // helper grouped with FirstKey/LastKey for readability -func (s *Store) edgeKey(cf string, last bool) ([]byte, bool, error) { s.mu.RLock() defer s.mu.RUnlock() @@ -339,11 +327,7 @@ func (s *Store) edgeKey(cf string, last bool) ([]byte, bool, error) { it := s.db.NewIteratorCF(s.ro, cfh) defer it.Close() - if last { - it.SeekToLast() - } else { - it.SeekToFirst() - } + it.SeekToLast() if !it.Valid() { // Empty CF (it.Err() is nil) or a mid-seek RocksDB error. return nil, false, it.Err() diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb_test.go index 999803b75..07fe7c4e7 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb_test.go @@ -141,27 +141,19 @@ func TestStore_PutGet_DefaultCF(t *testing.T) { assert.False(t, found3) } -func TestStore_FirstLastKey(t *testing.T) { +func TestStore_LastKey(t *testing.T) { s := openTestStore(t, nil) - // Empty default CF: ok=false, no error, at both ends. - _, ok, err := s.FirstKey("") - require.NoError(t, err) - require.False(t, ok) - _, ok, err = s.LastKey("") + // Empty default CF: ok=false, no error. + _, ok, err := s.LastKey("") require.NoError(t, err) require.False(t, ok) // EncodeUint32 is big-endian, so byte-lex key order is numeric order: - // insert out of order and expect the min/max back. + // insert out of order and expect the max back. for _, n := range []uint32{500, 1, 9999, 42} { require.NoError(t, s.Put("", EncodeUint32(n), []byte{byte(n)})) } - first, ok, err := s.FirstKey("") - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, uint32(1), DecodeUint32(first)) - last, ok, err := s.LastKey("") require.NoError(t, err) require.True(t, ok) @@ -169,28 +161,21 @@ func TestStore_FirstLastKey(t *testing.T) { // Unknown CF surfaces ErrCFNotFound (distinct from ok=false on an // empty-but-configured CF). - _, _, err = s.FirstKey("not-configured") - require.ErrorIs(t, err, ErrCFNotFound) _, _, err = s.LastKey("not-configured") require.ErrorIs(t, err, ErrCFNotFound) - // Non-default CF: FirstKey/LastKey resolve the requested CF - // independently of the default CF. + // Non-default CF: LastKey resolves the requested CF independently of the default. const altCF = "alt" sAlt := openTestStore(t, []string{altCF}) for _, n := range []uint32{7, 3, 8} { require.NoError(t, sAlt.Put(altCF, EncodeUint32(n), []byte{byte(n)})) } - first, ok, err = sAlt.FirstKey(altCF) - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, uint32(3), DecodeUint32(first)) last, ok, err = sAlt.LastKey(altCF) require.NoError(t, err) require.True(t, ok) require.Equal(t, uint32(8), DecodeUint32(last)) // The default CF of the same store is untouched → ok=false. - _, ok, err = sAlt.FirstKey("") + _, ok, err = sAlt.LastKey("") require.NoError(t, err) require.False(t, ok) } @@ -373,7 +358,6 @@ func TestStore_OpsAfterCloseFailWithErrStoreClosed(t *testing.T) { }{ {"Put", func() error { return s.Put(defaultCFName, []byte("k"), []byte("v")) }}, {"Get", func() error { _, _, err := s.Get(defaultCFName, []byte("k")); return err }}, - {"FirstKey", func() error { _, _, err := s.FirstKey(defaultCFName); return err }}, {"LastKey", func() error { _, _, err := s.LastKey(defaultCFName); return err }}, {"Delete", func() error { return s.Delete(defaultCFName, []byte("k")) }}, {"Iterate", func() error { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/cold_index.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/cold_index.go index 8c02c8107..c37a4f912 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/cold_index.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/cold_index.go @@ -131,9 +131,9 @@ func WriteColdIndex(ctx context.Context, chunkID chunk.ID, bitmaps events.Bitmap } var fp [IndexRecordFingerprintLen]byte copy(fp[:], term[:IndexRecordFingerprintLen]) - // Mutate in place — bitmaps is uniquely owned by the caller - // (built single-threaded for cold backfill, or Cloned via - // ConcurrentBitmaps.Snapshot for the live-chunk freeze path). + // Mutate in place — bitmaps is uniquely owned by the caller, built + // single-threaded either way: cold backfill from the .pack, or the freeze + // from the read-only hot DB. bitmap.RunOptimize() entries = append(entries, indexEntry{slot: slot, fp: fp, bitmap: bitmap}) } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go index 49523bed3..bda698c7f 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store_test.go @@ -476,7 +476,6 @@ func TestHotStore_IngestLedgerEvents_RejectsLedgerGap(t *testing.T) { require.NoError(t, ingestLedgerEvents(h.store, first, []events.Payload{p1})) countBefore := mustEventCount(t, h.store) - nextBefore := mustEventCount(t, h.store) // Skip first+1; jump directly to first+2. p2, _ := makePayload("c") @@ -484,7 +483,6 @@ func TestHotStore_IngestLedgerEvents_RejectsLedgerGap(t *testing.T) { require.ErrorIs(t, err, ErrLedgerOutOfOrder) assert.Equal(t, countBefore, mustEventCount(t, h.store)) - assert.Equal(t, nextBefore, mustEventCount(t, h.store)) } // TestHotStore_IngestLedgerEvents_RejectsOutOfRangeLedger pins the @@ -505,7 +503,6 @@ func TestHotStore_IngestLedgerEvents_RejectsOutOfRangeLedger(t *testing.T) { // State must be unchanged after both rejections. assert.Equal(t, uint32(0), mustEventCount(t, h.store)) - assert.Equal(t, uint32(0), mustEventCount(t, h.store)) } func TestHotStore_CloseIsIdempotent(t *testing.T) { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index 7ade848c3..f8b76ca96 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -86,9 +86,11 @@ func OpenExisting(path string, chunkID chunk.ID, logger *supportlog.Entry) (*DB, return open(path, chunkID, logger, false, true) } -// OpenReadOnly opens an EXISTING hot DB read-only — the freeze source's view. The -// freeze only ever opens a chunk ingestion has already cleanly closed, so all -// data is in SST (no WAL to replay); composing the facades only reads. +// OpenReadOnly opens an EXISTING hot DB read-only — the freeze source's view AND +// the startup watermark refiner's. RocksDB's read-only open recovers any un-synced +// WAL into in-memory memtables (persisting nothing), so a reader sees every synced +// write even after an ungraceful crash — the watermark refinement DEPENDS on that +// replay to read a correct MaxCommittedSeq. Composing the facades only reads. func OpenReadOnly(path string, chunkID chunk.ID, logger *supportlog.Entry) (*DB, error) { return open(path, chunkID, logger, true, false) } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go index b5840777a..adef22e4b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go @@ -50,7 +50,7 @@ func TestOpen_ValidatesInputs(t *testing.T) { func TestColumnFamilies_UnionIsNonColliding(t *testing.T) { cfs := ColumnFamilies() // 1 ledger CF + 3 events CFs + 1 txhash CF = 5. - require.Len(t, cfs, 1+len(eventstore.CFNames())+len(txhash.CFNames())) + require.Len(t, cfs, len(ledger.CFNames())+len(eventstore.CFNames())+len(txhash.CFNames())) seen := map[string]bool{} for _, cf := range cfs { require.False(t, seen[cf], "CF name %q collides across facades", cf) From c2d5cc9c0aebd1285a5bb80e2535bf4a4bf590d4 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 18:01:29 -0400 Subject: [PATCH 44/55] fullhistory/catalog: test DiscardHotChunk crash-resume + absent-key noop (#18 review) The two sweep siblings have crash-resume tests (TestSweepChunkArtifactsIdempotentOnMissingFiles, TestSweepIndexKeyFreezingDebris); DiscardHotChunk's documented 'a crash anywhere leaves the key transient for the next scan to finish' had none. Add one that seeds a transient key + leftover dir and asserts the next DiscardHotChunk completes it (dir + key gone), plus an absent-key no-op case. --- .../fullhistory/catalog/catalog_sweep_test.go | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_sweep_test.go b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_sweep_test.go index 2a287f61b..762e48ba8 100644 --- a/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_sweep_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/catalog/catalog_sweep_test.go @@ -1,11 +1,13 @@ package catalog import ( + "os" "testing" "github.com/stretchr/testify/require" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" ) // --------------------------------------------------------------------------- @@ -104,3 +106,32 @@ func TestSweepEmptyRefsNoop(t *testing.T) { cat, _ := testCatalog(t) require.NoError(t, cat.SweepChunkArtifacts(nil)) } + +// TestDiscardHotChunkResumesTransient mirrors the sweep siblings' crash-resume +// coverage for the hot-DB discard: a "transient" key (a discard that crashed after +// marking transient but before deleting the key) plus a leftover dir must be +// finished by the next DiscardHotChunk — the dir removed and the key deleted. +func TestDiscardHotChunkResumesTransient(t *testing.T) { + cat, _ := testCatalog(t) + c := chunk.ID(4) + + // The mid-discard crash state: a "transient" key + a real leftover dir. + require.NoError(t, cat.PutHotTransient(c)) + dir := cat.layout.HotChunkPath(c) + require.NoError(t, os.MkdirAll(dir, 0o755)) + + require.NoError(t, cat.DiscardHotChunk(c)) + + // The resume completed it: key gone, dir gone. + state, err := cat.HotState(c) + require.NoError(t, err) + require.Equal(t, geometry.HotState(""), state, "transient key finished") + require.NoDirExists(t, dir, "leftover hot dir swept") +} + +// TestDiscardHotChunkAbsentKeyNoop: an absent hot key is a clean no-op (nothing +// to finish). +func TestDiscardHotChunkAbsentKeyNoop(t *testing.T) { + cat, _ := testCatalog(t) + require.NoError(t, cat.DiscardHotChunk(chunk.ID(9))) +} From 44d4faa55661ca4a5c5999cafdfbac41f25b590e Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 18:04:42 -0400 Subject: [PATCH 45/55] fullhistory: trim over-narrated comments (#3516018538) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment-only cleanup per tamirms's round-3 note: cut counterfactuals and repeated rationale down to the load-bearing line — the #25-clamp narration in runLifecycle, the LIVE-CHUNK EXCLUSION block, the hotLedgerStream dead-branch aside, the deleted-snapshot-path tombstones in eventstore (index()/cold_index), and the restated stream-owns-core / hot-DB-nil contracts (OpenCore, NewHotService). The supervise/errgroup/No-os.Exit comment trims from the same note already landed with the #16 rework. --- cmd/stellar-rpc/internal/fullhistory/daemon.go | 9 ++------- cmd/stellar-rpc/internal/fullhistory/hotloop.go | 13 ++++++------- .../internal/fullhistory/ingest/service.go | 5 +---- .../internal/fullhistory/lifecycle/lifecycle.go | 12 ++++-------- .../fullhistory/pkg/stores/eventstore/cold_index.go | 10 +++------- .../fullhistory/pkg/stores/eventstore/hot_store.go | 8 +++----- .../fullhistory/pkg/stores/hotchunk/hotchunk.go | 4 +--- 7 files changed, 20 insertions(+), 41 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/daemon.go b/cmd/stellar-rpc/internal/fullhistory/daemon.go index 42abb08a7..1e5937bb1 100644 --- a/cmd/stellar-rpc/internal/fullhistory/daemon.go +++ b/cmd/stellar-rpc/internal/fullhistory/daemon.go @@ -369,13 +369,8 @@ func newCaptiveCoreOpener(ing IngestionConfig, dataDir string, logger *supportlo }, nil } -// OpenCore returns the live ingestion stream backed by captive stellar-core. The -// stream OWNS the core process lifecycle — a fresh core is started on the first -// RawLedgers pull and torn down when iteration ends (the ingestion loop exits) — -// so there is no eager PrepareRange and no separate closer here; a fresh core per -// run keeps supervised restarts clean. The loop pulls RawLedgers over the -// unbounded range from its resume ledger, consuming the cached raw frame directly -// (no GetLedger→MarshalBinary round-trip). +// OpenCore returns the live ingestion stream backed by captive stellar-core. A +// fresh core per run keeps supervised restarts clean. func (c *captiveCoreOpener) OpenCore(ctx context.Context) (ledgerbackend.LedgerStream, error) { cfg := c.config cfg.Context = ctx diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index def67a4e2..2496cbd35 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -116,13 +116,12 @@ type ingestionLoopConfig struct { // discard a dir a still-live writer holds. Publish fires only after the next DB is // open. The HotService is rebuilt each boundary. // -// LIVE-CHUNK EXCLUSION (one home): this loop is the SOLE writer of a chunk's hot -// DB, and closes the live DB before publishing the completed chunk (the fence -// above). The lifecycle tick only ever targets chunks at or below the highest -// durably-complete chunk — strictly below the live chunk — so the read-only freeze -// and watermark-refinement opens never touch a DB this loop holds. A read-only -// open skips the RocksDB LOCK, so that separation is a correctness invariant kept -// here in the producer by construction, not a lock the readers rely on. +// LIVE-CHUNK EXCLUSION: this loop is the SOLE writer of a chunk's hot DB and +// closes it before publishing the chunk complete (the fence above); the lifecycle +// only ever opens chunks at or below the highest complete one — strictly below the +// live chunk. Those opens are read-only, which takes no RocksDB LOCK, so +// writer/reader separation is a construction invariant here, not a lock readers +// rely on. func runIngestionLoop(ctx context.Context, cfg ingestionLoopConfig) (err error) { metrics := observability.MetricsOrNop(cfg.Metrics) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go index 796535f8a..0c95a5a62 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go @@ -31,10 +31,7 @@ type HotService struct { } // NewHotService builds a HotService that writes ledgers, txhash, and events into -// the shared per-chunk DB. db is REQUIRED (the hot DB is the sole copy of a -// chunk's un-frozen ledgers) — the caller opens it via openHotDBForChunk, which -// returns a non-nil DB or an error, so it is never nil on any wired path. A nil -// sink defaults to NopSink. +// the shared per-chunk DB. A nil sink defaults to NopSink. func NewHotService(db *hotchunk.DB, sink MetricSink) *HotService { return &HotService{db: db, sink: orNop(sink)} } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index 89aab47fb..856fcfdbd 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -90,14 +90,10 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu // Stage 1 — plan-and-execute (freeze + index fold) over [floor, lastChunk], via // the same entry point backfill uses (resolve → executePlan → Freeze metric, // recorded internally). A canceled ctx makes RunBackfill return ctx.Err(), which - // propagates up for supervise to treat as a clean shutdown. - // - // No rangeEnd clamp to the highest-complete chunk and no floor raise to - // lowestMaterializedChunk (both traced dead, #25): the Loop only ever fires for - // a genuinely completed lastChunk (the upstream boundary-handoff fence + seed - // guard), and recovery leaves chunk-aligned watermarks, so neither clamp can - // fire with a consequence beyond re-download churn. The only guard left is the - // empty-range check (floor above lastChunk when retention outran production). + // propagates up for supervise to treat as a clean shutdown. lastChunk is always + // a completed chunk (boundary fence + post-backfill seed), so the only guard + // needed is the empty-range check (floor above lastChunk when retention outran + // production). freezeStart := time.Now() start := ChunkIDOfLedger(floor) if start >= 0 && start <= int64(lastChunk) { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/cold_index.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/cold_index.go index c37a4f912..70ad7fb78 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/cold_index.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/cold_index.go @@ -50,13 +50,9 @@ import ( // hit a slow (*Bitmap).lazyOR path at query time and K≥12 regresses // catastrophically. // -// Bitmaps reach this function one way today: -// -// - Both cold backfill and the live-chunk freeze build a Bitmaps -// single-threaded by re-deriving terms from raw LCMs (per-event -// events.TermsFor + Bitmaps.AddTo) and hand it directly here. The -// freeze does NOT snapshot the hot in-memory mirror — that path has -// no production caller (see eventstore.HotStore.index). +// Both cold backfill and the live-chunk freeze build a Bitmaps single-threaded by +// re-deriving terms from raw LCMs (per-event events.TermsFor + Bitmaps.AddTo) and +// hand it directly here. // // index.hash is the MPHF serialized via buildMPHF. // diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index 054a0e542..0cc735072 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -166,11 +166,9 @@ func (h *HotStore) Offsets() (*events.LedgerOffsets, error) { return h.offsets.View(), nil } -// index returns the in-memory term mirror. Test-only write hook: no -// production path reads it. The live-chunk freeze re-derives the cold -// event index from raw LCMs (see backfill), so it never snapshots this -// mirror. Kept unexported until #772 decides whether the v2 read path -// hooks a snapshot here. +// index returns the in-memory term mirror. Test-only write hook: no production +// path reads it. Kept unexported until #772 decides whether the v2 read path +// hooks into it. func (h *HotStore) index() *events.ConcurrentBitmaps { return h.mirror } // Lookup returns the bitmap of event IDs in this Chunk that match diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index f8b76ca96..ab4b13e84 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -292,9 +292,7 @@ func (st *hotLedgerStream) RawLedgers( ctx context.Context, r ledgerbackend.Range, _ ...ledgerbackend.StreamOption, ) iter.Seq2[[]byte, error] { return func(yield func([]byte, error) bool) { - // The only caller is the freeze via Source(), which always passes a bounded - // chunk range over a constructor-set store (d.ledger). Assert the bound - // rather than carry the dead unbounded-range and nil-store branches. + // The freeze always passes a bounded chunk range; assert it. if !r.Bounded() { yield(nil, fmt.Errorf("hotLedgerStream requires a bounded range, got unbounded from %d", r.From())) return From 6ebcc029b9eeeeaa089e5d30af840fafec3a6bd9 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 18:24:06 -0400 Subject: [PATCH 46/55] fullhistory: guard the errgroup no-hang invariant in run() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review hardening: errgroup.WithContext cancels gctx (and so unblocks the lifecycle sibling waiting in g.Wait) only on a non-nil return, so the whole no-hang property rests on runIngestionLoop never returning nil while live. It upholds that today — every exit path is an error, including a clean stream end — but nothing enforces it. Convert a would-be nil return into an error at the errgroup seam so a future edit degrades to a supervised restart instead of a silent g.Wait hang. No behavior change on any current path. --- cmd/stellar-rpc/internal/fullhistory/startup.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index 830f19a94..b38b7a844 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -123,7 +123,7 @@ func run(ctx context.Context, cfg StartConfig) error { // classifies as clean. g, gctx := errgroup.WithContext(ctx) g.Go(func() error { - return runIngestionLoop(gctx, ingestionLoopConfig{ + err := runIngestionLoop(gctx, ingestionLoopConfig{ Stream: stream, Resume: resumeLedger, Catalog: cat, @@ -132,6 +132,14 @@ func run(ctx context.Context, cfg StartConfig) error { Metrics: metrics, Sink: cfg.Exec.Process.Sink, }) + if err == nil { + // WithContext cancels gctx (unblocking the lifecycle sibling in g.Wait) + // ONLY on a non-nil return. runIngestionLoop upholds that — every exit is + // an error, including a clean stream end — but guard it so a future nil + // return degrades to a supervised restart, never a silent g.Wait hang. + return errors.New("ingestion loop returned nil unexpectedly") + } + return err }) g.Go(func() error { return lifecycle.Loop(gctx, lifecycleCfg, cat, boundary) From e6de0f9e01c67c762e07db2bbdffe4539b3d9581 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 21:40:07 -0400 Subject: [PATCH 47/55] fullhistory: fix golangci-lint findings on the review-round changes The 10 --new-from-rev findings from the golangci-lint job, all on lines this review round touched: - lll (7): wrap the RunBackfill call (extract startChunk), loopConfig signature, and 5 WriteColdChunk/buildColdIngesters call sites that the added chunkID arg pushed past 120. - misspell: cancelling -> canceling (run()'s errgroup comment). - nolintlint: drop the now-unused //nolint:gosec in lastCompleteChunkAtID (gosec doesn't run on _test.go, so the directive suppressed nothing). - funcorder: move the unexported HotStore.index() below the exported IngestLedgerToBatch (next to the other unexported method, applyLedger). No behavior change. Build+vet+gofmt clean; ingest/eventstore/lifecycle -short green. --- .../internal/fullhistory/hotloop_test.go | 4 +++- .../internal/fullhistory/ingest/ingest_test.go | 15 ++++++++++----- .../internal/fullhistory/lifecycle/lifecycle.go | 3 ++- .../lifecycle/lifecycle_helpers_test.go | 2 +- .../pkg/stores/eventstore/hot_store.go | 10 +++++----- cmd/stellar-rpc/internal/fullhistory/startup.go | 2 +- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index 45ebfb3f1..07d0674ee 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -118,7 +118,9 @@ func (r *recordingBoundary) list() []chunk.ID { // a recording boundary. The loop opens the resume chunk's hot DB itself, so no DB // handle is passed — and the test must hold none on that dir while the loop runs (a // second read-write open would contend the RocksDB LOCK). -func loopConfig(stream ledgerbackend.LedgerStream, cat *catalog.Catalog, resume uint32) (ingestionLoopConfig, *recordingBoundary) { +func loopConfig( + stream ledgerbackend.LedgerStream, cat *catalog.Catalog, resume uint32, +) (ingestionLoopConfig, *recordingBoundary) { rec := &recordingBoundary{} return ingestionLoopConfig{ Stream: stream, diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go index 68709a917..89ec01b53 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go @@ -632,7 +632,8 @@ func TestColdService_Success(t *testing.T) { coldDir := t.TempDir() sink := &testSink{} - ings, err := buildColdIngesters(coldDirsAt(coldDir, chunkID), chunkID, sink, Config{Ledgers: true, Txhash: true, Events: true}) + ings, err := buildColdIngesters( + coldDirsAt(coldDir, chunkID), chunkID, sink, Config{Ledgers: true, Txhash: true, Events: true}) require.NoError(t, err) service := NewColdService(ings, sink) defer func() { require.NoError(t, service.Close()) }() @@ -833,7 +834,8 @@ func TestWriteColdChunk_RoundTrip(t *testing.T) { sink := &testSink{} require.NoError(t, WriteColdChunk( - context.Background(), logger, chunkID, rawChunk(stream, chunkID), coldDirsAt(coldDir, chunkID), sink, Config{Ledgers: true}, + context.Background(), logger, chunkID, rawChunk(stream, chunkID), + coldDirsAt(coldDir, chunkID), sink, Config{Ledgers: true}, )) path := packPath(filepath.Join(coldDir, "ledgers"), chunkID) @@ -862,7 +864,8 @@ func TestWriteColdChunk_ShortStream_NoArtifact(t *testing.T) { short := &fakeStream{t: t, count: 3} err := WriteColdChunk( - context.Background(), logger, chunkID, rawChunk(short, chunkID), coldDirsAt(coldDir, chunkID), nil, Config{Ledgers: true}, + context.Background(), logger, chunkID, rawChunk(short, chunkID), + coldDirsAt(coldDir, chunkID), nil, Config{Ledgers: true}, ) require.Error(t, err) require.Contains(t, err.Error(), "ended at") @@ -961,7 +964,8 @@ func TestWriteColdChunk_OutOfOrderSeq_NoArtifact(t *testing.T) { stream := &seqStream{t: t, seqs: seqs} err := WriteColdChunk( - context.Background(), logger, chunkID, rawChunk(stream, chunkID), coldDirsAt(coldDir, chunkID), nil, Config{Ledgers: true}, + context.Background(), logger, chunkID, rawChunk(stream, chunkID), + coldDirsAt(coldDir, chunkID), nil, Config{Ledgers: true}, ) require.Error(t, err) require.Contains(t, err.Error(), "yielded ledger") @@ -1020,7 +1024,8 @@ func TestWriteColdChunk_DrainStreamError_NoArtifact(t *testing.T) { stream := &errAtSeqStream{t: t, errAtSeq: failAt, err: wantErr} err := WriteColdChunk( - context.Background(), logger, chunkID, rawChunk(stream, chunkID), coldDirsAt(coldDir, chunkID), nil, Config{Ledgers: true}, + context.Background(), logger, chunkID, rawChunk(stream, chunkID), + coldDirsAt(coldDir, chunkID), nil, Config{Ledgers: true}, ) require.Error(t, err) require.ErrorIs(t, err, wantErr, "the backend error must propagate") diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index 856fcfdbd..d0016b35e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -97,7 +97,8 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu freezeStart := time.Now() start := ChunkIDOfLedger(floor) if start >= 0 && start <= int64(lastChunk) { - if eerr := backfill.RunBackfill(ctx, cfg.ExecConfig, chunk.ID(start), lastChunk); eerr != nil { //nolint:gosec // start in [0, lastChunk] + startChunk := chunk.ID(start) //nolint:gosec // start in [0, lastChunk] + if eerr := backfill.RunBackfill(ctx, cfg.ExecConfig, startChunk, lastChunk); eerr != nil { return fmt.Errorf("run backfill [%d,%s]: %w", start, lastChunk, eerr) } } else { diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go index 705951e32..39eac5642 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go @@ -129,7 +129,7 @@ func lastCompleteChunkAtID(ledger uint32) (chunk.ID, bool) { if c < 0 { return 0, false } - return chunk.ID(c), true //nolint:gosec // c >= 0 + return chunk.ID(c), true } // runTickForCatalog runs one lifecycle tick the way ingestion would drive it: it diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index 0cc735072..6c4852666 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -166,11 +166,6 @@ func (h *HotStore) Offsets() (*events.LedgerOffsets, error) { return h.offsets.View(), nil } -// index returns the in-memory term mirror. Test-only write hook: no production -// path reads it. Kept unexported until #772 decides whether the v2 read path -// hooks into it. -func (h *HotStore) index() *events.ConcurrentBitmaps { return h.mirror } - // Lookup returns the bitmap of event IDs in this Chunk that match // the given term. The returned bitmap is an immutable snapshot of // the live mirror — writers publish new pointers via atomic.Store @@ -472,6 +467,11 @@ func (h *HotStore) IngestLedgerToBatch( return func() { h.applyLedger(startID, termKeys) }, nil } +// index returns the in-memory term mirror. Test-only write hook: no production +// path reads it. Kept unexported until #772 decides whether the v2 read path +// hooks into it. +func (h *HotStore) index() *events.ConcurrentBitmaps { return h.mirror } + // applyLedger updates the mirror + offsets for a ledger whose rows are durable. // Infallible by construction (IngestLedgerToBatch validated seq under the // single-writer contract); the only non-completion is a crash, after which warmup diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index b38b7a844..2903435d0 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -116,7 +116,7 @@ func run(ctx context.Context, cfg StartConfig) error { // Ingestion and the lifecycle run as a joined pair under errgroup.WithContext: // gctx cancels as soon as EITHER returns — and WithContext records the returning - // goroutine's error BEFORE cancelling, so g.Wait surfaces the real cause, not the + // goroutine's error BEFORE canceling, so g.Wait surfaces the real cause, not the // sibling's induced context-canceled. g.Wait joins both before run returns, // restoring the single-lifecycle-goroutine invariant across supervisor restarts. // supervise is the one clean-vs-restart decision point; a canceled parent ctx From 10a7cafde3b02c56da0cd306f4292dfbe2dc7a59 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Jul 2026 22:05:23 -0400 Subject: [PATCH 48/55] fullhistory: fix 3 follow-on golangci findings (unparam + unused nolint:gosec) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The index() move and coldDirsAt signature change pulled these lines into the --new-from-rev window on the previous fix's run: - unparam: coldDirsAt's chunk param always receives chunk 0 — add the same //nolint:unparam the sibling packPath/txhashBinPath helpers already carry. - nolintlint x2: the //nolint:gosec on the two eventID := startID + uint32(i) range-index conversions in hot_store.go are unused (this gosec config doesn't flag range-loop indices) — remove them. No behavior change (comment-only in hot_store; test-helper directive). Build+vet+ gofmt clean; ingest/eventstore -short green. --- cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go | 2 ++ .../internal/fullhistory/pkg/stores/eventstore/hot_store.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go index 89ec01b53..080c326b1 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go @@ -193,6 +193,8 @@ func packPath(ledgersRoot string, c chunk.ID) string { // coldDirsAt resolves chunk c's three cold-artifact paths under one dir's per-type // roots — mirroring what geometry.Layout derives in production, so the readback // helpers (packPath/txhashBinPath) find what the ingesters wrote. +// +//nolint:unparam // chunk-general helper; every current caller uses chunk 0 func coldDirsAt(dir string, c chunk.ID) ColdDirs { return ColdDirs{ LedgerPack: packPath(filepath.Join(dir, dataTypeLedgers), c), diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go index 6c4852666..ea905d0a3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore/hot_store.go @@ -455,7 +455,7 @@ func (h *HotStore) IngestLedgerToBatch( return nil, fmt.Errorf("marshal payload %d for ledger %d: %w", i, ledgerSeq, err) } scratch = blob - eventID := startID + uint32(i) //nolint:gosec // i < len(payloads), overflow-guarded above + eventID := startID + uint32(i) b.Put(DataCF, encodeDataKey(eventID), blob) for _, key := range termKeys[i] { b.Put(IndexCF, encodeIndexKey(key, eventID), nil) @@ -487,7 +487,7 @@ func (h *HotStore) applyLedger(startID uint32, termKeys [][]events.TermKey) { // × unique-terms per ledger; the map grows past that. perKeyIDs := make(map[events.TermKey][]uint32, 64) for i, keys := range termKeys { - eventID := startID + uint32(i) //nolint:gosec // i < len(termKeys), overflow-guarded in IngestLedgerToBatch + eventID := startID + uint32(i) for _, key := range keys { perKeyIDs[key] = append(perKeyIDs[key], eventID) } From d2b1c127d832b5951aaa209a498ca35081f7ef36 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Fri, 3 Jul 2026 00:16:12 -0400 Subject: [PATCH 49/55] fullhistory: lifecycle tick cleanups + gauge correctness (round-4 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - last_committed_ledger gauge: split Metrics.LastCommitted(lc, floor) into LastCommitted(lc) + RetentionFloor(floor). The ingestion loop now owns the last-committed gauge (per-ledger, the true possibly-mid-chunk value); the tick owns only the floor. Fixes the tick regressing the gauge below the refined watermark on every mid-chunk restart (thread 3517302906). - Tick: compute earliest + retention gate ONCE and pass into the discard/prune scans (was re-derived per scan); pendingArtifacts takes the caller's coverage (no double read); add RetentionFloor.FirstChunk() as the one floor->chunk boundary shared by plan/prune/backfill; surface the LiveHotChunks scan error instead of swallowing it; delete the empty-range Freeze emission + dead timer (threads 3511860199, 3517303403). - LastCommittedLedger: delete the nil-logger positional mode (zero prod callers after the clamp deletion); logger is required; delete the three `if logger != nil` tick guards (thread 3517302730). Positional-term unit tests seed real empty hot DBs. - Move the signed pre-genesis chunk arithmetic (CompleteThrough, ChunkIDOfLedger, PreGenesisLedger) into geometry alongside LastCompleteChunkAt/ChunkFirstLedger — one -1 convention, one home; startup's mid-chunk exclusion now reads geometry.LastCompleteChunkAt directly (thread 3517303377). - observability_test: cover the previously-untested collectors (live_hot_chunks, chunk_boundaries_total, discarded_hot_chunks_total, discard phase) (3517303282). --- .../fullhistory/backfill/recorder_test.go | 11 +-- .../fullhistory/geometry/chunk_arith.go | 46 +++++++++++ .../fullhistory/geometry/chunk_arith_test.go | 57 ++++++++++++++ .../fullhistory/geometry/txhash_index.go | 16 ---- .../internal/fullhistory/helpers_test.go | 8 +- .../internal/fullhistory/hotloop.go | 4 + .../fullhistory/lifecycle/eligibility.go | 46 ++++------- .../fullhistory/lifecycle/lifecycle.go | 52 ++++++------- .../lifecycle/lifecycle_helpers_test.go | 21 +++-- .../fullhistory/lifecycle/lifecycle_test.go | 6 +- .../fullhistory/lifecycle/progress.go | 55 ++++--------- .../lifecycle/progress_realdb_test.go | 3 +- .../lifecycle/progress_shim_test.go | 20 ++--- .../fullhistory/lifecycle/progress_test.go | 78 +++++-------------- .../fullhistory/lifecycle/retention.go | 14 +++- .../fullhistory/lifecycle/retention_test.go | 2 +- .../observability/observability.go | 36 ++++++--- .../observability/observability_test.go | 20 ++++- .../internal/fullhistory/startup.go | 17 ++-- 19 files changed, 286 insertions(+), 226 deletions(-) create mode 100644 cmd/stellar-rpc/internal/fullhistory/geometry/chunk_arith.go create mode 100644 cmd/stellar-rpc/internal/fullhistory/geometry/chunk_arith_test.go diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go b/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go index bb849fbfd..74261ef90 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/recorder_test.go @@ -41,10 +41,11 @@ func (r *recordingMetrics) Prune(count int, d time.Duration) { r.prune = append(r.prune, pruneRec{count, d}) } -func (*recordingMetrics) LastCommitted(uint32, uint32) {} -func (*recordingMetrics) ChunkBoundary() {} -func (*recordingMetrics) BackfillPass(time.Duration) {} -func (*recordingMetrics) LiveHotChunks(int) {} -func (*recordingMetrics) Discard(int, time.Duration) {} +func (*recordingMetrics) LastCommitted(uint32) {} +func (*recordingMetrics) RetentionFloor(uint32) {} +func (*recordingMetrics) ChunkBoundary() {} +func (*recordingMetrics) BackfillPass(time.Duration) {} +func (*recordingMetrics) LiveHotChunks(int) {} +func (*recordingMetrics) Discard(int, time.Duration) {} var _ observability.Metrics = (*recordingMetrics)(nil) diff --git a/cmd/stellar-rpc/internal/fullhistory/geometry/chunk_arith.go b/cmd/stellar-rpc/internal/fullhistory/geometry/chunk_arith.go new file mode 100644 index 000000000..cb7437de1 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/geometry/chunk_arith.go @@ -0,0 +1,46 @@ +package geometry + +import "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" + +// Signed pre-genesis chunk arithmetic — the single home for the chunk↔ledger maps +// that run in int64 so the pre-genesis sentinel (-1 = "nothing complete") never +// underflows the uint32 domain. Keeping all of it here (rather than split across +// lifecycle progress and this package) means there is one -1 convention, not two. + +// PreGenesisLedger is the last-committed ledger when nothing is complete +// (FirstLedgerSeq-1) — the ledger-domain image of the -1 chunk sentinel. +const PreGenesisLedger uint32 = chunk.FirstLedgerSeq - 1 + +// CompleteThrough maps a signed chunk index to its "complete through" last ledger: +// c < 0 ⇒ PreGenesisLedger; c >= 0 ⇒ chunk.ID(c).LastLedger(). +func CompleteThrough(c int64) uint32 { + if c < 0 { + return PreGenesisLedger + } + return chunk.ID(c).LastLedger() //nolint:gosec // c >= 0 and bounded by real chunk ids +} + +// ChunkIDOfLedger maps a ledger to its chunk, signed so a sub-genesis ledger +// yields -1 instead of panicking. +func ChunkIDOfLedger(ledger uint32) int64 { + if ledger < chunk.FirstLedgerSeq { + return -1 + } + return int64(chunk.IDFromLedger(ledger)) +} + +// LastCompleteChunkAt is the inverse of chunk.ID.LastLedger: the largest chunk +// whose last ledger is <= ledger. Returns SIGNED int64 so a sub-genesis ledger +// (the sub-genesis sentinel) maps to -1 ("before the first chunk") rather than +// wrapping; the cast-before-subtract keeps it in int64 (uint32 ledger-1 would +// underflow for ledger 0). +func LastCompleteChunkAt(ledger uint32) int64 { + return (int64(ledger)+1-int64(chunk.FirstLedgerSeq))/int64(chunk.LedgersPerChunk) - 1 +} + +// ChunkFirstLedger maps a non-negative signed chunk index to its first ledger. +// It is the signed-domain companion of chunk.ID.FirstLedger used after a +// max(..., 0) clamp. +func ChunkFirstLedger(c int64) uint32 { + return chunk.ID(c).FirstLedger() //nolint:gosec // c >= 0 (clamped) and bounded by real chunk ids +} diff --git a/cmd/stellar-rpc/internal/fullhistory/geometry/chunk_arith_test.go b/cmd/stellar-rpc/internal/fullhistory/geometry/chunk_arith_test.go new file mode 100644 index 000000000..e784494f9 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/geometry/chunk_arith_test.go @@ -0,0 +1,57 @@ +package geometry + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" +) + +// --------------------------------------------------------------------------- +// CompleteThrough — sentinel-safe signed->ledger map. +// +// ALIASING TRAP: a guard-less impl wraps -1 to exactly PreGenesisLedger anyway +// (MaxUint32+1 overflows to 0), so a -1-only test is blind to a dropped guard. +// The -2/-100 rows are the load-bearing ones (they wrap to large, distinct values +// the guard must squash). +// --------------------------------------------------------------------------- + +func TestCompleteThrough(t *testing.T) { + tests := []struct { + name string + in int64 + want uint32 + }{ + {"pre-genesis sentinel -1 => FirstLedgerSeq-1, not MaxUint32 (aliases the wrap)", -1, PreGenesisLedger}, + {"sentinel -2 does NOT alias the wrap (guard-less would yield 4294957297)", -2, PreGenesisLedger}, + {"deeply negative still pre-genesis", -100, PreGenesisLedger}, + {"chunk 0 last ledger", 0, chunk.ID(0).LastLedger()}, + {"chunk 5 last ledger", 5, chunk.ID(5).LastLedger()}, + } + require.Equal(t, uint32(1), PreGenesisLedger, "FirstLedgerSeq-1 == 1 (the doc's chunkLastLedger(-1))") + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, CompleteThrough(tc.in)) + }) + } + + // Assert the aliasing trap directly so the comment above can't rot: -1 wraps to + // PreGenesisLedger, -2 does not. Computed from chunk arithmetic, not hardcoded. + guardlessWrap := func(c int64) uint32 { + return chunk.ID(uint32(c)).LastLedger() + } + require.Equal(t, PreGenesisLedger, guardlessWrap(-1), + "-1 aliases PreGenesisLedger under the wrap — the coincidence this test must not rely on") + require.NotEqual(t, PreGenesisLedger, guardlessWrap(-2), + "-2 must NOT alias — proving the guard (not a coincidence) is what makes CompleteThrough(-2) safe") +} + +// ChunkIDOfLedger maps a ledger to its containing chunk, signed so a sub-genesis +// ledger yields -1 rather than panicking. +func TestChunkIDOfLedger(t *testing.T) { + require.Equal(t, int64(-1), ChunkIDOfLedger(chunk.FirstLedgerSeq-1), "sub-genesis => -1 sentinel") + require.Equal(t, int64(0), ChunkIDOfLedger(chunk.FirstLedgerSeq), "genesis => chunk 0") + require.Equal(t, int64(0), ChunkIDOfLedger(chunk.ID(0).LastLedger()), "chunk 0's last ledger => chunk 0") + require.Equal(t, int64(1), ChunkIDOfLedger(chunk.ID(1).FirstLedger()), "chunk 1's first ledger => chunk 1") +} diff --git a/cmd/stellar-rpc/internal/fullhistory/geometry/txhash_index.go b/cmd/stellar-rpc/internal/fullhistory/geometry/txhash_index.go index 14f7a99f0..b63164925 100644 --- a/cmd/stellar-rpc/internal/fullhistory/geometry/txhash_index.go +++ b/cmd/stellar-rpc/internal/fullhistory/geometry/txhash_index.go @@ -80,19 +80,3 @@ func (l TxHashIndexLayout) LastChunk(id TxHashIndexID) chunk.ID { func (l TxHashIndexLayout) IsTerminalCoverage(cov TxHashIndexCoverage) bool { return cov.Hi == l.LastChunk(cov.Index) } - -// LastCompleteChunkAt is the inverse of chunk.ID.LastLedger: the largest chunk -// whose last ledger is <= ledger. Returns SIGNED int64 so a sub-genesis ledger -// (the sub-genesis sentinel) maps to -1 ("before the first chunk") rather than -// wrapping; the cast-before-subtract keeps it in int64 (uint32 ledger-1 would -// underflow for ledger 0). -func LastCompleteChunkAt(ledger uint32) int64 { - return (int64(ledger)+1-int64(chunk.FirstLedgerSeq))/int64(chunk.LedgersPerChunk) - 1 -} - -// ChunkFirstLedger maps a non-negative signed chunk index to its first ledger. -// It is the signed-domain companion of chunk.ID.FirstLedger used by -// retentionFloorChunk after the max(..., 0) clamp. -func ChunkFirstLedger(c int64) uint32 { - return chunk.ID(c).FirstLedger() //nolint:gosec // c >= 0 (clamped) and bounded by real chunk ids -} diff --git a/cmd/stellar-rpc/internal/fullhistory/helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/helpers_test.go index 3627a9e6a..7d6b3da31 100644 --- a/cmd/stellar-rpc/internal/fullhistory/helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/helpers_test.go @@ -93,12 +93,18 @@ func newRecordingMetrics() *recordingMetrics { return &recordingMetrics{gaugesSet: map[string]int{}} } -func (r *recordingMetrics) LastCommitted(uint32, uint32) { +func (r *recordingMetrics) LastCommitted(uint32) { r.mu.Lock() defer r.mu.Unlock() r.gaugesSet["last_committed"]++ } +func (r *recordingMetrics) RetentionFloor(uint32) { + r.mu.Lock() + defer r.mu.Unlock() + r.gaugesSet["retention_floor"]++ +} + func (r *recordingMetrics) BackfillPass(time.Duration) { r.mu.Lock() defer r.mu.Unlock() diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index 2496cbd35..d778c90ac 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -164,6 +164,10 @@ func runIngestionLoop(ctx context.Context, cfg ingestionLoopConfig) (err error) if ierr := hotService.Ingest(ctx, vl.Seq, vl.View); ierr != nil { return fmt.Errorf("ingest ledger %d: %w", vl.Seq, ierr) } + // The ingestion loop owns the last-committed gauge: this is the TRUE + // committed ledger (mid-chunk included), one atomic gauge set per ledger. + // The tick must not touch it — its chunk-aligned value would regress it. + metrics.LastCommitted(vl.Seq) // Chunk boundary: this seq is the chunk's last ledger. closed := chunk.IDFromLedger(vl.Seq) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go index 8ef4ef987..4db9c212c 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/eligibility.go @@ -16,16 +16,7 @@ import ( // otherwise (live, or frozen awaiting coverage) → leave alone. // catalog.DiscardHotChunk is idempotent, so a crash between freeze and discard // self-heals next tick. -func eligibleDiscardOps(cfg Config, cat *catalog.Catalog, through uint32) ([]func() error, error) { - earliest, _, err := cat.EarliestLedger() - if err != nil { - return nil, err - } - // The "past retention" test shares one definition with the read gate - // (retention.go), so a hot DB retires on exactly the floor the reader stops - // admitting at. A shortened retentionChunks raises the floor at once. - gate := NewRetentionFloor(through, cfg.RetentionChunks, earliest) - +func eligibleDiscardOps(cat *catalog.Catalog, gate RetentionFloor, through uint32) ([]func() error, error) { hot, err := cat.HotChunkKeys() if err != nil { return nil, err @@ -38,14 +29,17 @@ func eligibleDiscardOps(cfg Config, cat *catalog.Catalog, through uint32) ([]fun case gate.Excludes(c): ops = append(ops, func() error { return cat.DiscardHotChunk(c) }) case last <= through: - pending, perr := pendingArtifacts(c, cat) - if perr != nil { - return nil, perr - } + // Coverage is read once here and passed into pendingArtifacts — the + // discard requires covers independently, so the whole predicate is + // ledgers-frozen && events-frozen && covers. covers, cerr := cat.FrozenIndexCovers(c) if cerr != nil { return nil, cerr } + pending, perr := pendingArtifacts(c, cat, covers) + if perr != nil { + return nil, perr + } if pending.Empty() && covers { ops = append(ops, func() error { return cat.DiscardHotChunk(c) }) } @@ -58,9 +52,9 @@ func eligibleDiscardOps(cfg Config, cat *catalog.Catalog, through uint32) ([]fun // pendingArtifacts lists which outputs chunk still needs: ledgers and events must // be frozen; txhash/.bin is exempt when the window's index already covers the -// chunk (after finalization the chunk:c:txhash key is demoted/swept, so -// regenerating the .bin would orphan it). -func pendingArtifacts(c chunk.ID, cat *catalog.Catalog) (catalog.ArtifactSet, error) { +// chunk (covers, computed by the caller — after finalization the chunk:c:txhash +// key is demoted/swept, so regenerating the .bin would orphan it). +func pendingArtifacts(c chunk.ID, cat *catalog.Catalog, covers bool) (catalog.ArtifactSet, error) { var need catalog.ArtifactSet for _, kind := range []geometry.Kind{geometry.KindLedgers, geometry.KindEvents} { state, err := cat.State(c, kind) @@ -75,14 +69,8 @@ func pendingArtifacts(c chunk.ID, cat *catalog.Catalog) (catalog.ArtifactSet, er if err != nil { return need, err } - if txState != geometry.StateFrozen { - covers, cerr := cat.FrozenIndexCovers(c) - if cerr != nil { - return need, cerr - } - if !covers { - need = need.Add(geometry.KindTxHash) - } + if txState != geometry.StateFrozen && !covers { + need = need.Add(geometry.KindTxHash) } return need, nil } @@ -96,13 +84,7 @@ func pendingArtifacts(c chunk.ID, cat *catalog.Catalog) (catalog.ArtifactSet, er // index-key op plus every ref in the single batched chunk sweep), so the caller // meters Prune in artifacts — the same unit the Phase 1 sweep reports — rather // than in op closures (the chunk family collapses N artifacts into one op). -func eligiblePruneOps(cfg Config, cat *catalog.Catalog, through uint32) ([]func() error, int, error) { - earliest, _, err := cat.EarliestLedger() - if err != nil { - return nil, 0, err - } - gate := NewRetentionFloor(through, cfg.RetentionChunks, earliest) - +func eligiblePruneOps(cat *catalog.Catalog, gate RetentionFloor) ([]func() error, int, error) { var ops []func() error artifacts := 0 diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index d0016b35e..c0d42ef81 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -70,22 +70,24 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu metrics := observability.MetricsOrNop(cfg.Metrics) logger := cfg.Logger - // The one snapshot every stage shares. + // The one snapshot every stage shares. earliest and the retention gate are read + // and computed ONCE here (not re-derived per scan), then passed to both scans. through := lastChunk.LastLedger() earliest, _, err := cat.EarliestLedger() if err != nil { return fmt.Errorf("read earliest ledger: %w", err) } - floor := EffectiveRetentionFloor(through, cfg.RetentionChunks, earliest) - - // Progress gauges: derived last-committed ledger and effective retention floor. - metrics.LastCommitted(through, floor) - if logger != nil { - logger.WithField("through", through). - WithField("floor", floor). - Debug("streaming: lifecycle tick — derived snapshot") - } + floorLedger := EffectiveRetentionFloor(through, cfg.RetentionChunks, earliest) + gate := RetentionFloorAt(floorLedger) + + // Retention-floor gauge only. The last-committed gauge is owned by the ingestion + // loop (which holds the true, possibly mid-chunk value); re-emitting it here from + // the chunk-aligned `through` would regress it on every tick. + metrics.RetentionFloor(floorLedger) + logger.WithField("through", through). + WithField("floor_chunk", gate.FirstChunk().String()). + Debug("streaming: lifecycle tick — derived snapshot") // Stage 1 — plan-and-execute (freeze + index fold) over [floor, lastChunk], via // the same entry point backfill uses (resolve → executePlan → Freeze metric, @@ -93,23 +95,17 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu // propagates up for supervise to treat as a clean shutdown. lastChunk is always // a completed chunk (boundary fence + post-backfill seed), so the only guard // needed is the empty-range check (floor above lastChunk when retention outran - // production). - freezeStart := time.Now() - start := ChunkIDOfLedger(floor) - if start >= 0 && start <= int64(lastChunk) { - startChunk := chunk.ID(start) //nolint:gosec // start in [0, lastChunk] - if eerr := backfill.RunBackfill(ctx, cfg.ExecConfig, startChunk, lastChunk); eerr != nil { - return fmt.Errorf("run backfill [%d,%s]: %w", start, lastChunk, eerr) + // production). An empty range emits no Freeze sample — the Discard/Prune samples + // below carry empty-tick visibility. + if start := gate.FirstChunk(); start <= lastChunk { + if eerr := backfill.RunBackfill(ctx, cfg.ExecConfig, start, lastChunk); eerr != nil { + return fmt.Errorf("run backfill [%s,%s]: %w", start, lastChunk, eerr) } - } else { - // floor above lastChunk: nothing to produce, but report an empty freeze so - // the empty-tick rate stays visible. Scans below still run. - metrics.Freeze(time.Since(freezeStart)) } // Stage 2 — discard scan. discardStart := time.Now() - discardOps, err := eligibleDiscardOps(cfg, cat, through) + discardOps, err := eligibleDiscardOps(cat, gate, through) if err != nil { return fmt.Errorf("eligible discard ops: %w", err) } @@ -117,18 +113,20 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu return fmt.Errorf("discard op: %w", err) } metrics.Discard(len(discardOps), time.Since(discardStart)) - if logger != nil && len(discardOps) > 0 { + if len(discardOps) > 0 { logger.WithField("discarded", len(discardOps)).Info("streaming: lifecycle discard stage complete") } // Live hot-chunk gauge after the discard stage. - if hot, herr := cat.HotChunkKeys(); herr == nil { - metrics.LiveHotChunks(len(hot)) + hot, err := cat.HotChunkKeys() + if err != nil { + return fmt.Errorf("read hot chunk keys: %w", err) } + metrics.LiveHotChunks(len(hot)) // Stage 3 — prune scan. pruneStart := time.Now() - pruneOps, prunedArtifacts, err := eligiblePruneOps(cfg, cat, through) + pruneOps, prunedArtifacts, err := eligiblePruneOps(cat, gate) if err != nil { return fmt.Errorf("eligible prune ops: %w", err) } @@ -136,7 +134,7 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu return fmt.Errorf("prune op: %w", err) } metrics.Prune(prunedArtifacts, time.Since(pruneStart)) - if logger != nil && prunedArtifacts > 0 { + if prunedArtifacts > 0 { logger.WithField("pruned", prunedArtifacts).Info("streaming: lifecycle prune stage complete") } return nil diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go index 39eac5642..13059bb14 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go @@ -159,25 +159,34 @@ func makeReadyHotDirNoData(t *testing.T, cat *catalog.Catalog, c chunk.ID) { require.NoError(t, db.Close()) } +// gateFor builds the retention gate the tick passes into the eligibility scans, +// from the same (through, retention, earliest) snapshot runLifecycle uses. +func gateFor(t *testing.T, cfg Config, cat *catalog.Catalog, through uint32) RetentionFloor { + t.Helper() + earliest, _, err := cat.EarliestLedger() + require.NoError(t, err) + return NewRetentionFloor(through, cfg.RetentionChunks, earliest) +} + // assertQuiescent re-runs the tick's three derivations against the SAME through // snapshot and asserts none schedule work — the quiescence postcondition. func assertQuiescent(t *testing.T, cfg Config, cat *catalog.Catalog, through uint32) { t.Helper() earliest, _, err := cat.EarliestLedger() require.NoError(t, err) - floor := EffectiveRetentionFloor(through, cfg.RetentionChunks, earliest) - start := ChunkIDOfLedger(floor) - if rangeEnd, ok := lastCompleteChunkAtID(through); ok && start >= 0 && start <= int64(rangeEnd) { + gate := NewRetentionFloor(through, cfg.RetentionChunks, earliest) + start := gate.FirstChunk() + if rangeEnd, ok := lastCompleteChunkAtID(through); ok && start <= rangeEnd { // At quiescence resolve finds an empty plan, so RunBackfill (resolve + // executePlan) is a no-op that returns nil — even with no Backend wired, // since an empty plan never reaches backfillSource. - perr := backfill.RunBackfill(context.Background(), cfg.ExecConfig, chunk.ID(start), rangeEnd) + perr := backfill.RunBackfill(context.Background(), cfg.ExecConfig, start, rangeEnd) assert.NoError(t, perr, "re-running backfill schedules no work at quiescence") } - dops, err := eligibleDiscardOps(cfg, cat, through) + dops, err := eligibleDiscardOps(cat, gate, through) require.NoError(t, err) assert.Empty(t, dops, "re-scan finds no discard work at quiescence") - pops, _, err := eligiblePruneOps(cfg, cat, through) + pops, _, err := eligiblePruneOps(cat, gate) require.NoError(t, err) assert.Empty(t, pops, "re-scan finds no prune work at quiescence") } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go index 1e3f0679d..3d3398d11 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_test.go @@ -93,7 +93,7 @@ func TestRunLifecycleTick_DiscardGatedOnIndexCoverage(t *testing.T) { // txhash is frozen, ledgers/events frozen, but the window has no FROZEN coverage // yet => indexCovers(0) is false => NOT discarded (still needed for lookups via // its .bin/hot DB until the index folds it in). - ops, err := eligibleDiscardOps(cfg, cat, through) + ops, err := eligibleDiscardOps(cat, gateFor(t, cfg, cat, through), through) require.NoError(t, err) require.Empty(t, ops, "no index coverage yet: the hot DB stays") @@ -104,7 +104,7 @@ func TestRunLifecycleTick_DiscardGatedOnIndexCoverage(t *testing.T) { require.NoError(t, err) require.True(t, covered) - ops, err = eligibleDiscardOps(cfg, cat, through) + ops, err = eligibleDiscardOps(cat, gateFor(t, cfg, cat, through), through) require.NoError(t, err) require.Len(t, ops, 1, "covered + nothing pending => discard eligible") require.NoError(t, ops[0]()) @@ -173,7 +173,7 @@ func TestRunLifecycleTick_PrunesTransientIndexDebris(t *testing.T) { through, err := deriveCompleteThrough(cat) require.NoError(t, err) - ops, artifacts, err := eligiblePruneOps(cfg, cat, through) + ops, artifacts, err := eligiblePruneOps(cat, gateFor(t, cfg, cat, through)) require.NoError(t, err) require.Len(t, ops, 1, "the freezing debris is swept") require.Equal(t, 1, artifacts, "one index artifact swept") diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go index 04595aad5..13534bfea 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go @@ -13,19 +13,8 @@ import ( // Progress is derived, never stored. "Highest complete chunk" arithmetic runs in // int64 (-1 = "nothing complete") to avoid uint32 wraparound on the pre-genesis -// sentinel; CompleteThrough is the chokepoint. - -// preGenesisLedger is the last-committed ledger when nothing is complete (FirstLedgerSeq-1). -const preGenesisLedger uint32 = chunk.FirstLedgerSeq - 1 - -// CompleteThrough maps a signed chunk index to its "complete through" last ledger: -// c < 0 ⇒ preGenesisLedger; c >= 0 ⇒ chunk.ID(c).LastLedger(). -func CompleteThrough(c int64) uint32 { - if c < 0 { - return preGenesisLedger - } - return chunk.ID(c).LastLedger() //nolint:gosec // c >= 0 and bounded by real chunk ids -} +// sentinel; geometry.CompleteThrough is the chokepoint (the signed chunk↔ledger +// maps live in geometry so there is one -1 convention across the daemon). // LastCommittedLedger is the single highest-durably-committed-ledger derivation. // It maxes three terms, each in the signed domain so a fresh/young store never @@ -33,36 +22,33 @@ func CompleteThrough(c int64) uint32 { // // - COLD — highest chunk with all artifacts durable (highestDurableChunk; -1 on // a fresh start). Leads at startup before any hot key exists. -// - HOT — only when hot > cold, only over "ready" keys. logger == nil gives the -// positional term CompleteThrough(hot-1); logger != nil refines with one -// read-only MaxCommittedSeq read (safe: derivation runs before ingestion locks -// the DB). +// - HOT — only when hot > cold, over "ready" keys: one read-only MaxCommittedSeq +// read of the highest ready hot DB (empty DB ⇒ positional CompleteThrough(hot-1)). +// Safe: derivation runs before ingestion locks the DB. // - FLOOR — EarliestLedger()-1 as int64(earliest)-1, so an absent/zero pin // yields the pre-genesis sentinel rather than underflowing. +// +// logger is required (hotchunk.OpenReadOnly needs it); there is no logger-less +// mode — the tick derives the frontier the same way startup does. func LastCommittedLedger(cat *catalog.Catalog, logger *supportlog.Entry) (uint32, error) { cold, err := highestDurableChunk(cat) if err != nil { return 0, err } - through := CompleteThrough(cold) + through := geometry.CompleteThrough(cold) hot, err := highestReadyChunkSigned(cat) if err != nil { return 0, err } if hot > cold { - if logger == nil { - // Positional term: everything below the live (highest ready) chunk. - through = max(through, CompleteThrough(hot-1)) - } else { - // One refinement read of the highest ready hot DB; loss detected lazily - // on this open (no eager scan over every ready key). - refined, rerr := refineWithHotDB(cat, logger, hot) - if rerr != nil { - return 0, rerr - } - through = max(through, refined) + // One refinement read of the highest ready hot DB; loss detected lazily on + // this open (no eager scan over every ready key). + refined, rerr := refineWithHotDB(cat, logger, hot) + if rerr != nil { + return 0, rerr } + through = max(through, refined) } earliest, ok, err := cat.EarliestLedger() @@ -100,7 +86,7 @@ func refineWithHotDB(cat *catalog.Catalog, logger *supportlog.Entry, live int64) return maxSeq, nil } // Empty live DB: positional fallback (everything below it). - return CompleteThrough(live - 1), nil + return geometry.CompleteThrough(live - 1), nil } // highestReadyChunkSigned returns the highest "ready" hot chunk id as int64, or -1 @@ -174,12 +160,3 @@ func highestDurableChunk(cat *catalog.Catalog) (int64, error) { } return highest, nil } - -// ChunkIDOfLedger maps a ledger to its chunk, signed so a sub-genesis ledger -// yields -1 instead of panicking. -func ChunkIDOfLedger(ledger uint32) int64 { - if ledger < chunk.FirstLedgerSeq { - return -1 - } - return int64(chunk.IDFromLedger(ledger)) -} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go index 6da718c8f..a01633f2d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" @@ -79,7 +80,7 @@ func TestDeriveWatermark_RealHotDB_RefinementIsNotStale(t *testing.T) { // Sanity: positional baseline (live chunk 5 ⇒ everything below 5) is chunk 4's // last ledger, strictly below the committed top — so the assertion below can // only pass if the refinement actually read the real DB. - baseline := mustDeriveCompleteThrough(t, cat) + baseline := geometry.CompleteThrough(int64(live) - 1) require.Equal(t, chunk.ID(4).LastLedger(), baseline) require.Greater(t, committedTop, baseline, "fixture must put the real frontier above the baseline") diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_shim_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_shim_test.go index c62afc007..271b0c282 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_shim_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_shim_test.go @@ -6,19 +6,15 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" ) -// Test-only aliases for the consolidated progress derivation. The design folded -// deriveCompleteThrough + deriveWatermark into ONE LastCommittedLedger(cat[, logger]): -// -// - deriveCompleteThrough(cat) == LastCommittedLedger(cat, nil) (chunk -// granularity, pure catalog read — the positional term, no hot DB open). -// - deriveWatermark(cat, logger) == LastCommittedLedger(cat, logger) (one -// read-only refinement of the highest ready hot DB opened by its Layout path, -// loss detected LAZILY on it). -// -// These shims keep the tests' intent legible; production callers use -// LastCommittedLedger directly. +// Test-only aliases for the single progress derivation, LastCommittedLedger. +// There is no logger-less mode: when a "ready" hot key leads the cold term the +// derivation always opens that DB read-only, so both aliases pass a real logger. +// deriveCompleteThrough names the cold/floor/positional-selection intent (its +// callers seed no ready-above-cold hot key, or seed an empty real hot DB whose +// refinement falls back to the positional term); deriveWatermark names the +// refinement-value intent. Production callers use LastCommittedLedger directly. func deriveCompleteThrough(cat *catalog.Catalog) (uint32, error) { - return LastCommittedLedger(cat, nil) + return LastCommittedLedger(cat, silentLogger()) } func deriveWatermark(cat *catalog.Catalog, logger *supportlog.Entry) (uint32, error) { diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go index 9587fb428..c4bd5f50d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_test.go @@ -22,8 +22,10 @@ func makeChunkDurable(t *testing.T, cat *catalog.Catalog, c chunk.ID) { freezeKinds(t, cat, c, geometry.KindLedgers, geometry.KindEvents, geometry.KindTxHash) } -// makeHotDir creates the on-disk hot dir for a chunk so deriveWatermark's -// per-ready-key dir-existence loop sees it present. +// makeHotDir creates the on-disk hot dir for a chunk. The refinement opens only +// the HIGHEST ready chunk, so a lower ready key needs only its dir present, not a +// real DB (readyHot pairs this with the key); the highest ready chunk in a +// positional-term test needs a real empty DB via seedReadyLiveDB. func makeHotDir(t *testing.T, cat *catalog.Catalog, c chunk.ID) { t.Helper() require.NoError(t, os.MkdirAll(cat.Layout().HotChunkPath(c), 0o755)) @@ -38,47 +40,9 @@ func readyHot(t *testing.T, cat *catalog.Catalog, c chunk.ID) { makeHotDir(t, cat, c) } -// --------------------------------------------------------------------------- -// CompleteThrough — sentinel-safe signed->ledger map. -// -// ALIASING TRAP: a guard-less impl wraps -1 to exactly preGenesisLedger anyway -// (MaxUint32+1 overflows to 0), so a -1-only test is blind to a dropped guard. -// The -2/-100 rows are the load-bearing ones (they wrap to large, distinct values -// the guard must squash). -// --------------------------------------------------------------------------- - -func TestCompleteThrough(t *testing.T) { - tests := []struct { - name string - in int64 - want uint32 - }{ - {"pre-genesis sentinel -1 => FirstLedgerSeq-1, not MaxUint32 (aliases the wrap)", -1, preGenesisLedger}, - {"sentinel -2 does NOT alias the wrap (guard-less would yield 4294957297)", -2, preGenesisLedger}, - {"deeply negative still pre-genesis", -100, preGenesisLedger}, - {"chunk 0 last ledger", 0, chunk.ID(0).LastLedger()}, - {"chunk 5 last ledger", 5, chunk.ID(5).LastLedger()}, - } - require.Equal(t, uint32(1), preGenesisLedger, "FirstLedgerSeq-1 == 1 (the doc's chunkLastLedger(-1))") - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.want, CompleteThrough(tc.in)) - }) - } - - // Assert the aliasing trap directly so the comment above can't rot: -1 wraps to - // preGenesisLedger, -2 does not. Computed from chunk arithmetic, not hardcoded. - guardlessWrap := func(c int64) uint32 { - return chunk.ID(uint32(c)).LastLedger() - } - require.Equal(t, preGenesisLedger, guardlessWrap(-1), - "-1 aliases preGenesisLedger under the wrap — the coincidence this test must not rely on") - require.NotEqual(t, preGenesisLedger, guardlessWrap(-2), - "-2 must NOT alias — proving the guard (not a coincidence) is what makes CompleteThrough(-2) safe") -} - // --------------------------------------------------------------------------- // LastCommittedLedger — chunk-granularity bound, pure catalog read. +// (CompleteThrough / ChunkIDOfLedger arithmetic is tested in geometry.) // --------------------------------------------------------------------------- func TestLastCommittedLedger(t *testing.T) { @@ -87,7 +51,7 @@ func TestLastCommittedLedger(t *testing.T) { cat, _ := testCatalog(t) got, err := deriveCompleteThrough(cat) require.NoError(t, err) - require.Equal(t, preGenesisLedger, got) + require.Equal(t, geometry.PreGenesisLedger, got) }) t.Run("cold term leads: highest fully-durable chunk", func(t *testing.T) { @@ -135,10 +99,11 @@ func TestLastCommittedLedger(t *testing.T) { t.Run("positional term leads in steady state: everything below the live chunk", func(t *testing.T) { cat, _ := testCatalog(t) // No cold artifacts yet (steady state: chunks complete before cold exists). - // Ready hot keys 3,4,5 => live chunk is 5 => everything below 5 complete. + // Ready hot keys 3,4,5 => live chunk is 5 => everything below 5 complete. Only + // the highest (5) is opened; empty DB ⇒ positional fallback CompleteThrough(4). readyHot(t, cat, 3) readyHot(t, cat, 4) - readyHot(t, cat, 5) + seedReadyLiveDB(t, cat, 5, 0) got, err := deriveCompleteThrough(cat) require.NoError(t, err) require.Equal(t, chunk.ID(4).LastLedger(), got, "max ready (5) - 1 = chunk 4's last ledger") @@ -146,7 +111,7 @@ func TestLastCommittedLedger(t *testing.T) { t.Run("transient hot key does NOT advance the positional term", func(t *testing.T) { cat, _ := testCatalog(t) - readyHot(t, cat, 3) + seedReadyLiveDB(t, cat, 3, 0) // highest ready, empty DB ⇒ positional CompleteThrough(2) // A transient key above the highest ready one must be excluded. require.NoError(t, cat.PutHotTransient(9)) got, err := deriveCompleteThrough(cat) @@ -158,10 +123,10 @@ func TestLastCommittedLedger(t *testing.T) { // The exact uint32-underflow trap: max ready = 0, so 0-1 must be the // pre-genesis sentinel, not ID(4294967295).LastLedger(). cat, _ := testCatalog(t) - readyHot(t, cat, 0) + seedReadyLiveDB(t, cat, 0, 0) // ready chunk 0, empty DB ⇒ positional fallback got, err := deriveCompleteThrough(cat) require.NoError(t, err) - require.Equal(t, preGenesisLedger, got) + require.Equal(t, geometry.PreGenesisLedger, got) }) t.Run("earliest pin floor leads when above cold/positional terms", func(t *testing.T) { @@ -179,13 +144,13 @@ func TestLastCommittedLedger(t *testing.T) { require.NoError(t, cat.PinEarliestLedger(chunk.FirstLedgerSeq)) got, err := deriveCompleteThrough(cat) require.NoError(t, err) - require.Equal(t, preGenesisLedger, got, "earliest 2 - 1 = 1, not MaxUint32") + require.Equal(t, geometry.PreGenesisLedger, got, "earliest 2 - 1 = 1, not MaxUint32") }) t.Run("max of all three terms", func(t *testing.T) { cat, _ := testCatalog(t) - makeChunkDurable(t, cat, 0) // cold => chunk 0 last ledger - readyHot(t, cat, 4) // positional => chunk 3 last ledger (highest) + makeChunkDurable(t, cat, 0) // cold => chunk 0 last ledger + seedReadyLiveDB(t, cat, 4, 0) // positional (empty DB) => chunk 3 last ledger (highest) require.NoError(t, cat.PinEarliestLedger(2)) got, err := deriveCompleteThrough(cat) require.NoError(t, err) @@ -220,7 +185,9 @@ func TestDeriveWatermark(t *testing.T) { chunk4Last := chunk.ID(4).LastLedger() seedReadyLiveDB(t, cat, 4, chunk4Last) require.NoError(t, cat.PutHotTransient(5)) // the crashed live chunk - require.Equal(t, chunk.ID(3).LastLedger(), mustDeriveCompleteThrough(t, cat), + // The positional term alone (highest ready 4, minus 1) under-counts to chunk 3; + // only the refinement below, opening chunk 4's real DB, recovers chunk 4's frontier. + require.Equal(t, chunk.ID(3).LastLedger(), geometry.CompleteThrough(3), "positional term alone under-counts to chunk 3") got, err := deriveWatermark(cat, silentLogger()) @@ -261,13 +228,6 @@ func TestDeriveWatermark(t *testing.T) { seedReadyLiveDB(t, cat, 0, 0) // ready + real dir, nothing committed got, err := deriveWatermark(cat, silentLogger()) require.NoError(t, err) - require.Equal(t, preGenesisLedger, got) + require.Equal(t, geometry.PreGenesisLedger, got) }) } - -func mustDeriveCompleteThrough(t *testing.T, cat *catalog.Catalog) uint32 { - t.Helper() - got, err := deriveCompleteThrough(cat) - require.NoError(t, err) - return got -} diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention.go index 951d54bb7..c96b65678 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention.go @@ -20,7 +20,14 @@ type RetentionFloor struct { // NewRetentionFloor pins the floor for one (through, retentionChunks, earliest) // snapshot. A shortened retentionChunks raises the floor at once. func NewRetentionFloor(through, retentionChunks, earliest uint32) RetentionFloor { - return RetentionFloor{chunk: chunk.IDFromLedger(EffectiveRetentionFloor(through, retentionChunks, earliest))} + return RetentionFloorAt(EffectiveRetentionFloor(through, retentionChunks, earliest)) +} + +// RetentionFloorAt pins the floor from an already-computed floor ledger, so the +// tick derives EffectiveRetentionFloor once and shares it between the gauge and +// the gate rather than recomputing it per scan. +func RetentionFloorAt(floorLedger uint32) RetentionFloor { + return RetentionFloor{chunk: chunk.IDFromLedger(floorLedger)} } // Excludes reports whether chunk c is below the floor (past retention). The scans @@ -28,6 +35,11 @@ func NewRetentionFloor(through, retentionChunks, earliest uint32) RetentionFloor // its last chunk is, as Excludes(layout.LastChunk(idx)) for a whole index. func (f RetentionFloor) Excludes(c chunk.ID) bool { return c < f.chunk } +// FirstChunk is the lowest in-retention chunk — the single floor→chunk boundary +// definition shared by prune (the gate), the lifecycle plan range, and startup +// backfill, so the three can never disagree on where retention begins. +func (f RetentionFloor) FirstChunk() chunk.ID { return f.chunk } + // EffectiveRetentionFloor is the chunk-aligned lower bound of the retention // window: the HIGHER of the sliding floor (retentionChunks back from the last // complete chunk) and the fixed earliest_ledger. slidingChunk is signed so a diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go index c43012ae4..7ced429e6 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/retention_test.go @@ -136,7 +136,7 @@ func TestReaderRetention_WindowStraddlingFloorServesInRangeNotBelow(t *testing.T assert.False(t, floor.Excludes(wins.LastChunk(0)), "a straddling window is not wholly below the floor — its .idx is kept") cfg := lifecycleTestConfig(t, cat, 2) - pops, _, err := eligiblePruneOps(cfg, cat, through) + pops, _, err := eligiblePruneOps(cat, gateFor(t, cfg, cat, through)) require.NoError(t, err) for _, op := range pops { require.NoError(t, op()) diff --git a/cmd/stellar-rpc/internal/fullhistory/observability/observability.go b/cmd/stellar-rpc/internal/fullhistory/observability/observability.go index 708e9111d..cb5e0c0ef 100644 --- a/cmd/stellar-rpc/internal/fullhistory/observability/observability.go +++ b/cmd/stellar-rpc/internal/fullhistory/observability/observability.go @@ -10,9 +10,17 @@ import ( // per-phase wall-clock timings; distinct from the per-data-type ingest.MetricSink. // All methods must be safe for concurrent use. type Metrics interface { - // LastCommitted sets the derived last-committed ledger and the effective - // retention floor (the two advance together each backfill pass / lifecycle tick). - LastCommitted(lastCommitted, retentionFloor uint32) + // LastCommitted sets the derived last-committed ledger gauge. Owned by the two + // call sites that know the TRUE value: startup/backfill (as history advances) + // and the ingestion loop (one atomic gauge set per committed ledger). The tick + // must NOT set it — its chunk-aligned lastChunk.LastLedger() would regress the + // gauge below a mid-chunk refined watermark on every restart. + LastCommitted(lastCommitted uint32) + + // RetentionFloor sets the effective retention floor gauge (lowest in-window + // ledger). Owned by startup/backfill and the lifecycle tick; the floor depends + // only on the last complete chunk, so it does not regress in the tick's window. + RetentionFloor(retentionFloor uint32) // ChunkBoundary counts one ingestion chunk-boundary handoff. The closed chunk // id is logged at the call site; this metric is a plain counter. @@ -38,14 +46,15 @@ type Metrics interface { // NopMetrics discards every signal — the default when a config carries no Metrics. type NopMetrics struct{} -func (NopMetrics) LastCommitted(uint32, uint32) {} -func (NopMetrics) ChunkBoundary() {} -func (NopMetrics) LiveHotChunks(int) {} -func (NopMetrics) BackfillPass(time.Duration) {} -func (NopMetrics) Freeze(time.Duration) {} -func (NopMetrics) Rebuild(time.Duration) {} -func (NopMetrics) Discard(int, time.Duration) {} -func (NopMetrics) Prune(int, time.Duration) {} +func (NopMetrics) LastCommitted(uint32) {} +func (NopMetrics) RetentionFloor(uint32) {} +func (NopMetrics) ChunkBoundary() {} +func (NopMetrics) LiveHotChunks(int) {} +func (NopMetrics) BackfillPass(time.Duration) {} +func (NopMetrics) Freeze(time.Duration) {} +func (NopMetrics) Rebuild(time.Duration) {} +func (NopMetrics) Discard(int, time.Duration) {} +func (NopMetrics) Prune(int, time.Duration) {} // MetricsOrNop returns m, or NopMetrics{} when nil, so call sites never nil-check. func MetricsOrNop(m Metrics) Metrics { @@ -125,8 +134,11 @@ func NewPrometheusMetrics(registry *prometheus.Registry, namespace string) *Prom return m } -func (m *PrometheusMetrics) LastCommitted(lastCommitted, retentionFloor uint32) { +func (m *PrometheusMetrics) LastCommitted(lastCommitted uint32) { m.lastCommitted.Set(float64(lastCommitted)) +} + +func (m *PrometheusMetrics) RetentionFloor(retentionFloor uint32) { m.retentionFloor.Set(float64(retentionFloor)) } diff --git a/cmd/stellar-rpc/internal/fullhistory/observability/observability_test.go b/cmd/stellar-rpc/internal/fullhistory/observability/observability_test.go index 9814ff648..336dfcff5 100644 --- a/cmd/stellar-rpc/internal/fullhistory/observability/observability_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/observability/observability_test.go @@ -17,10 +17,14 @@ import ( func TestMetricsOrNop_NilNeverPanics(t *testing.T) { m := MetricsOrNop(nil) require.NotNil(t, m) - m.LastCommitted(5, 2) + m.LastCommitted(5) + m.RetentionFloor(2) + m.ChunkBoundary() + m.LiveHotChunks(3) m.BackfillPass(time.Second) m.Freeze(time.Second) m.Rebuild(time.Second) + m.Discard(1, time.Second) m.Prune(2, time.Second) } @@ -34,10 +38,15 @@ func TestPrometheusMetrics_RegistersAndRecords(t *testing.T) { reg := prometheus.NewRegistry() m := NewPrometheusMetrics(reg, "test_ns") - m.LastCommitted(58, 12) + m.LastCommitted(58) + m.RetentionFloor(12) + m.LiveHotChunks(4) + m.ChunkBoundary() + m.ChunkBoundary() m.BackfillPass(250 * time.Millisecond) m.Freeze(100 * time.Millisecond) m.Rebuild(50 * time.Millisecond) + m.Discard(3, 20*time.Millisecond) m.Prune(2, 5*time.Millisecond) families, err := reg.Gather() @@ -61,10 +70,13 @@ func TestPrometheusMetrics_RegistersAndRecords(t *testing.T) { assert.InDelta(t, float64(58), values["test_ns_fullhistory_streaming_last_committed_ledger"], 0) assert.InDelta(t, float64(12), values["test_ns_fullhistory_streaming_retention_floor_ledger"], 0) + assert.InDelta(t, float64(4), values["test_ns_fullhistory_streaming_live_hot_chunks"], 0) + assert.InDelta(t, float64(2), values["test_ns_fullhistory_streaming_chunk_boundaries_total"], 0) + assert.InDelta(t, float64(3), values["test_ns_fullhistory_streaming_discarded_hot_chunks_total"], 0) assert.InDelta(t, float64(2), values["test_ns_fullhistory_streaming_pruned_artifacts_total"], 0) - // Phase-duration histogram saw backfill_pass + freeze + rebuild + prune = 4 observations. - assert.Equal(t, uint64(4), counts["test_ns_fullhistory_streaming_phase_duration_seconds"]) + // Phase-duration histogram saw backfill_pass + freeze + rebuild + discard + prune = 5 observations. + assert.Equal(t, uint64(5), counts["test_ns_fullhistory_streaming_phase_duration_seconds"]) } // Double-registration on the same registry panics (one sink per registry). diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index 2903435d0..cb6e5dda2 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -57,8 +57,8 @@ func run(ctx context.Context, cfg StartConfig) error { } metrics := observability.MetricsOrNop(cfg.Exec.Metrics) - metrics.LastCommitted(lastCommitted, - lifecycle.EffectiveRetentionFloor(lastCommitted, cfg.RetentionChunks, earliest)) + metrics.LastCommitted(lastCommitted) + metrics.RetentionFloor(lifecycle.EffectiveRetentionFloor(lastCommitted, cfg.RetentionChunks, earliest)) logger.WithField("last_committed", lastCommitted). WithField("earliest", earliest). WithField("pinned", pinned). @@ -192,9 +192,11 @@ func backfillToTip(ctx context.Context, cfg StartConfig, lastCommitted, earliest rangeEndSigned := geometry.LastCompleteChunkAt(anchor) // Mid-chunk resume exclusion: a mid-chunk last-committed within one chunk of the tip - // leaves the partial resume chunk to ingestion. Signed so genesis reads as a boundary. + // leaves the partial resume chunk to ingestion. Under the mid-chunk precondition + // (guarded here) the last COMPLETE chunk is exactly one short of the live chunk, + // so LastCompleteChunkAt names it directly — same vocabulary as rangeEndSigned above. if withinOneChunkOfTip(tip, lastCommitted) && lastCommittedMidChunk(lastCommitted) { - rangeEndSigned = lifecycle.ChunkIDOfLedger(lastCommitted) - 1 // one short of the live chunk + rangeEndSigned = geometry.LastCompleteChunkAt(lastCommitted) } // Break on an empty or non-advancing range. @@ -221,7 +223,8 @@ func backfillToTip(ctx context.Context, cfg StartConfig, lastCommitted, earliest metrics.BackfillPass(passDuration) // Refresh the derived gauges as last-committed advances and the floor rises with it. - metrics.LastCommitted(lastCommitted, lifecycle.EffectiveRetentionFloor(lastCommitted, retentionChunks, earliest)) + metrics.LastCommitted(lastCommitted) + metrics.RetentionFloor(lifecycle.EffectiveRetentionFloor(lastCommitted, retentionChunks, earliest)) logger.WithField("range_lo", rangeStart.String()). WithField("range_hi", rangeEnd.String()). WithField("last_committed", lastCommitted). @@ -240,8 +243,8 @@ func withinOneChunkOfTip(tip, lastCommitted uint32) bool { // lastCommittedMidChunk reports whether lastCommitted falls strictly inside a chunk. // The genesis sentinel reads as a boundary, never mid-chunk. func lastCommittedMidChunk(lastCommitted uint32) bool { - c := lifecycle.ChunkIDOfLedger(lastCommitted) - return lastCommitted != lifecycle.CompleteThrough(c) + c := geometry.ChunkIDOfLedger(lastCommitted) + return lastCommitted != geometry.CompleteThrough(c) } // --------------------------------------------------------------------------- From 3cb0e7d1d35c50fdcd505e485034a26636f25dd8 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Fri, 3 Jul 2026 00:26:26 -0400 Subject: [PATCH 50/55] fullhistory/ingest,stores: delete dead seams (round-4 review) - Delete the LedgerIngester interface: it had one consumer (drain) and one implementor reaching it (*ColdService); drain takes *ColdService directly. The ColdIngester seam already provides drain's testability (the overrun test now fakes that layer) (thread 3517303073). - Terminal-error emission is now structural: coldMetrics.observe emits the single per-ingester ColdIngest itself when err != nil (an Ingest error is terminal by the ColdIngester contract), deleting the hand-paired observe+emit(0,nil) copied in all three cold ingesters. Fix the ColdService.Close doc to match the emit-only-on-a-terminal-step semantics (thread 3517303336). - Delete the ChunkID() accessor + chunkID field + ctor param from the two thin hot facades (ledger.HotStore, txhash.HotStore): the accessor had no production caller and the "driver can reject a mismatched store" binding it documented was never implemented. eventstore.HotStore keeps its chunkID (load-bearing for range checks + error messages) (thread 3511861630). --- .../internal/fullhistory/ingest/doc.go | 2 +- .../internal/fullhistory/ingest/driver.go | 4 +-- .../internal/fullhistory/ingest/events.go | 5 ++-- .../fullhistory/ingest/ingest_test.go | 8 ++++-- .../internal/fullhistory/ingest/ingester.go | 28 +++---------------- .../internal/fullhistory/ingest/ledgers.go | 3 +- .../internal/fullhistory/ingest/metrics.go | 6 +++- .../internal/fullhistory/ingest/service.go | 13 ++++----- .../internal/fullhistory/ingest/txhash.go | 3 +- .../pkg/stores/hotchunk/hotchunk.go | 4 +-- .../pkg/stores/ledger/hot_store.go | 24 ++++++---------- .../pkg/stores/ledger/hot_store_test.go | 20 +++++-------- .../pkg/stores/txhash/hot_store.go | 20 ++++--------- .../pkg/stores/txhash/hot_store_test.go | 20 +++++-------- 14 files changed, 58 insertions(+), 102 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go b/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go index 1c387ec5e..4eeb79f70 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/doc.go @@ -58,7 +58,7 @@ // // Inputs are borrowed: every Ingest receives a view over the source // stream's buffer, valid only until the next ledger is pulled, and -// each ingester copies what it retains (see LedgerIngester). The raw +// each ingester copies what it retains (see ColdIngester). The raw // ledger iterator's contract includes yielding an error on ctx // cancellation — the drain loop relies on it for cancellation rather // than polling ctx itself. Metrics flow through MetricSink (Prometheus in prod, diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go index 3f50cc908..e8734937e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go @@ -76,7 +76,7 @@ func SeqValidatedCursor( // runs before Finalize, so a short stream never finalizes a truncated artifact. // Cancellation is the iterator's job (RawLedgers errors on canceled ctx), so no // ctx poll here. The per-ledger sequence guard lives in the shared cursor. -func drain(ctx context.Context, ledgers iter.Seq2[[]byte, error], chunkID chunk.ID, ing LedgerIngester) error { +func drain(ctx context.Context, ledgers iter.Seq2[[]byte, error], chunkID chunk.ID, svc *ColdService) error { first, last := chunkID.FirstLedger(), chunkID.LastLedger() seq := first for vl, verr := range SeqValidatedCursor(ledgers, first) { @@ -90,7 +90,7 @@ func drain(ctx context.Context, ledgers iter.Seq2[[]byte, error], chunkID chunk. return fmt.Errorf("ingest: stream for chunk %d yielded a ledger past %d (chunk overrun)", uint32(chunkID), last) } - if err := ing.Ingest(ctx, vl.Seq, vl.View); err != nil { + if err := svc.Ingest(ctx, vl.Seq, vl.View); err != nil { return err } seq = vl.Seq + 1 diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/events.go b/cmd/stellar-rpc/internal/fullhistory/ingest/events.go index c39e5e4f0..c49f337bd 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/events.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/events.go @@ -69,10 +69,9 @@ func NewEventsColdIngester(bucketDir string, chunkID chunk.ID, sink MetricSink) func (e *eventsCold) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { start := time.Now() n, ierr := e.ingestSeq(seq, lcm) - e.metrics.observe(time.Since(start), n, ierr) + e.metrics.observe(time.Since(start), n, ierr) // terminal on err: observe emits the per-ingester signal if ierr != nil { - e.failed = true - e.metrics.emit(0, nil) // an Ingest error abandons the chunk; meter it now (Close no longer emits) + e.failed = true // refuse a post-failure Finalize return ierr } return nil diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go index 080c326b1..8eb3182b0 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go @@ -1331,13 +1331,16 @@ func TestColdService_Finalize_FirstErrorStopsRemaining(t *testing.T) { // ───────────────────────── drain overrun guard ───────────────────────── // countingIngester counts Ingest calls; used to prove the overrun guard fires -// BEFORE the out-of-chunk ledger is handed to the ingesters. +// BEFORE the out-of-chunk ledger is handed to the ingesters. It fakes the +// ColdIngester seam (a ColdService drives it), the layer drain consumes. type countingIngester struct{ ingested int } func (c *countingIngester) Ingest(context.Context, uint32, xdr.LedgerCloseMetaView) error { c.ingested++ return nil } +func (*countingIngester) Finalize(context.Context) error { return nil } +func (*countingIngester) Close() error { return nil } // TestDrain_OverrunPastChunk asserts a stream that keeps yielding in order // PAST the chunk's last ledger is rejected before the overrun ledger is @@ -1349,8 +1352,9 @@ func TestDrain_OverrunPastChunk(t *testing.T) { // One ledger past the chunk, still in order. stream := &fakeStream{t: t, count: ledgersInChunk + 1} counter := &countingIngester{} + service := NewColdService([]ColdIngester{counter}, nil) - err := drain(context.Background(), rawChunk(stream, chunkID), chunkID, counter) + err := drain(context.Background(), rawChunk(stream, chunkID), chunkID, service) require.Error(t, err) require.Contains(t, err.Error(), "overrun") require.Equal(t, int(ledgersInChunk), counter.ingested, diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go index 6b4d1638b..a70069f4d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go @@ -6,29 +6,6 @@ import ( "github.com/stellar/go-stellar-sdk/xdr" ) -// LedgerIngester is drain's per-ledger consumer: it ingests one ledger by -// sequence into a caller-owned store. ColdService implements it (drain drives the -// cold materializer through it); the hot tier's HotService satisfies the same -// shape and the ingestion loop calls it, though the loop drives HotService -// directly rather than through this interface. -// -// Ownership: the store is INJECTED into the implementation's constructor and -// owned by the caller (the daemon). The implementation does NOT open the store -// and does NOT close it — Close is intentionally absent from this interface. -// -// Input: seq is the CURSOR-VALIDATED ledger sequence of lcm — the shared -// seq-validated cursor (SeqValidatedCursor) has already read it off the view and -// checked it is contiguous (no gap / duplicate / out-of-order), so implementations -// consume it directly instead of re-deriving and re-error-handling it. lcm is a -// zero-copy xdr.LedgerCloseMetaView (a []byte alias over the source stream's -// BORROWED buffer), valid only for the current iteration step; an implementation -// must copy any bytes it retains. Ledgers are ingested sequentially — the source -// pulls the next only after Ingest returns — so synchronous consumption inside -// Ingest is safe. -type LedgerIngester interface { - Ingest(ctx context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error -} - // ColdIngester ingests one data type for one chunk into a per-chunk cold writer. // // Ownership: the ingester OPENS its own per-chunk writer in its constructor and @@ -44,7 +21,10 @@ type LedgerIngester interface { // artifact; implementations are encouraged to latch the failure and refuse // (eventsCold does). // -// Input: same cursor-validated-seq and borrowed-view contract as LedgerIngester. +// Input: seq is the cursor-validated ledger sequence of lcm (the shared +// SeqValidatedCursor has already checked contiguity), and lcm is a zero-copy +// xdr.LedgerCloseMetaView over the source stream's BORROWED buffer, valid only for +// the current iteration step — an implementation must copy any bytes it retains. // ColdService drives the per-ledger Ingest calls sequentially, so each view is // fully consumed before the next. type ColdIngester interface { diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go index b303efe6e..74457c13e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go @@ -43,8 +43,7 @@ func NewLedgerColdIngester(packPath string, chunkID chunk.ID, sink MetricSink) ( func (c *ledgerCold) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { start := time.Now() if err := c.writer.AppendLedger(seq, []byte(lcm)); err != nil { - c.metrics.observe(time.Since(start), 0, err) - c.metrics.emit(0, nil) // an Ingest error abandons the chunk; meter it now (Close no longer emits) + c.metrics.observe(time.Since(start), 0, err) // terminal: observe emits the per-ingester signal return fmt.Errorf("AppendLedger(seq=%d): %w", seq, err) } c.metrics.sink.IngestStage(dataTypeLedgers, tierCold, stageWrite, time.Since(start), 1) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go index 7586e9abc..08c4fe152 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go @@ -123,12 +123,16 @@ func newColdMetrics(sink MetricSink, dataType string) coldMetrics { return coldMetrics{sink: orNop(sink), dataType: dataType} } -// observe records one Ingest's wall-clock and (on error) the first error. +// observe records one Ingest's wall-clock and (on error) the first error. An +// Ingest error is TERMINAL by the ColdIngester contract (the chunk is abandoned +// and the ingester is never reused), so observe emits the single per-ingester +// ColdIngest itself here — callers just observe-and-return, no hand-paired emit. func (m *coldMetrics) observe(d time.Duration, items int, err error) { m.accum += d m.items += items if err != nil { m.firstErr = errOrFirst(m.firstErr, err) + m.emit(0, nil) } } diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go index 0c95a5a62..bb5750132 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go @@ -126,13 +126,12 @@ func (s *ColdService) Finalize(ctx context.Context) error { } // Close closes every cold ingester, joining each Close error, and emits the -// aggregate ColdChunkTotal if Finalize never reached it (the failure path). The -// per-ingester ColdIngest is emitted on a terminal Ingest error or in Finalize, -// never from an ingester's Close — so a chunk that failed after ingesting still -// produced one per-ingester signal, while one rolled back before any work -// produces none (only the aggregate here). Idempotent: on the failure path a -// writer's Close drops its partial file; after a successful Finalize this is a -// no-op for the aggregate. +// aggregate ColdChunkTotal if Finalize never reached it (the failure path). A +// per-ingester ColdIngest is emitted only from a TERMINAL step (a failed Ingest, +// via coldMetrics.observe, or Finalize) — never from Close, so an ingester rolled +// back before any work produces no per-ingester sample (only the aggregate here). +// Idempotent: on the failure path a writer's Close drops its partial file; after +// a successful Finalize this is a no-op for the aggregate. func (s *ColdService) Close() error { var err error for _, ing := range s.ingesters { diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go b/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go index dd9ce305f..364e6dbe7 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go @@ -59,8 +59,7 @@ func (t *txhashCold) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMe // chunk that intermediate would be hundreds of MB of transient garbage. hashes, err := sdkingest.ExtractTxHashes(lcm) if err != nil { - t.metrics.observe(time.Since(start), 0, err) - t.metrics.emit(0, nil) // an Ingest error abandons the chunk; meter it now (Close no longer emits) + t.metrics.observe(time.Since(start), 0, err) // terminal: observe emits the per-ingester signal return fmt.Errorf("ExtractTxHashes seq %d: %w", seq, err) } for i := range hashes { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index ab4b13e84..b3f407759 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -115,8 +115,8 @@ func open(path string, chunkID chunk.ID, logger *supportlog.Entry, readOnly, mus return &DB{ store: store, chunkID: chunkID, - ledger: ledger.NewWithStore(store, chunkID), - txhash: txhash.NewWithStore(store, chunkID), + ledger: ledger.NewWithStore(store), + txhash: txhash.NewWithStore(store), events: es, }, nil } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go index 3077cddb2..b860a93cb 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store.go @@ -9,7 +9,6 @@ import ( "iter" "sync" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/zstd" @@ -32,10 +31,9 @@ type Entry struct { } // HotStore — RocksDB-backed hot ledger store. Keys are 4-byte BE sequences; -// values are zstd-compressed (internal). Chunk-bound: accumulates one chunk's -// ledgers before freezing, with the binding recorded at open time (ChunkID) so -// the ingest driver can reject a mismatched store. The store does not itself -// range-check writes (the driver's drain loop already validates every sequence). +// values are zstd-compressed (internal). It accumulates one chunk's ledgers +// before freezing; it does not itself range-check writes (the driver's drain loop +// already validates every sequence against the chunk). // // Concurrency: all methods are safe for concurrent use, including use alongside // the caller-owned rocksdb.Store.Close. A read/write racing Close either completes @@ -43,9 +41,8 @@ type Entry struct { // adds no unguarded state of its own — the compressor pool and decompressor are // both concurrent-safe. type HotStore struct { - store *rocksdb.Store - chunkID chunk.ID - dec *zstd.Decompressor + store *rocksdb.Store + dec *zstd.Decompressor // compPool — per-store pool of zstd.Compressors; each concurrent // AddLedgerToBatch borrows one for its Encode call. compPool sync.Pool @@ -55,21 +52,16 @@ type HotStore struct { // LedgersCF. The store is owned by the caller — in production, hotchunk.DB // composes this facade over the shared multi-CF DB and closes that DB once. The // store must have LedgersCF registered. -func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) *HotStore { +func NewWithStore(store *rocksdb.Store) *HotStore { return &HotStore{ - store: store, - chunkID: chunkID, - dec: zstd.NewDecompressor(), + store: store, + dec: zstd.NewDecompressor(), compPool: sync.Pool{ New: func() any { return zstd.NewCompressor() }, }, } } -// ChunkID returns the chunk this store is bound to (constructor-supplied; -// never reads the store). -func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } - // AddLedgerToBatch compresses one ledger and queues its Put into b on LedgersCF // — the building block hotchunk uses to fold the ledger write into the one // shared per-ledger WriteBatch (decision (a)). Does not commit (caller owns the diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go index 50fc4aac5..fc53cab30 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger/hot_store_test.go @@ -17,7 +17,6 @@ import ( supportlog "github.com/stellar/go-stellar-sdk/support/log" "github.com/stellar/go-stellar-sdk/xdr" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores" ) @@ -32,11 +31,11 @@ func silentLogger() *supportlog.Entry { func openTestHotStore(t *testing.T) *HotStore { t.Helper() - h, _ := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) + h, _ := openTestHotStoreAt(t, t.TempDir()) return h } -func openTestHotStoreAt(t *testing.T, path string, chunkID chunk.ID) (*HotStore, *rocksdb.Store) { +func openTestHotStoreAt(t *testing.T, path string) (*HotStore, *rocksdb.Store) { t.Helper() store, err := rocksdb.New(rocksdb.Config{ Path: path, @@ -45,12 +44,7 @@ func openTestHotStoreAt(t *testing.T, path string, chunkID chunk.ID) (*HotStore, }) require.NoError(t, err) t.Cleanup(func() { _ = store.Close() }) - return NewWithStore(store, chunkID), store -} - -func TestNewWithStore_RecordsChunkBinding(t *testing.T) { - h, _ := openTestHotStoreAt(t, t.TempDir(), chunk.ID(7)) - require.Equal(t, chunk.ID(7), h.ChunkID()) + return NewWithStore(store), store } func TestHotStore_AddGetRoundTripVerbatim(t *testing.T) { @@ -212,11 +206,11 @@ func TestHotStore_GracefulCloseAndReopen(t *testing.T) { {Seq: 15, Bytes: []byte("payload-15")}, } - first, firstStore := openTestHotStoreAt(t, path, chunk.ID(0)) + first, firstStore := openTestHotStoreAt(t, path) require.NoError(t, addLedgers(first, seeded...)) require.NoError(t, firstStore.Close()) - second, _ := openTestHotStoreAt(t, path, chunk.ID(0)) + second, _ := openTestHotStoreAt(t, path) for _, want := range seeded { got, err := second.GetLedgerRaw(want.Seq) @@ -226,7 +220,7 @@ func TestHotStore_GracefulCloseAndReopen(t *testing.T) { } func TestHotStore_PostCloseOps(t *testing.T) { - h, store := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) + h, store := openTestHotStoreAt(t, t.TempDir()) require.NoError(t, store.Close()) require.ErrorIs(t, addLedgers(h, Entry{Seq: 1, Bytes: []byte("v")}), stores.ErrStoreClosed) @@ -248,7 +242,7 @@ func TestHotStore_PostCloseOps(t *testing.T) { } func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { - h, store := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) + h, store := openTestHotStoreAt(t, t.TempDir()) for i := range uint32(50) { require.NoError(t, addLedgers(h, Entry{Seq: i, Bytes: []byte("v")})) } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go index da2bb38b6..1c1ed81fe 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store.go @@ -4,7 +4,6 @@ package txhash import ( - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores" ) @@ -25,22 +24,19 @@ type Entry struct { // // Like every hot store, a HotStore instance is chunk-bound: it // accumulates exactly one chunk's (txhash → seq) tuples before being -// frozen into the chunk's cold .bin artifact. The binding is recorded -// at open time (ChunkID) so the ingest driver can reject a store -// bound to a different chunk than it is ingesting; the store does not -// itself range-check writes (the driver's drain loop already -// validates every ledger sequence against the chunk). +// frozen into the chunk's cold .bin artifact. The store does not itself +// range-check writes (the driver's drain loop already validates every ledger +// sequence against the chunk). type HotStore struct { - store *rocksdb.Store - chunkID chunk.ID + store *rocksdb.Store } // NewWithStore wraps an ALREADY-OPEN rocksdb.Store as a txhash HotStore on the // single txhash CF (CFNames()). The store is owned by the caller — in production, // hotchunk.DB composes this facade over the shared per-chunk DB and closes that DB // once. The store must have CFNames() registered. -func NewWithStore(store *rocksdb.Store, chunkID chunk.ID) *HotStore { - return &HotStore{store: store, chunkID: chunkID} +func NewWithStore(store *rocksdb.Store) *HotStore { + return &HotStore{store: store} } // CFNames returns the single txhash CF name this facade owns. Exported so @@ -103,10 +99,6 @@ func Tuning() rocksdb.Tuning { } } -// ChunkID returns the chunk this store is bound to (constructor-supplied; -// never reads the store). -func (h *HotStore) ChunkID() chunk.ID { return h.chunkID } - // AddEntriesToBatch queues each (txhash → ledgerSeq) Put into b on the txhash // CF — the building block hotchunk uses to fold the tx-hash writes into the one // shared per-ledger WriteBatch (decision (a)). Does not commit (caller owns the diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go index 4c63a16cd..7e0a117e0 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash/hot_store_test.go @@ -13,7 +13,6 @@ import ( supportlog "github.com/stellar/go-stellar-sdk/support/log" - "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores" ) @@ -40,11 +39,11 @@ func txhashFor(nibble, tag byte) [32]byte { func openTestHotStore(t *testing.T) *HotStore { t.Helper() - s, _ := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) + s, _ := openTestHotStoreAt(t, t.TempDir()) return s } -func openTestHotStoreAt(t *testing.T, path string, chunkID chunk.ID) (*HotStore, *rocksdb.Store) { +func openTestHotStoreAt(t *testing.T, path string) (*HotStore, *rocksdb.Store) { t.Helper() store, err := rocksdb.New(rocksdb.Config{ Path: path, @@ -54,12 +53,7 @@ func openTestHotStoreAt(t *testing.T, path string, chunkID chunk.ID) (*HotStore, }) require.NoError(t, err) t.Cleanup(func() { _ = store.Close() }) - return NewWithStore(store, chunkID), store -} - -func TestNewWithStore_RecordsChunkBinding(t *testing.T) { - s, _ := openTestHotStoreAt(t, t.TempDir(), chunk.ID(7)) - require.Equal(t, chunk.ID(7), s.ChunkID()) + return NewWithStore(store), store } func TestHotStore_AddGetRoundTrip(t *testing.T) { @@ -140,7 +134,7 @@ func TestHotStore_AddEntriesMultiple(t *testing.T) { } func TestHotStore_PostCloseOps(t *testing.T) { - s, store := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) + s, store := openTestHotStoreAt(t, t.TempDir()) require.NoError(t, store.Close()) h := txhashFor(0x5, 1) @@ -155,7 +149,7 @@ func TestHotStore_PostCloseOps(t *testing.T) { func TestHotStore_GracefulCloseAndReopenRoundTrips(t *testing.T) { path := t.TempDir() - first, firstStore := openTestHotStoreAt(t, path, chunk.ID(0)) + first, firstStore := openTestHotStoreAt(t, path) for n := range 16 { require.NoError(t, addEntries(first, []Entry{ {Hash: txhashFor(byte(n), 1), LedgerSeq: uint32(n) + 1}, @@ -163,7 +157,7 @@ func TestHotStore_GracefulCloseAndReopenRoundTrips(t *testing.T) { } require.NoError(t, firstStore.Close()) - second, _ := openTestHotStoreAt(t, path, chunk.ID(0)) + second, _ := openTestHotStoreAt(t, path) for n := range 16 { got, err := second.Get(txhashFor(byte(n), 1)) @@ -173,7 +167,7 @@ func TestHotStore_GracefulCloseAndReopenRoundTrips(t *testing.T) { } func TestHotStore_ConcurrentOpsAndCloseRaceFree(t *testing.T) { - s, store := openTestHotStoreAt(t, t.TempDir(), chunk.ID(0)) + s, store := openTestHotStoreAt(t, t.TempDir()) // Pre-populate a spread of distinct keys. pre := make([]Entry, 16) for n := range 16 { From 11ed6fa4e2f9b6284fad1f064d1f7f242882129f Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Fri, 3 Jul 2026 00:37:36 -0400 Subject: [PATCH 51/55] fullhistory: missing test pins + stale doc/comment fixes (round-4 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: - hotchunk: TestIngestLedger_EventlessTxStillIndexesHash — a ledger with an eventful + an event-less tx; both hashes land in the txhash CF, only the eventful tx's event in the events CF. Pins the post-#18 invariant that txhash completeness rests on ExtractLedgerEvents yielding an element per applied tx, event-less included (thread 3517303006). - rocksdb: TestNew_MustExist_EmptyReadyDBReopens (must-exist reopen of an empty ready DB succeeds) + TestNew_MustExist_GuttedDirFailsOpen (a gutted dir fails the open, never auto-heals) — the two MustExist pins at the rocksdb level (threads 3516017921/3517306778). Docs/comments: - Delete the vacuous LOCK-release assertion in TestRun_ServeReadsErrorSurfaces (ServeReads runs before the loop opens the hot DB) + stale getter-era comments in startup_test.go / hotloop_test.go (thread 3517303157). - Doc strays: progress.go read-only-open-takes-no-LOCK; hotloop.go "fatal" -> "won't-open error"; hotchunk.OpenReadOnly "un-synced" -> "synced-but-unflushed" WAL; helpers_test.go file refs (hotloop.go / hotloop_test.go) (3517303282/3517303185). --- .../fullhistory/backfill/hotsource_test.go | 2 +- .../internal/fullhistory/hotloop.go | 2 +- .../internal/fullhistory/hotloop_test.go | 2 +- .../fullhistory/lifecycle/helpers_test.go | 4 +- .../fullhistory/lifecycle/progress.go | 3 +- .../lifecycle/progress_realdb_test.go | 2 +- .../fullhistory/pkg/rocksdb/rocksdb_test.go | 41 +++++++++++++++++++ .../pkg/stores/hotchunk/hotchunk.go | 10 +++-- .../pkg/stores/hotchunk/hotchunk_test.go | 39 ++++++++++++++++++ .../internal/fullhistory/startup_test.go | 17 +++----- 10 files changed, 100 insertions(+), 22 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/backfill/hotsource_test.go b/cmd/stellar-rpc/internal/fullhistory/backfill/hotsource_test.go index a27f542e6..fc67d74b1 100644 --- a/cmd/stellar-rpc/internal/fullhistory/backfill/hotsource_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/backfill/hotsource_test.go @@ -28,7 +28,7 @@ func seedReadyHotChunk(t *testing.T, cat *catalog.Catalog, c chunk.ID, top uint3 Logger: silentLogger(), }) require.NoError(t, err) - h := ledger.NewWithStore(store, c) + h := ledger.NewWithStore(store) require.NoError(t, store.Batch(func(b *rocksdb.BatchWriter) error { return h.AddLedgerToBatch(b, ledger.Entry{Seq: top, Bytes: []byte("ledger")}) })) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index d778c90ac..1e2dd2c4d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -68,7 +68,7 @@ func openHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlo // The dir + dirent must be durable BEFORE the key flips to "ready", else a // crash between the flip and the dir's durability fabricates the "ready but - // dir missing" fatal above for a DB that was actually fine. FsyncNewDirs + // dir missing" won't-open error above for a DB that was actually fine. FsyncNewDirs // syncs the leaf then its parent dirent (the one audited barrier for a // freshly created dir). if syncErr := geometry.FsyncNewDirs(filepath.Dir(dir), dir); syncErr != nil { diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index 07d0674ee..ff1d061fa 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -38,7 +38,7 @@ type fakeCoreStream struct { yieldErrAt uint32 // if non-zero, yield errAt at this seq instead of bytes errAt error - calls atomic.Int32 // seqs considered (mirrors the old per-GetLedger count) + calls atomic.Int32 // seqs yielded by the stream firstSeen atomic.Uint32 sawFirst atomic.Bool } diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go index 9143a9edf..09bc2bad1 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/helpers_test.go @@ -116,7 +116,7 @@ func zeroTxLCMBytes(t *testing.T, seq uint32) []byte { // --------------------------------------------------------------------------- // Hot-tier test scaffolding: a test-local equivalent of the root package's hot -// DB opener (startup.go's openHotDBForChunk). It uses only the public +// DB opener (hotloop.go's openHotDBForChunk). It uses only the public // hotchunk/catalog APIs the production code uses, so a lifecycle test creates the // SAME on-disk "ready" hot DB the real daemon would — which the freeze and the // watermark refinement then open by Layout path, exactly as production does. @@ -126,7 +126,7 @@ func zeroTxLCMBytes(t *testing.T, seq uint32) []byte { // hot:chunk bracket (transient -> create -> ready) and returns an open handle the // caller owns. The test equivalent of the production opener, trimmed to the // create branch the lifecycle tests need (no crash-recovery / fsync — those edges -// are covered by the root ingest_test.go opener tests). +// are covered by the root hotloop_test.go opener tests). func openHotDBForChunk(cat *catalog.Catalog, chunkID chunk.ID, logger *supportlog.Entry) (*hotchunk.DB, error) { dir := cat.Layout().HotChunkPath(chunkID) if err := os.RemoveAll(dir); err != nil { diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go index 13534bfea..d25e62224 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress.go @@ -24,7 +24,8 @@ import ( // a fresh start). Leads at startup before any hot key exists. // - HOT — only when hot > cold, over "ready" keys: one read-only MaxCommittedSeq // read of the highest ready hot DB (empty DB ⇒ positional CompleteThrough(hot-1)). -// Safe: derivation runs before ingestion locks the DB. +// The read-only open takes no RocksDB LOCK, so it never contends with a writer; +// in practice it runs before ingestion opens the live chunk anyway. // - FLOOR — EarliestLedger()-1 as int64(earliest)-1, so an absent/zero pin // yields the pre-genesis sentinel rather than underflowing. // diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go index a01633f2d..394816e2b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/progress_realdb_test.go @@ -26,7 +26,7 @@ func seedLedgersCF(t *testing.T, cat *catalog.Catalog, c chunk.ID, entries ...le Logger: silentLogger(), }) require.NoError(t, err) - h := ledger.NewWithStore(store, c) + h := ledger.NewWithStore(store) require.NoError(t, store.Batch(func(b *rocksdb.BatchWriter) error { for _, e := range entries { if berr := h.AddLedgerToBatch(b, e); berr != nil { diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb_test.go index 07fe7c4e7..f1a726875 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/rocksdb/rocksdb_test.go @@ -48,6 +48,47 @@ func openTestStore(t *testing.T, cfNames []string) *Store { return s } +// TestNew_MustExist_EmptyReadyDBReopens pins that a must-exist read-write open of +// an already-created but EMPTY DB succeeds: the mode refuses only to CREATE, it +// never requires committed data. This is the "ready" hot-chunk reopen path (an +// ingester that crashed before committing its first ledger must still reopen). +func TestNew_MustExist_EmptyReadyDBReopens(t *testing.T) { + path := t.TempDir() + cf := []string{"c0"} + + // Create an empty DB the normal way (create-if-missing), then close it. + s, err := New(Config{Path: path, ColumnFamilies: cf, Logger: silentLogger()}) + require.NoError(t, err) + require.NoError(t, s.Close()) + + // Reopen must-exist: succeeds against the existing empty DB. + reopened, err := New(Config{Path: path, ColumnFamilies: cf, Logger: silentLogger(), MustExist: true}) + require.NoError(t, err, "must-exist reopen of an empty ready DB succeeds") + require.NoError(t, reopened.Close()) +} + +// TestNew_MustExist_GuttedDirFailsOpen pins that a must-exist open of a directory +// that exists but holds no valid RocksDB (no CURRENT) FAILS. The daemon depends on +// this: a "ready" hot key whose DB was wiped must never silently auto-heal into a +// fresh empty DB, which would regress the watermark. +func TestNew_MustExist_GuttedDirFailsOpen(t *testing.T) { + path := t.TempDir() + cf := []string{"c0"} + + // Create a real DB, close it, then gut the dir (remove every file, keep the dir). + s, err := New(Config{Path: path, ColumnFamilies: cf, Logger: silentLogger()}) + require.NoError(t, err) + require.NoError(t, s.Close()) + entries, err := os.ReadDir(path) + require.NoError(t, err) + for _, e := range entries { + require.NoError(t, os.RemoveAll(filepath.Join(path, e.Name()))) + } + + _, err = New(Config{Path: path, ColumnFamilies: cf, Logger: silentLogger(), MustExist: true}) + require.Error(t, err, "must-exist open of a gutted dir (no CURRENT) fails, never auto-heals") +} + func TestMain(m *testing.M) { if os.Getenv("ROCKSDB_LOCK_PROBE") == "1" { _, err := New(Config{ diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index b3f407759..f9ac2d850 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -87,10 +87,12 @@ func OpenExisting(path string, chunkID chunk.ID, logger *supportlog.Entry) (*DB, } // OpenReadOnly opens an EXISTING hot DB read-only — the freeze source's view AND -// the startup watermark refiner's. RocksDB's read-only open recovers any un-synced -// WAL into in-memory memtables (persisting nothing), so a reader sees every synced -// write even after an ungraceful crash — the watermark refinement DEPENDS on that -// replay to read a correct MaxCommittedSeq. Composing the facades only reads. +// the startup watermark refiner's. RocksDB's read-only open replays the +// synced-but-unflushed WAL into in-memory memtables (persisting nothing), so a +// reader sees every synced write even after an ungraceful crash — the watermark +// refinement DEPENDS on that replay to read a correct MaxCommittedSeq. (An +// unsynced tail is exactly what a crash loses, and is not recovered.) Composing +// the facades only reads. func OpenReadOnly(path string, chunkID chunk.ID, logger *supportlog.Entry) (*DB, error) { return open(path, chunkID, logger, true, false) } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go index adef22e4b..776d626ce 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go @@ -277,6 +277,45 @@ func TestIngestLedger_WritesEveryHotType(t *testing.T) { assert.Equal(t, uint64(1), bm.GetCardinality()) } +// TestIngestLedger_EventlessTxStillIndexesHash pins the post-merge txhash +// completeness invariant: after #18 folded the txhash and events walks into one +// ExtractLedgerEvents pass, txhash coverage rests entirely on that walk yielding +// an element per APPLIED tx — hash included — even for an event-less transaction +// (the common classic-only case). Every other hotchunk test uses one-tx-one-event +// ledgers, so nothing else pins it: an SDK change that dropped event-less txs from +// the walk would silently gut the txhash index for every classic-only transaction. +func TestIngestLedger_EventlessTxStillIndexesHash(t *testing.T) { + chunkID := chunk.ID(0) + first := chunkID.FirstLedger() + db := openTestDB(t) + + // Two applied txs in one ledger: one carries a contract event, one carries none. + eventful := xdr.TransactionMeta{V: 4, V4: &xdr.TransactionMetaV4{ + Operations: []xdr.OperationMetaV2{{Events: []xdr.ContractEvent{buildContractEvent("eventful")}}}, + }} + eventless := xdr.TransactionMeta{V: 4, V4: &xdr.TransactionMetaV4{ + Operations: []xdr.OperationMetaV2{{}}, // one op, no events + }} + lcm, hashes := buildLCM(t, first, []xdr.TransactionMeta{eventful, eventless}) + require.Len(t, hashes, 2) + raw, err := lcm.MarshalBinary() + require.NoError(t, err) + + counts, _, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) + require.NoError(t, err) + assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 2, Events: 1}, counts, + "both applied txs' hashes indexed (event-less included); only the eventful tx contributed an event") + + // Both hashes resolve in the txhash CF to this ledger. + for _, h := range hashes { + seq, gerr := db.Txhash().Get(h) + require.NoError(t, gerr, "event-less tx hash must still be indexed") + assert.Equal(t, first, seq) + } + // The events CF holds exactly the one eventful tx's event. + assert.Equal(t, uint32(1), eventCount(t, db.Events())) +} + // TestReopen_RecoversEventsMirror confirms the events facade's warmup runs over // the shared store on reopen (the mirror/offsets are reconstructed from the // events CFs), so a reopened DB assigns event IDs continuing from disk. diff --git a/cmd/stellar-rpc/internal/fullhistory/startup_test.go b/cmd/stellar-rpc/internal/fullhistory/startup_test.go index 4c83d7e2d..a44b130a8 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup_test.go @@ -356,7 +356,7 @@ func TestBackfill_LaggingBulkTipFoldsLastCommittedChunk(t *testing.T) { // A young-network first start does no backfill, opens the resume hot DB, starts // the (blocking) fake core, serves reads, and runs the ingestion loop — which -// surfaces the ctx-canceled GetLedger error on a clean shutdown (the daemon top +// surfaces the ctx-canceled stream error on a clean shutdown (the daemon top // level classifies it as clean). The resume ledger is genesis (watermark+1). func TestRun_FirstStartServeIngestCleanShutdown(t *testing.T) { cat, _ := testCatalog(t) @@ -373,7 +373,7 @@ func TestRun_FirstStartServeIngestCleanShutdown(t *testing.T) { go func() { errCh <- run(ctx, cfg) }() // Wait until the loop has opened the hot DB, started core, served, and parked on - // the blocking getter, then request a clean shutdown. + // the blocking stream, then request a clean shutdown. require.Eventually(t, func() bool { return served.Load() == 1 }, 2*time.Second, 5*time.Millisecond) cancel() @@ -395,10 +395,10 @@ func TestRun_FirstStartServeIngestCleanShutdown(t *testing.T) { assert.Equal(t, geometry.HotReady, state) } -// A ServeReads error is surfaced wrapped as a restartable failure (NOT clean) and -// the already-opened resume hot DB is closed on the way out, so a restart can -// reopen it (the rocksdb LOCK is released). ServeReads runs after the hot DB opens -// and core starts but before the blocking ingestion loop, so run returns here. +// A ServeReads error is surfaced wrapped as a restartable failure (NOT clean). +// ServeReads runs after core starts but BEFORE the ingestion loop launches, so run +// returns without the loop ever opening the resume hot DB (the boundary-close +// fence that releases the write handle is pinned where the loop owns it). func TestRun_ServeReadsErrorSurfaces(t *testing.T) { cat, _ := testCatalog(t) pinGenesis(t, cat) @@ -412,11 +412,6 @@ func TestRun_ServeReadsErrorSurfaces(t *testing.T) { require.Contains(t, err.Error(), "serve reads") require.NotErrorIs(t, err, context.Canceled, "a ServeReads error is restartable, not a clean shutdown") require.Equal(t, int32(1), core.openedCount.Load(), "core was started before serving") - - // The resume hot DB was closed on the error path (LOCK released): reopening it succeeds. - db, err := openHotDBForChunk(cat, chunk.IDFromLedger(chunk.FirstLedgerSeq), silentLogger()) - require.NoError(t, err, "the resume hot DB is reopenable — run released its LOCK") - require.NoError(t, db.Close()) } // run errors on a first start with an unavailable tip (restartable, no sentinel); From 44d66ae79e2570e61ccfd8281f59dcc01a3ca980 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Fri, 3 Jul 2026 00:57:19 -0400 Subject: [PATCH 52/55] fullhistory/ingest,hotchunk: unify hot metrics into one phase-keyed family (#3517302950) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the three overlapping hot signal families (HotLedgerTotal, the five IngestStage phase timings, HotItems) with ONE phase-keyed family, and fixes the mislabels the old shape carried. - hotchunk.IngestLedger now returns a single LedgerReport (per-phase samples + the phase that failed) instead of (LedgerCounts, LedgerPhases, error). Phases are a typed enum (Phase) indexed into a fixed-size array, so an out-of-table phase is unrepresentable. - MetricSink: HotItems + HotLedgerTotal collapse into HotPhase(phase, d, items, err). The per-ledger total is the SUM of the phase durations; a decode failure lands on PhaseExtract and a commit failure on PhaseCommit BY CONSTRUCTION (no more hot_commit_errors_total incrementing for any IngestLedger failure, no more whole-call duration doc'd as batch-only). Item volume folds onto the write phases. The "batch" pseudo-type and the hot IngestStage plumbing are retired. - PrometheusSink: one hot family (hot_phase_{duration_seconds,items_total, errors_total}{phase}) resolved into a [NumPhases] array (no nil-map emit). The cold side now enumerates its 8 real (data_type, stage) pairs instead of registering the 3×4 cross-product (4 series nothing emits). IngestStage is cold-only (tier label dropped). - HotService.Ingest emits one HotPhase per phase from the report: phases [0, Failed] on error (with zero items — nothing landed), all phases on success. Tests: TestHotService_EmitsEveryPhaseOnSuccess + TestHotService_CommitErrorLandsOnCommitPhase (the emission contract had none); removed the dead testSink residue (hotDataTypes, hotLedgerTotals, the empty P1-c banner). (#3517303118) --- .../internal/fullhistory/hotloop_test.go | 2 +- .../internal/fullhistory/ingest/events.go | 8 +- .../fullhistory/ingest/ingest_test.go | 143 ++++++--- .../internal/fullhistory/ingest/ledgers.go | 4 +- .../internal/fullhistory/ingest/metrics.go | 277 ++++++++---------- .../internal/fullhistory/ingest/service.go | 55 ++-- .../internal/fullhistory/ingest/txhash.go | 4 +- .../lifecycle/lifecycle_helpers_test.go | 2 +- .../pkg/stores/hotchunk/hotchunk.go | 119 +++++--- .../pkg/stores/hotchunk/hotchunk_test.go | 40 ++- 10 files changed, 358 insertions(+), 296 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index ff1d061fa..e1b540f7d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -165,7 +165,7 @@ func seedWatermark(t *testing.T, cat *catalog.Catalog, c chunk.ID, seq uint32) u t.Helper() db := openLiveHotDB(t, cat, c) for s := c.FirstLedger(); s <= seq; s++ { - _, _, err := db.IngestLedger(s, zeroTxLCMBytes(t, s)) + _, err := db.IngestLedger(s, zeroTxLCMBytes(t, s)) require.NoError(t, err) } require.NoError(t, db.Close()) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/events.go b/cmd/stellar-rpc/internal/fullhistory/ingest/events.go index c49f337bd..98be9f62e 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/events.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/events.go @@ -106,7 +106,7 @@ func (e *eventsCold) Finalize(ctx context.Context) error { e.metrics.emit(time.Since(start), err) return err } - e.metrics.sink.IngestStage(dataTypeEvents, tierCold, stageFinalize, time.Since(start), 0) + e.metrics.sink.IngestStage(dataTypeEvents, stageFinalize, time.Since(start), 0) e.metrics.emit(time.Since(start), nil) return nil } @@ -128,7 +128,7 @@ func (e *eventsCold) ingestSeq(seq uint32, lcm xdr.LedgerCloseMetaView) (int, er if err != nil { return 0, err } - e.metrics.sink.IngestStage(dataTypeEvents, tierCold, stageExtract, time.Since(estart), len(payloads)) + e.metrics.sink.IngestStage(dataTypeEvents, stageExtract, time.Since(estart), len(payloads)) startID := e.offsets.TotalEvents() if uint64(startID)+uint64(len(payloads)) > math.MaxUint32 { @@ -166,7 +166,7 @@ func (e *eventsCold) ingestSeq(seq uint32, lcm xdr.LedgerCloseMetaView) (int, er } writeDur += time.Since(wstart) } - e.metrics.sink.IngestStage(dataTypeEvents, tierCold, stageTermIndex, termDur, len(payloads)) + e.metrics.sink.IngestStage(dataTypeEvents, stageTermIndex, termDur, len(payloads)) // offsets.Append LAST — it is the commit point for the ledger. Its cost folds // into the write stage (rather than landing in the per-chunk total but in no @@ -177,7 +177,7 @@ func (e *eventsCold) ingestSeq(seq uint32, lcm xdr.LedgerCloseMetaView) (int, er //nolint:gosec // the overflow guard above proved startID+len(payloads) fits in uint32 oerr := e.offsets.Append(seq, uint32(len(payloads))) writeDur += time.Since(wstart) - e.metrics.sink.IngestStage(dataTypeEvents, tierCold, stageWrite, writeDur, len(payloads)) + e.metrics.sink.IngestStage(dataTypeEvents, stageWrite, writeDur, len(payloads)) if oerr != nil { return 0, fmt.Errorf("offsets append seq %d: %w", seq, oerr) } diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go index 8eb3182b0..903a1c257 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go @@ -14,6 +14,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" @@ -25,6 +26,7 @@ import ( "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/events" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/chunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/eventstore" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/ledger" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/txhash" ) @@ -35,9 +37,10 @@ const testPassphrase = "Public Global Stellar Network ; September 2015" // ───────────────────────── test metric sink ───────────────────────── -type hotItemCall struct { - dataType string - items int +type hotPhaseCall struct { + phase hotchunk.Phase + items int + err error } type coldCall struct { @@ -48,7 +51,6 @@ type coldCall struct { type stageCall struct { dataType string - tier string stage string items int } @@ -57,17 +59,16 @@ type stageCall struct { // use (the hot methods fire from the per-ledger ingestion goroutine). type testSink struct { mu sync.Mutex - hotItems []hotItemCall + hotPhases []hotPhaseCall coldIngests []coldCall stages []stageCall - hotLedgerTotals int coldChunkTotals int } -func (s *testSink) HotItems(dataType string, items int) { +func (s *testSink) HotPhase(phase hotchunk.Phase, _ time.Duration, items int, err error) { s.mu.Lock() defer s.mu.Unlock() - s.hotItems = append(s.hotItems, hotItemCall{dataType, items}) + s.hotPhases = append(s.hotPhases, hotPhaseCall{phase, items, err}) } func (s *testSink) ColdIngest(dataType string, _ time.Duration, items int, err error) { @@ -76,45 +77,52 @@ func (s *testSink) ColdIngest(dataType string, _ time.Duration, items int, err e s.coldIngests = append(s.coldIngests, coldCall{dataType, items, err}) } -func (s *testSink) HotLedgerTotal(_ time.Duration, _ error) { - s.mu.Lock() - defer s.mu.Unlock() - s.hotLedgerTotals++ -} - func (s *testSink) ColdChunkTotal(time.Duration) { s.mu.Lock() defer s.mu.Unlock() s.coldChunkTotals++ } -func (s *testSink) IngestStage(dataType, tier, stage string, _ time.Duration, items int) { +func (s *testSink) IngestStage(dataType, stage string, _ time.Duration, items int) { s.mu.Lock() defer s.mu.Unlock() - s.stages = append(s.stages, stageCall{dataType, tier, stage, items}) + s.stages = append(s.stages, stageCall{dataType, stage, items}) } -// stageCounts counts IngestStage calls keyed "dataType/tier/stage". +// stageCounts counts cold IngestStage calls keyed "dataType/stage". func (s *testSink) stageCounts() map[string]int { s.mu.Lock() defer s.mu.Unlock() m := map[string]int{} for _, c := range s.stages { - m[c.dataType+"/"+c.tier+"/"+c.stage]++ + m[c.dataType+"/"+c.stage]++ } return m } -func (s *testSink) hotDataTypes() map[string]int { +// hotPhaseItems returns the items reported per hot phase, keyed by phase. +func (s *testSink) hotPhaseItems() map[hotchunk.Phase]int { s.mu.Lock() defer s.mu.Unlock() - m := map[string]int{} - for _, c := range s.hotItems { - m[c.dataType]++ + m := map[hotchunk.Phase]int{} + for _, c := range s.hotPhases { + m[c.phase] += c.items } return m } +// hotPhaseErr returns the phase that carried a non-nil error, or (0,false) if none. +func (s *testSink) hotPhaseErr() (hotchunk.Phase, bool) { + s.mu.Lock() + defer s.mu.Unlock() + for _, c := range s.hotPhases { + if c.err != nil { + return c.phase, true + } + } + return 0, false +} + func (s *testSink) coldDataTypes() map[string]int { s.mu.Lock() defer s.mu.Unlock() @@ -688,14 +696,14 @@ func TestColdService_Success(t *testing.T) { // events now emits term_index/write for every ledger, and txhash's extract // spans its whole per-ledger Ingest. require.Equal(t, map[string]int{ - dataTypeLedgers + "/" + tierCold + "/" + stageWrite: 2, - dataTypeLedgers + "/" + tierCold + "/" + stageFinalize: 1, - dataTypeTxhash + "/" + tierCold + "/" + stageExtract: 2, - dataTypeTxhash + "/" + tierCold + "/" + stageFinalize: 1, - dataTypeEvents + "/" + tierCold + "/" + stageExtract: 2, - dataTypeEvents + "/" + tierCold + "/" + stageTermIndex: 2, - dataTypeEvents + "/" + tierCold + "/" + stageWrite: 2, - dataTypeEvents + "/" + tierCold + "/" + stageFinalize: 1, + dataTypeLedgers + "/" + stageWrite: 2, + dataTypeLedgers + "/" + stageFinalize: 1, + dataTypeTxhash + "/" + stageExtract: 2, + dataTypeTxhash + "/" + stageFinalize: 1, + dataTypeEvents + "/" + stageExtract: 2, + dataTypeEvents + "/" + stageTermIndex: 2, + dataTypeEvents + "/" + stageWrite: 2, + dataTypeEvents + "/" + stageFinalize: 1, }, sink.stageCounts()) // No double-emit: the deferred Close (after this body) must not add a second @@ -804,19 +812,16 @@ func TestPrometheusSink_Smoke(t *testing.T) { reg := prometheus.NewRegistry() require.NotPanics(t, func() { sink := NewPrometheusSink(reg, "test") - sink.HotItems(dataTypeLedgers, 1) - sink.HotItems(dataTypeEvents, 3) + // The five hot per-ledger phases: extract/commit carry no items, the write + // phases carry per-type volume; the commit phase exercises the error dimension. + sink.HotPhase(hotchunk.PhaseExtract, time.Millisecond, 0, nil) + sink.HotPhase(hotchunk.PhaseLedgers, time.Millisecond, 1, nil) + sink.HotPhase(hotchunk.PhaseTxhash, time.Millisecond, 5, nil) + sink.HotPhase(hotchunk.PhaseEvents, time.Millisecond, 3, nil) + sink.HotPhase(hotchunk.PhaseCommit, time.Millisecond, 0, errFailingCold) sink.ColdIngest(dataTypeTxhash, time.Second, 100, nil) - sink.HotLedgerTotal(time.Millisecond, nil) - sink.HotLedgerTotal(time.Millisecond, errFailingCold) // exercise the commit-error counter sink.ColdChunkTotal(time.Second) - // The five hot per-ledger phases (batch-scoped extract/commit + per-type write). - sink.IngestStage(dataTypeBatch, tierHot, stageExtract, time.Millisecond, 0) - sink.IngestStage(dataTypeLedgers, tierHot, stageWrite, time.Millisecond, 0) - sink.IngestStage(dataTypeTxhash, tierHot, stageWrite, time.Millisecond, 0) - sink.IngestStage(dataTypeEvents, tierHot, stageWrite, time.Millisecond, 0) - sink.IngestStage(dataTypeBatch, tierHot, stageCommit, time.Millisecond, 0) - sink.IngestStage(dataTypeEvents, tierCold, stageFinalize, time.Second, 0) + sink.IngestStage(dataTypeEvents, stageFinalize, time.Second, 0) }) mfs, err := reg.Gather() @@ -1045,7 +1050,61 @@ func TestWriteColdChunk_DrainStreamError_NoArtifact(t *testing.T) { // pkg/stores/txhash (cold_bin_test.go); these tests only cover the // ingester-level behavior on top of it. -// ───────────────────────── hot ingester failure path (P1-c) ───────────────────────── +// ───────────────────────── hot service emission ───────────────────────── + +func hotTestLogger() *supportlog.Entry { + l := supportlog.New() + l.SetLevel(logrus.ErrorLevel) + return l +} + +// TestHotService_EmitsEveryPhaseOnSuccess constructs a HotService over a real hot +// DB with a recording sink and asserts one successful ingest emits every phase +// once, the write phases carry per-type volume (extract/commit carry none), and no +// phase carries an error. +func TestHotService_EmitsEveryPhaseOnSuccess(t *testing.T) { + db, err := hotchunk.Open(t.TempDir(), chunk.ID(0), hotTestLogger()) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + sink := &testSink{} + svc := NewHotService(db, sink) + first := chunk.ID(0).FirstLedger() + raw, _, _ := marshalLCMWithEvent(t, first) // one tx, one event + require.NoError(t, svc.Ingest(context.Background(), first, xdr.LedgerCloseMetaView(raw))) + + require.Len(t, sink.hotPhases, int(hotchunk.NumPhases), "every phase emitted once on success") + items := sink.hotPhaseItems() + assert.Equal(t, 1, items[hotchunk.PhaseLedgers], "one ledger") + assert.Equal(t, 1, items[hotchunk.PhaseTxhash], "one tx hash") + assert.Equal(t, 1, items[hotchunk.PhaseEvents], "one event") + assert.Zero(t, items[hotchunk.PhaseExtract], "extract carries no items") + assert.Zero(t, items[hotchunk.PhaseCommit], "commit carries no items") + _, hadErr := sink.hotPhaseErr() + assert.False(t, hadErr, "success path carries no phase error") +} + +// TestHotService_CommitErrorLandsOnCommitPhase asserts a commit failure (a closed +// DB) surfaces the error on the commit phase — by construction, not by a +// separately-maintained label — and emits no items on the failure path. +func TestHotService_CommitErrorLandsOnCommitPhase(t *testing.T) { + db, err := hotchunk.Open(t.TempDir(), chunk.ID(0), hotTestLogger()) + require.NoError(t, err) + require.NoError(t, db.Close()) // closed => the batch commit fails + + sink := &testSink{} + svc := NewHotService(db, sink) + first := chunk.ID(0).FirstLedger() + raw, _, _ := marshalLCMWithEvent(t, first) + require.Error(t, svc.Ingest(context.Background(), first, xdr.LedgerCloseMetaView(raw))) + + phase, hadErr := sink.hotPhaseErr() + require.True(t, hadErr, "the failure must be reported on a phase") + assert.Equal(t, hotchunk.PhaseCommit, phase, "a commit failure lands on the commit phase") + for p, n := range sink.hotPhaseItems() { + assert.Zero(t, n, "no items on the failure path (phase %v)", p) + } +} // ───────────────────────── cold txhash .bin content (P1-d) ───────────────────────── diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go index 74457c13e..5acf01b91 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ledgers.go @@ -46,7 +46,7 @@ func (c *ledgerCold) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMe c.metrics.observe(time.Since(start), 0, err) // terminal: observe emits the per-ingester signal return fmt.Errorf("AppendLedger(seq=%d): %w", seq, err) } - c.metrics.sink.IngestStage(dataTypeLedgers, tierCold, stageWrite, time.Since(start), 1) + c.metrics.sink.IngestStage(dataTypeLedgers, stageWrite, time.Since(start), 1) c.appended = true c.metrics.observe(time.Since(start), 1, nil) return nil @@ -66,7 +66,7 @@ func (c *ledgerCold) Finalize(_ context.Context) error { c.metrics.emit(time.Since(start), err) return err } - c.metrics.sink.IngestStage(dataTypeLedgers, tierCold, stageFinalize, time.Since(start), 0) + c.metrics.sink.IngestStage(dataTypeLedgers, stageFinalize, time.Since(start), 0) c.metrics.emit(time.Since(start), nil) return nil } diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go index 08c4fe152..3e4e60933 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go @@ -4,38 +4,46 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk" ) // Data-type labels reported to a MetricSink. These match the per-type -// subdirectory names used on disk. +// subdirectory names used on disk. (The hot tier keys its per-ledger phases by +// hotchunk.Phase, not by data type — see MetricSink.HotPhase.) const ( dataTypeLedgers = "ledgers" dataTypeTxhash = "txhash" dataTypeEvents = "events" - // dataTypeBatch labels the hot per-ledger phases that are ledger-scoped rather - // than per data type — the shared extract walk and the batch commit — so they - // share one axis instead of being triple-counted across the three types. - dataTypeBatch = "batch" ) -// Tier labels reported to a MetricSink. -const ( - tierHot = "hot" - tierCold = "cold" -) +// Tier label reported to a MetricSink (cold stages; the hot phases have their own +// enum-keyed family). +const tierCold = "cold" -// Stage labels reported via MetricSink.IngestStage. These sit at the seams +// Cold stage labels reported via MetricSink.IngestStage. These sit at the seams // the rpc-hack bench collectors measured (per-stage extract / term-index / // store-write samples plus a per-chunk finish), so a CSV sink can reproduce // those reports from production ingesters without re-instrumenting. const ( stageExtract = "extract" // view → payloads / hashes derivation stageTermIndex = "term_index" // per-event term derivation + mirror update (events cold) - stageWrite = "write" // store write / pack append (cold) / queue-into-batch (hot) + stageWrite = "write" // store write / pack append stageFinalize = "finalize" // per-chunk commit (pack trailer, index build, .bin write) - stageCommit = "commit" // hot: the RocksDB batch write (WAL append + fsync + memtable) ) +// coldStagePairs is the set of (data_type, stage) pairs the cold ingesters +// actually emit — the eight real ones, not the 3×4 cross-product. A sink +// pre-resolves exactly these, so it registers no series no code path can feed. +// +//nolint:gochecknoglobals // fixed label set, read-only +var coldStagePairs = []struct{ dataType, stage string }{ + {dataTypeLedgers, stageWrite}, {dataTypeLedgers, stageFinalize}, + {dataTypeTxhash, stageExtract}, {dataTypeTxhash, stageFinalize}, + {dataTypeEvents, stageExtract}, {dataTypeEvents, stageTermIndex}, + {dataTypeEvents, stageWrite}, {dataTypeEvents, stageFinalize}, +} + // MetricSink receives ingest timing and volume signals. Ingesters report their // own per-call latency / item counts / errors (they know the item count); the // per-tier services report aggregate per-ledger (hot) and per-chunk (cold) @@ -43,51 +51,43 @@ const ( // a CSV recorder in benchmarks, or a test recorder — interchangeably. // // Implementations must be safe for concurrent use across ALL methods: the live -// hot ingestion loop reports HotItems/HotLedgerTotal from its own goroutine -// while the lifecycle may freeze several chunks concurrently (each its own -// WriteColdChunk), so the cold methods (ColdIngest, ColdChunkTotal) can likewise -// be called from several goroutines at once. +// hot ingestion loop reports HotPhase from its own goroutine while the lifecycle +// may freeze several chunks concurrently (each its own WriteColdChunk), so the +// cold methods (ColdIngest, ColdChunkTotal, IngestStage) can likewise be called +// from several goroutines at once. type MetricSink interface { - // HotItems reports the per-type volume of one HotService.Ingest: how many items - // (events, txhashes, or 1 ledger) that type contributed to the ledger's atomic - // batch. Emitted on the success path only — a failed atomic batch wrote nothing - // durably. There is deliberately no per-type hot DURATION or ERROR: the whole - // ledger commits as ONE synced batch (one fsync), and post-#18 a single shared - // ExtractLedgerEvents walk feeds both txhash and events, so neither timing nor - // an extraction failure is attributable per type — both live on HotLedgerTotal. - HotItems(dataType string, items int) + // HotPhase reports ONE phase of one hot ledger ingest — the single hot-tier + // signal family. It carries that phase's wall-clock, its item count (0 for the + // extract/commit phases, the per-type write volume for the write phases, on the + // success path), and its outcome (err is non-nil only on the phase that failed, + // so a decode failure lands on PhaseExtract and a commit failure on PhaseCommit + // by construction). The per-ledger total is the sum of the phase durations; the + // caller emits phases [0, Failed] on error and all phases on success. + HotPhase(phase hotchunk.Phase, d time.Duration, items int, err error) // ColdIngest reports one cold ingester's per-chunk total: the summed Ingest // wall-clock plus its Finalize, items the total items written for the chunk, // err the first error (nil on success). ColdIngest(dataType string, d time.Duration, items int, err error) - // HotLedgerTotal is the ONE batch-level signal per hot ledger: d is the - // wall-clock of the single atomic synced WriteBatch across all CFs, err its - // commit outcome (nil on success). It carries the hot tier's only honest - // per-ledger duration and error — attribution is batch-scoped, not per type. - HotLedgerTotal(d time.Duration, err error) // ColdChunkTotal reports the per-chunk wall-clock across all cold ingesters' // ingests plus their Finalizes (the ColdService lifetime). ColdChunkTotal(d time.Duration) - // IngestStage reports one ingester's per-stage wall-clock INSIDE an + // IngestStage reports one COLD ingester's per-stage wall-clock inside an // Ingest/Finalize call: stage is one of the stage* constants (extract, - // term_index, write, finalize), tier "hot" or "cold", items the stage's - // natural item count (0 where none applies). The whole-call HotLedgerTotal / - // ColdIngest signals above cannot be decomposed by a sink after the - // fact, so the per-stage granularity the bench reports need is exposed - // as its own signal — a sink that doesn't want it (production - // Prometheus, optionally) can no-op it. - IngestStage(dataType, tier, stage string, d time.Duration, items int) + // term_index, write, finalize), items the stage's natural item count (0 where + // none applies). The whole-call ColdIngest signal cannot be decomposed by a + // sink after the fact, so the per-stage granularity the bench reports need is + // exposed as its own signal — a sink that doesn't want it can no-op it. + IngestStage(dataType, stage string, d time.Duration, items int) } // NopSink is a MetricSink that discards everything. It is the default when a // caller passes a nil sink to a service or ingester. type NopSink struct{} -func (NopSink) HotItems(string, int) {} -func (NopSink) ColdIngest(string, time.Duration, int, error) {} -func (NopSink) HotLedgerTotal(time.Duration, error) {} -func (NopSink) ColdChunkTotal(time.Duration) {} -func (NopSink) IngestStage(string, string, string, time.Duration, int) {} +func (NopSink) HotPhase(hotchunk.Phase, time.Duration, int, error) {} +func (NopSink) ColdIngest(string, time.Duration, int, error) {} +func (NopSink) ColdChunkTotal(time.Duration) {} +func (NopSink) IngestStage(string, string, time.Duration, int) {} // orNop returns sink, or NopSink{} when sink is nil, so call sites never // nil-check before reporting. @@ -176,31 +176,9 @@ var ( coldStageBuckets = prometheus.ExponentialBuckets(0.001, 4, 12) ) -// ingestStages is the construction-time stage label set used to pre-resolve -// the per-(data_type, stage) children. -// -//nolint:gochecknoglobals // fixed label set, read-only -var ingestStages = []string{stageExtract, stageTermIndex, stageWrite, stageFinalize} - -// hotPhaseKeys is the fixed set of per-ledger phase children the hot ingest path -// reports via IngestStage(dataType, tierHot, stage): the shared extract walk and -// the batch commit under the batch pseudo-type, plus per-type queue-into-batch -// timings under stageWrite. Not the cold cross-product — the hot path has its own -// phase taxonomy — so the sink pre-resolves exactly these keys. -// -//nolint:gochecknoglobals // fixed label set, read-only -var hotPhaseKeys = []struct{ dataType, stage string }{ - {dataTypeBatch, stageExtract}, - {dataTypeLedgers, stageWrite}, - {dataTypeTxhash, stageWrite}, - {dataTypeEvents, stageWrite}, - {dataTypeBatch, stageCommit}, -} - -// ingestCollectors bundles the pre-resolved per-(data_type, tier) children. -// The label space is fixed at construction (three data types × two tiers), so -// resolving the children once removes the per-emit label-map allocation and -// hashed vector lookups from the hot per-ledger path. +// ingestCollectors bundles the pre-resolved per-cold-data-type children. The +// label space is fixed at construction, so resolving the children once removes +// the per-emit label-map allocation and hashed vector lookup. type ingestCollectors struct { duration prometheus.Observer items prometheus.Counter @@ -225,25 +203,22 @@ func (c ingestCollectors) observe(d time.Duration, items int, err error) { // passing it into the ingest drivers) is a follow-up — there is no full-history // ingest daemon startup path yet. This type only provides the registerable sink. type PrometheusSink struct { - // Per-type hot volume counters (HotItems), keyed by data type. The hot tier has - // no per-type duration or error — one atomic batch, one shared extraction walk — - // so unlike cold it needs only an item counter per type. - hotItems map[string]prometheus.Counter - // hotCommitErrors counts failed hot batch commits (HotLedgerTotal's err); it is - // batch-scoped (not per data type) because one fsync commits all CFs together. - hotCommitErrors prometheus.Counter - // Pre-resolved per-cold-ingester children, keyed by data type (duration - // histogram, items counter, errors counter). Producers draw their data_type - // from the same unexported constant set the map is built from, so a lookup can - // never miss — indexed directly, with no on-the-fly vector fallback. + // Hot per-ledger phases — the single hot signal family, one set of children per + // hotchunk.Phase, indexed by the phase value into a fixed-size ARRAY (not a map), + // so an out-of-table phase is a bounds panic at the index rather than a silent + // nil-map emit. The per-ledger total is the sum of hotPhaseDur; commit errors are + // hotPhaseErrs[PhaseCommit]; decode errors hotPhaseErrs[PhaseExtract]. + hotPhaseDur [hotchunk.NumPhases]prometheus.Observer + hotPhaseItems [hotchunk.NumPhases]prometheus.Counter + hotPhaseErrs [hotchunk.NumPhases]prometheus.Counter + // Pre-resolved per-cold-ingester children, keyed by data type. Producers draw + // their data_type from the same constant set the map is built from, so a lookup + // can never miss — indexed directly, no on-the-fly vector fallback. cold map[string]ingestCollectors - // Per-stage durations (IngestStage), pre-resolved per - // (data_type, stage) with per-tier buckets, keyed "dataType/stage". - hotStage map[string]prometheus.Observer + // Per-cold-stage durations, pre-resolved for the eight real (data_type, stage) + // pairs only (coldStagePairs), keyed "dataType/stage". coldStage map[string]prometheus.Observer - // Aggregate per-tier wall-clock: hot per-ledger batch, cold per-chunk - // service lifetime. Separate histograms so each tier gets fitting buckets. - hotLedgerTotal prometheus.Observer + // Aggregate per-chunk cold wall-clock (ColdService lifetime). coldChunkTotal prometheus.Observer } @@ -251,6 +226,25 @@ type PrometheusSink struct { // registry under namespace + the fullhistory_ingest subsystem. namespace is the // daemon convention value (interfaces.PrometheusNamespace). func NewPrometheusSink(registry *prometheus.Registry, namespace string) *PrometheusSink { + hotPhaseDurVec := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, Subsystem: metricsSubsystem, + Name: "hot_phase_duration_seconds", + Help: "per-ledger phase wall-clock (extract, ledgers, txhash, events, commit; the phases sum to the per-ledger total)", + Buckets: hotBuckets, + }, []string{"phase"}) + + hotPhaseItemsVec := prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, Subsystem: metricsSubsystem, + Name: "hot_phase_items_total", + Help: "items written per hot phase (the write phases carry per-type volume; extract/commit are 0)", + }, []string{"phase"}) + + hotPhaseErrsVec := prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, Subsystem: metricsSubsystem, + Name: "hot_phase_errors_total", + Help: "hot ledger failures by the phase that failed (decode->extract, commit->commit, by construction)", + }, []string{"phase"}) + coldDuration := prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: metricsSubsystem, Name: "cold_ingest_duration_seconds", @@ -258,30 +252,17 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh Buckets: coldBuckets, }, []string{"data_type"}) - ingestItems := prometheus.NewCounterVec(prometheus.CounterOpts{ + coldItems := prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: metricsSubsystem, - Name: "items_total", - Help: "items written per ingester (events, txhashes, or ledgers)", - }, []string{"data_type", "tier"}) - - ingestErrors := prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, Subsystem: metricsSubsystem, - Name: "errors_total", - Help: "ingester Ingest/Finalize errors", - }, []string{"data_type", "tier"}) - - hotLedgerTotal := prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: namespace, Subsystem: metricsSubsystem, - Name: "hot_ledger_duration_seconds", - Help: "per-ledger wall-clock of one HotService.Ingest (single atomic batch across all CFs)", - Buckets: hotBuckets, - }) + Name: "cold_items_total", + Help: "items written per cold ingester (events, txhashes, or ledgers)", + }, []string{"data_type"}) - hotCommitErrors := prometheus.NewCounter(prometheus.CounterOpts{ + coldErrors := prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: metricsSubsystem, - Name: "hot_commit_errors_total", - Help: "failed hot batch commits (batch-scoped: one fsync commits all CFs)", - }) + Name: "cold_errors_total", + Help: "cold ingester Ingest/Finalize errors", + }, []string{"data_type"}) coldChunkTotal := prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: metricsSubsystem, @@ -290,14 +271,6 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh Buckets: coldBuckets, }) - hotStageVec := prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, Subsystem: metricsSubsystem, - Name: "hot_stage_duration_seconds", - Help: "per-ledger phase wall-clock (batch/extract, {ledgers,txhash,events}/write, " + - "batch/commit; the phases sum to the per-ledger total)", - Buckets: hotBuckets, - }, []string{"data_type", "stage"}) - coldStageVec := prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: metricsSubsystem, Name: "cold_stage_duration_seconds", @@ -306,43 +279,41 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh Buckets: coldStageBuckets, }, []string{"data_type", "stage"}) - registry.MustRegister(coldDuration, ingestItems, ingestErrors, - hotLedgerTotal, hotCommitErrors, coldChunkTotal, hotStageVec, coldStageVec) + registry.MustRegister(hotPhaseDurVec, hotPhaseItemsVec, hotPhaseErrsVec, + coldDuration, coldItems, coldErrors, coldChunkTotal, coldStageVec) - hotItems := make(map[string]prometheus.Counter, 3) - cold := make(map[string]ingestCollectors, 3) - coldStage := make(map[string]prometheus.Observer, 3*len(ingestStages)) + sink := &PrometheusSink{ + cold: make(map[string]ingestCollectors, 3), + coldStage: make(map[string]prometheus.Observer, len(coldStagePairs)), + coldChunkTotal: coldChunkTotal, + } + // Hot phases: one child per phase, indexed by the phase value. + for p := hotchunk.Phase(0); p < hotchunk.NumPhases; p++ { + sink.hotPhaseDur[p] = hotPhaseDurVec.WithLabelValues(p.String()) + sink.hotPhaseItems[p] = hotPhaseItemsVec.WithLabelValues(p.String()) + sink.hotPhaseErrs[p] = hotPhaseErrsVec.WithLabelValues(p.String()) + } for _, dataType := range []string{dataTypeLedgers, dataTypeTxhash, dataTypeEvents} { - hotItems[dataType] = ingestItems.WithLabelValues(dataType, tierHot) - cold[dataType] = ingestCollectors{ + sink.cold[dataType] = ingestCollectors{ duration: coldDuration.WithLabelValues(dataType), - items: ingestItems.WithLabelValues(dataType, tierCold), - errors: ingestErrors.WithLabelValues(dataType, tierCold), + items: coldItems.WithLabelValues(dataType), + errors: coldErrors.WithLabelValues(dataType), } - for _, stage := range ingestStages { - coldStage[dataType+"/"+stage] = coldStageVec.WithLabelValues(dataType, stage) - } - } - // Hot phases are a fixed 5-key set (not the cold cross-product). - hotStage := make(map[string]prometheus.Observer, len(hotPhaseKeys)) - for _, k := range hotPhaseKeys { - hotStage[k.dataType+"/"+k.stage] = hotStageVec.WithLabelValues(k.dataType, k.stage) } - - return &PrometheusSink{ - hotItems: hotItems, - hotCommitErrors: hotCommitErrors, - cold: cold, - hotStage: hotStage, - coldStage: coldStage, - hotLedgerTotal: hotLedgerTotal, - coldChunkTotal: coldChunkTotal, + // Cold stages: only the eight real (data_type, stage) pairs. + for _, k := range coldStagePairs { + sink.coldStage[k.dataType+"/"+k.stage] = coldStageVec.WithLabelValues(k.dataType, k.stage) } + return sink } -func (p *PrometheusSink) HotItems(dataType string, items int) { +func (p *PrometheusSink) HotPhase(phase hotchunk.Phase, d time.Duration, items int, err error) { + p.hotPhaseDur[phase].Observe(d.Seconds()) if items > 0 { - p.hotItems[dataType].Add(float64(items)) + p.hotPhaseItems[phase].Add(float64(items)) + } + if err != nil { + p.hotPhaseErrs[phase].Inc() } } @@ -350,25 +321,13 @@ func (p *PrometheusSink) ColdIngest(dataType string, d time.Duration, items int, p.cold[dataType].observe(d, items, err) } -func (p *PrometheusSink) HotLedgerTotal(d time.Duration, err error) { - p.hotLedgerTotal.Observe(d.Seconds()) - if err != nil { - p.hotCommitErrors.Inc() - } -} - func (p *PrometheusSink) ColdChunkTotal(d time.Duration) { p.coldChunkTotal.Observe(d.Seconds()) } -// IngestStage records the per-stage duration into the tier's stage histogram. -// The per-stage item counts are not exported to Prometheus (the per-Ingest -// items_total already carries volume); they exist on the interface for the -// CSV bench sink. -func (p *PrometheusSink) IngestStage(dataType, tier, stage string, d time.Duration, _ int) { - resolved := p.hotStage - if tier == tierCold { - resolved = p.coldStage - } - resolved[dataType+"/"+stage].Observe(d.Seconds()) +// IngestStage records the per-stage cold duration. The per-stage item counts are +// not exported to Prometheus (cold_items_total already carries volume); they exist +// on the interface for the CSV bench sink. +func (p *PrometheusSink) IngestStage(dataType, stage string, d time.Duration, _ int) { + p.coldStage[dataType+"/"+stage].Observe(d.Seconds()) } diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go index bb5750132..ec0c317d7 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/service.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/service.go @@ -22,9 +22,9 @@ func errOrFirst(prev, cur error) error { } // HotService commits one ledger to the shared per-chunk hot DB as ONE atomic -// synced WriteBatch across all hot CFs (decision (a)) and emits per-ledger -// wall-clock + per-type volume signals. No fan-out — the three types are CFs of -// one RocksDB committing in one WriteBatch (hotchunk.DB.IngestLedger). +// synced WriteBatch across all hot CFs (decision (a)) and emits the single hot +// signal family: one HotPhase per hotchunk.Phase. No fan-out — the three types are +// CFs of one RocksDB committing in one WriteBatch (hotchunk.DB.IngestLedger). type HotService struct { db *hotchunk.DB sink MetricSink @@ -37,37 +37,34 @@ func NewHotService(db *hotchunk.DB, sink MetricSink) *HotService { } // Ingest commits lcm to the shared hot DB in one atomic synced WriteBatch -// (decision (a)) and emits the ledger's metrics. The batch OUTCOME is batch-scoped -// — HotLedgerTotal carries the whole-batch wall-clock and the commit error (one -// fsync commits all CFs, so there is no per-type commit error). Per-type HotItems -// reports VOLUME, on success. Per-PHASE timing is a separate axis: extract (the -// shared walk + shaping) and commit (the RocksDB write) are batch-scoped, the three -// queue steps per type — together they partition the per-ledger wall-clock, and -// commit surfaces the fsync-wait split that CPU profiles can't. +// (decision (a)) and emits one HotPhase per phase from the ledger report. Each +// phase carries its own wall-clock (the phases partition the per-ledger total), +// the write phases carry per-type item volume on success, and the outcome lands on +// the phase that failed BY CONSTRUCTION — a decode failure on PhaseExtract, a +// commit failure on PhaseCommit — so there is no mislabeled batch-scoped error. +// On failure only phases [0, Failed] ran, so only those are emitted (and with zero +// items — nothing landed durably); on success every phase is emitted. func (s *HotService) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMetaView) error { - start := time.Now() - counts, phases, err := s.db.IngestLedger(seq, lcm) - s.sink.HotLedgerTotal(time.Since(start), err) - if err == nil { - s.sink.HotItems(dataTypeLedgers, counts.Ledgers) - s.sink.HotItems(dataTypeTxhash, counts.Txhash) - s.sink.HotItems(dataTypeEvents, counts.Events) - s.reportPhases(phases) + rep, err := s.db.IngestLedger(seq, lcm) + + last := hotchunk.NumPhases - 1 + if err != nil { + last = rep.Failed + } + for p := hotchunk.Phase(0); p <= last; p++ { + items := rep.Phases[p].Items + var perr error + if err != nil { + items = 0 // the failure path committed nothing durably + if p == rep.Failed { + perr = err + } + } + s.sink.HotPhase(p, rep.Phases[p].Dur, items, perr) } return err } -// reportPhases emits the per-ledger phase timings via the hot IngestStage plumbing: -// the shared extract + commit under the batch pseudo-type, the three queue steps -// under each type's stageWrite. Item counts are 0 — HotItems already carries volume. -func (s *HotService) reportPhases(p hotchunk.LedgerPhases) { - s.sink.IngestStage(dataTypeBatch, tierHot, stageExtract, p.Extract, 0) - s.sink.IngestStage(dataTypeLedgers, tierHot, stageWrite, p.Ledgers, 0) - s.sink.IngestStage(dataTypeTxhash, tierHot, stageWrite, p.Txhash, 0) - s.sink.IngestStage(dataTypeEvents, tierHot, stageWrite, p.Events, 0) - s.sink.IngestStage(dataTypeBatch, tierHot, stageCommit, p.Commit, 0) -} - // ColdService drives a set of ColdIngesters for one chunk: sequential per-ledger // Ingest, then Finalize on each. It times from the first Ingest (or, if none ran, // from the Finalize/Close call) and emits the aggregate ColdChunkTotal exactly diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go b/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go index 364e6dbe7..7d98b0a70 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/txhash.go @@ -76,7 +76,7 @@ func (t *txhashCold) Ingest(_ context.Context, seq uint32, lcm xdr.LedgerCloseMe // write is the finalize stage; there is no separate cold write stage for // txhash.) d := time.Since(start) - t.metrics.sink.IngestStage(dataTypeTxhash, tierCold, stageExtract, d, len(hashes)) + t.metrics.sink.IngestStage(dataTypeTxhash, stageExtract, d, len(hashes)) t.metrics.observe(d, len(hashes), nil) return nil } @@ -93,7 +93,7 @@ func (t *txhashCold) Finalize(_ context.Context) error { }) err := txhash.WriteColdBin(t.binPath, t.entries) if err == nil { - t.metrics.sink.IngestStage(dataTypeTxhash, tierCold, stageFinalize, time.Since(start), len(t.entries)) + t.metrics.sink.IngestStage(dataTypeTxhash, stageFinalize, time.Since(start), len(t.entries)) } t.metrics.emit(time.Since(start), err) return err diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go index 13059bb14..72c8471f6 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle_helpers_test.go @@ -97,7 +97,7 @@ func ingestFullHotChunk(t *testing.T, cat *catalog.Catalog, c chunk.ID) { } else { raw = zeroTxLCMBytes(t, seq) } - _, _, err := db.IngestLedger(seq, xdr.LedgerCloseMetaView(raw)) + _, err := db.IngestLedger(seq, xdr.LedgerCloseMetaView(raw)) require.NoError(t, err) } require.NoError(t, db.Close()) // release the write handle (boundary handoff) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index f9ac2d850..7aa8d7d06 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -157,30 +157,60 @@ func (d *DB) MaxCommittedSeq() (uint32, bool, error) { return d.ledger.LastSeq() } -// LedgerCounts reports how many items each data type contributed to one -// IngestLedger call, so the caller can emit per-type volume metrics. -type LedgerCounts struct { - Ledgers int - Txhash int - Events int +// Phase enumerates the ordered phases of one IngestLedger call. It is a typed +// index into a fixed-size array (LedgerReport.Phases), so an out-of-table phase is +// unrepresentable — no string label to mistype and no map lookup to nil-panic in a +// sink. The phases partition the per-ledger wall-clock: +// - PhaseExtract: the shared ExtractLedgerEvents walk + txhash-entry build + +// event shaping (all pre-batch — every decode failure lands here by construction); +// - PhaseLedgers/PhaseTxhash/PhaseEvents: each facade's queue-into-batch step; +// - PhaseCommit: the RocksDB batch write (WAL append + fsync + memtable) = the +// whole Batch call minus the three queue steps — the fsync wait pprof can't see. +type Phase uint8 + +const ( + PhaseExtract Phase = iota + PhaseLedgers + PhaseTxhash + PhaseEvents + PhaseCommit + // NumPhases is the array size; it is not itself a phase. + NumPhases +) + +// String is the metric label for a phase. +func (p Phase) String() string { + switch p { + case PhaseExtract: + return "extract" + case PhaseLedgers: + return "ledgers" + case PhaseTxhash: + return "txhash" + case PhaseEvents: + return "events" + case PhaseCommit: + return "commit" + default: + return "unknown" + } } -// LedgerPhases reports the per-phase wall-clock of one IngestLedger call so the -// caller can attribute hot per-ledger CPU vs IO without re-instrumenting: -// - Extract: the shared ExtractLedgerEvents walk + txhash-entry build + event -// shaping (all pre-batch); -// - Ledgers/Txhash/Events: each facade's queue-into-batch step; -// - Commit: the RocksDB batch write (WAL append + fsync + memtable) = the whole -// Batch call minus the three queue steps — the fsync wait pprof can't see. -// -// The phases sum to ~the whole call (minus the tiny post-commit mirror apply); -// fields are zero for a phase that an error return preempted. -type LedgerPhases struct { - Extract time.Duration - Ledgers time.Duration - Txhash time.Duration - Events time.Duration - Commit time.Duration +// PhaseSample is one phase's wall-clock and item count (Items is 0 where a phase +// handles no per-type volume — extract and commit). +type PhaseSample struct { + Dur time.Duration + Items int +} + +// LedgerReport is the single result of IngestLedger: the per-phase samples, plus +// the phase that failed when the call returns a non-nil error. Phases that never +// ran (after a failure) keep their zero sample; the caller emits phases up to and +// including Failed on error, and all phases on success. +type LedgerReport struct { + Phases [NumPhases]PhaseSample + // Failed is meaningful only when IngestLedger returns a non-nil error. + Failed Phase } // IngestLedger commits ONE ledger as a SINGLE atomic synced WriteBatch across all @@ -191,11 +221,8 @@ type LedgerPhases struct { // lcm is a borrowed zero-copy view; every extractor copies what it retains, so // the view need not outlive this call. Store.Batch's lifecycle RLock + checkOpen // is the authoritative closed-store guard, so there is no separate pre-check here. -func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts, LedgerPhases, error) { - var ( - counts LedgerCounts - phases LedgerPhases - ) +func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerReport, error) { + var rep LedgerReport // Pre-extract anything that can fail BEFORE opening the batch, so a decode // error rejects the ledger without a half-built batch. @@ -213,26 +240,30 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts extractStart := time.Now() txEvents, err := sdkingest.ExtractLedgerEvents(lcm) if err != nil { - return counts, phases, fmt.Errorf("extract ledger events seq %d: %w", seq, err) + rep.Failed = PhaseExtract + return rep, fmt.Errorf("extract ledger events seq %d: %w", seq, err) } txEntries := make([]txhash.Entry, len(txEvents)) for i := range txEvents { txEntries[i] = txhash.Entry{Hash: txEvents[i].Hash, LedgerSeq: seq} } - counts.Txhash = len(txEntries) closedAt, err := lcm.LedgerCloseTime() if err != nil { - return counts, phases, fmt.Errorf("ledger close time seq %d: %w", seq, err) + rep.Failed = PhaseExtract + return rep, fmt.Errorf("ledger close time seq %d: %w", seq, err) } // A pre-Soroban ledger yields zero payloads, no error. payloads, err := events.PayloadsFromLedgerEvents(txEvents, seq, closedAt) if err != nil { - return counts, phases, fmt.Errorf("shape events seq %d: %w", seq, err) + rep.Failed = PhaseExtract + return rep, fmt.Errorf("shape events seq %d: %w", seq, err) } - counts.Events = len(payloads) - counts.Ledgers = 1 - phases.Extract = time.Since(extractStart) + rep.Phases[PhaseExtract].Dur = time.Since(extractStart) + // Per-type write volume lives on the write phases (emitted on success). + rep.Phases[PhaseLedgers].Items = 1 + rep.Phases[PhaseTxhash].Items = len(txEntries) + rep.Phases[PhaseEvents].Items = len(payloads) // The events facade validates + marshals inside the batch callback (so a // rejected ledger never leaves committed rows) and returns the post-commit @@ -241,41 +272,49 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerCounts // step is timed individually; Commit (below) is the whole Batch minus those — // the RocksDB write (WAL append + fsync + memtable). var applyEvents func() + // A batch error not attributed to a specific queue step below is the commit + // itself (the RocksDB write); a queue-step error narrows Failed to its phase. + failed := PhaseCommit batchStart := time.Now() cerr := d.store.Batch(func(b *rocksdb.BatchWriter) error { ls := time.Now() if err := d.ledger.AddLedgerToBatch(b, ledger.Entry{Seq: seq, Bytes: []byte(lcm)}); err != nil { + failed = PhaseLedgers return fmt.Errorf("queue ledger seq %d: %w", seq, err) } - phases.Ledgers = time.Since(ls) + rep.Phases[PhaseLedgers].Dur = time.Since(ls) ts := time.Now() if len(txEntries) > 0 { if err := d.txhash.AddEntriesToBatch(b, txEntries); err != nil { + failed = PhaseTxhash return fmt.Errorf("queue tx hashes seq %d: %w", seq, err) } } - phases.Txhash = time.Since(ts) + rep.Phases[PhaseTxhash].Dur = time.Since(ts) es := time.Now() apply, err := d.events.IngestLedgerToBatch(b, seq, payloads) if err != nil { + failed = PhaseEvents return fmt.Errorf("queue events seq %d: %w", seq, err) } - phases.Events = time.Since(es) + rep.Phases[PhaseEvents].Dur = time.Since(es) applyEvents = apply return nil }) if cerr != nil { - return counts, phases, fmt.Errorf("commit ledger %d to chunk %s: %w", seq, d.chunkID, cerr) + rep.Failed = failed + return rep, fmt.Errorf("commit ledger %d to chunk %s: %w", seq, d.chunkID, cerr) } // The three queue steps are strictly nested inside the Batch call (monotonic // clock), so Commit is the non-negative remainder: the RocksDB write itself. - phases.Commit = time.Since(batchStart) - phases.Ledgers - phases.Txhash - phases.Events + rep.Phases[PhaseCommit].Dur = time.Since(batchStart) - + rep.Phases[PhaseLedgers].Dur - rep.Phases[PhaseTxhash].Dur - rep.Phases[PhaseEvents].Dur // Batch is durable — now and only now apply the events mirror/offsets update. applyEvents() - return counts, phases, nil + return rep, nil } // hotLedgerStream is a ledgerbackend.LedgerStream over a ledger.HotStore, so the diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go index 776d626ce..d2ca307c4 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go @@ -39,6 +39,15 @@ func openTestDB(t *testing.T) *DB { return db } +// assertWriteItems checks the per-type write volume the report carries on the +// write phases (the item counts that used to be LedgerCounts). +func assertWriteItems(t *testing.T, rep LedgerReport, ledgers, txhash, events int) { + t.Helper() + assert.Equal(t, ledgers, rep.Phases[PhaseLedgers].Items, "ledgers items") + assert.Equal(t, txhash, rep.Phases[PhaseTxhash].Items, "txhash items") + assert.Equal(t, events, rep.Phases[PhaseEvents].Items, "events items") +} + func TestOpen_ValidatesInputs(t *testing.T) { _, err := Open("", chunk.ID(0), silentLogger()) require.ErrorIs(t, err, stores.ErrInvalidConfig) @@ -82,13 +91,13 @@ func TestIngestLedger_AllCFsAdvanceTogether(t *testing.T) { rawA, hashA, termA := lcmWithEvent(t, first) rawB, hashB, _ := lcmWithEvent(t, first+1) - counts, _, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(rawA)) + repA, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(rawA)) require.NoError(t, err) - assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 1, Events: 1}, counts) + assertWriteItems(t, repA, 1, 1, 1) - counts, _, err = db.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawB)) + repB, err := db.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawB)) require.NoError(t, err) - assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 1, Events: 1}, counts) + assertWriteItems(t, repB, 1, 1, 1) // ledgers CF. gotA, err := db.Ledgers().GetLedgerRaw(first) @@ -130,7 +139,7 @@ func TestIngestLedger_RejectedLedgerPersistsNothingAcrossAnyCF(t *testing.T) { badSeq := chunkID.LastLedger() + 1 raw, hash, term := lcmWithEvent(t, badSeq) - _, _, err := db.IngestLedger(badSeq, xdr.LedgerCloseMetaView(raw)) + _, err := db.IngestLedger(badSeq, xdr.LedgerCloseMetaView(raw)) require.Error(t, err) require.ErrorIs(t, err, eventstore.ErrLedgerOutOfRange) @@ -166,7 +175,7 @@ func TestIngestLedger_MidBatchCommitFailurePersistsNothing(t *testing.T) { // Commit one good ledger so there is a known watermark, then close the DB. rawGood, hashGood, _ := lcmWithEvent(t, first) - _, _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(rawGood)) + _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(rawGood)) require.NoError(t, err) require.NoError(t, db.Close()) @@ -184,7 +193,7 @@ func TestIngestLedger_MidBatchCommitFailurePersistsNothing(t *testing.T) { // store: the commit fails, and nothing for that ledger persists anywhere. require.NoError(t, db2.Close()) rawNext, hashNext, _ := lcmWithEvent(t, first+1) - _, _, err = db2.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawNext)) + _, err = db2.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawNext)) require.Error(t, err) // Reopen a third time: the failed ledger left NO trace in any CF, and the @@ -260,9 +269,9 @@ func TestIngestLedger_WritesEveryHotType(t *testing.T) { db := openTestDB(t) raw, hash, term := lcmWithEvent(t, first) - counts, _, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) + rep, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) require.NoError(t, err) - assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 1, Events: 1}, counts) + assertWriteItems(t, rep, 1, 1, 1) got, err := db.Ledgers().GetLedgerRaw(first) require.NoError(t, err) @@ -301,10 +310,9 @@ func TestIngestLedger_EventlessTxStillIndexesHash(t *testing.T) { raw, err := lcm.MarshalBinary() require.NoError(t, err) - counts, _, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) + rep, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) require.NoError(t, err) - assert.Equal(t, LedgerCounts{Ledgers: 1, Txhash: 2, Events: 1}, counts, - "both applied txs' hashes indexed (event-less included); only the eventful tx contributed an event") + assertWriteItems(t, rep, 1, 2, 1) // both hashes indexed (event-less included); one event // Both hashes resolve in the txhash CF to this ledger. for _, h := range hashes { @@ -327,7 +335,7 @@ func TestReopen_RecoversEventsMirror(t *testing.T) { db, err := Open(dir, chunkID, silentLogger()) require.NoError(t, err) raw, _, _ := lcmWithEvent(t, first) - _, _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) + _, err = db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) require.NoError(t, err) require.NoError(t, db.Close()) @@ -350,7 +358,7 @@ func TestOpenReadOnly_ReadsCommittedAndRejectsWrites(t *testing.T) { db, err := Open(dir, chunkID, silentLogger()) require.NoError(t, err) for _, seq := range []uint32{first, first + 1} { - _, _, ierr := db.IngestLedger(seq, xdr.LedgerCloseMetaView(zeroTxLCM(t, seq))) + _, ierr := db.IngestLedger(seq, xdr.LedgerCloseMetaView(zeroTxLCM(t, seq))) require.NoError(t, ierr) } require.NoError(t, db.Close()) @@ -366,7 +374,7 @@ func TestOpenReadOnly_ReadsCommittedAndRejectsWrites(t *testing.T) { assert.Equal(t, first+1, seq, "read-only handle sees the committed data") // A write through the read-only handle must fail — the freeze never mutates. - _, _, err = ro.IngestLedger(first+2, xdr.LedgerCloseMetaView(zeroTxLCM(t, first+2))) + _, err = ro.IngestLedger(first+2, xdr.LedgerCloseMetaView(zeroTxLCM(t, first+2))) require.Error(t, err, "read-only DB must reject writes") } @@ -381,7 +389,7 @@ func TestIngestLedger_ClosedDBFails(t *testing.T) { require.NoError(t, db.Close()) raw := zeroTxLCM(t, chunkID.FirstLedger()) - _, _, err = db.IngestLedger(chunkID.FirstLedger(), xdr.LedgerCloseMetaView(raw)) + _, err = db.IngestLedger(chunkID.FirstLedger(), xdr.LedgerCloseMetaView(raw)) require.ErrorIs(t, err, rocksdb.ErrStoreClosed) } From 79704d422b8631e06804a0e198590262eee34503 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Fri, 3 Jul 2026 01:09:08 -0400 Subject: [PATCH 53/55] fullhistory: delete SeqValidatedCursor, enforce in-order at the source (#3517303043) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The LedgerStream contract already promises in-order delivery and the sources enforce it structurally; the shared cursor re-checked it redundantly. - Delete SeqValidatedCursor + ValidatedLedger. - hotLedgerStream (the one source that could be mis-keyed — it wraps the sole writer of recent history) now key-checks its own keyspace (a gap is a defect) and self-bounds an unbounded range at its committed frontier, mirroring packStream. packStream is already positionally contiguous; the SDK backends validate their own output. - drain and the hot ingestion loop each consume the raw stream on a local sequence counter, keeping their overrun / completeness / boundary logic without re-parsing every view's sequence. Tests: hotchunk TestSource_RejectsGap + TestSource_SelfBoundsUnboundedRange pin the relocated guard; the drain per-seq-guard tests (which fed an artificially mis-ordered stream) were deleted with the cursor; the two drain stream-error tests assert the new chunk-scoped wrapping. Full -short suite + non-short E2E (85s) + golangci --new-from-rev all green. --- .../internal/fullhistory/hotloop.go | 78 ++++++------- .../internal/fullhistory/ingest/driver.go | 73 +++--------- .../fullhistory/ingest/ingest_test.go | 105 ++---------------- .../internal/fullhistory/ingest/ingester.go | 4 +- .../internal/fullhistory/ingest/metrics.go | 20 ++-- .../pkg/stores/hotchunk/hotchunk.go | 27 ++++- .../pkg/stores/hotchunk/hotchunk_test.go | 69 ++++++++++-- 7 files changed, 161 insertions(+), 215 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index 1e2dd2c4d..4d276fe12 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -9,6 +9,7 @@ import ( "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" supportlog "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/geometry" @@ -149,57 +150,56 @@ func runIngestionLoop(ctx context.Context, cfg ingestionLoopConfig) (err error) // rebuilds it for the reopened chunk DB below. hotService := ingest.NewHotService(hotDB, cfg.Sink) - // One continuous sequence-validated stream from the resume ledger. The cursor - // restores the per-ledger sequence guard the cold drain also uses (defense in - // depth against a mis-keyed source writing the sole copy of recent history). A - // stream / decode / sequence error ends the loop for the daemon to classify. - raw := cfg.Stream.RawLedgers(ctx, ledgerbackend.UnboundedRange(cfg.Resume)) - for vl, verr := range ingest.SeqValidatedCursor(raw, cfg.Resume) { + // One continuous stream from the resume ledger, consumed on a local sequence + // counter. The in-order contract is enforced at the SOURCE — captive core (and + // every SDK backend) validates its own output — so the loop trusts the counter + // rather than re-parsing each view's sequence. A stream / decode error ends the + // loop for the daemon to classify. + seq := cfg.Resume + for raw, verr := range cfg.Stream.RawLedgers(ctx, ledgerbackend.UnboundedRange(cfg.Resume)) { if verr != nil { return fmt.Errorf("ingestion stream: %w", verr) } - // One atomic synced WriteBatch across all hot CFs (via hotDB.IngestLedger), - // reporting per-type LedgerCounts to the sink. - if ierr := hotService.Ingest(ctx, vl.Seq, vl.View); ierr != nil { - return fmt.Errorf("ingest ledger %d: %w", vl.Seq, ierr) + // One atomic synced WriteBatch across all hot CFs (via hotDB.IngestLedger). + if ierr := hotService.Ingest(ctx, seq, xdr.LedgerCloseMetaView(raw)); ierr != nil { + return fmt.Errorf("ingest ledger %d: %w", seq, ierr) } // The ingestion loop owns the last-committed gauge: this is the TRUE // committed ledger (mid-chunk included), one atomic gauge set per ledger. // The tick must not touch it — its chunk-aligned value would regress it. - metrics.LastCommitted(vl.Seq) + metrics.LastCommitted(seq) // Chunk boundary: this seq is the chunk's last ledger. - closed := chunk.IDFromLedger(vl.Seq) - if vl.Seq != closed.LastLedger() { - continue - } - next := closed + 1 - // Handoff fence: close the write handle BEFORE the next chunk's key is - // created (that key is what makes THIS chunk complete to a tick, which may - // then freeze and discard its hot DB — no writer may hold it then). - if cerr := hotDB.Close(); cerr != nil { - hotDB = nil // closed (failed) — do not double-close in defer - return fmt.Errorf("close hot DB at boundary chunk %s: %w", closed, cerr) - } - hotDB = nil // released; reopen below republishes it for the defer + if closed := chunk.IDFromLedger(seq); seq == closed.LastLedger() { + next := closed + 1 + // Handoff fence: close the write handle BEFORE the next chunk's key is + // created (that key is what makes THIS chunk complete to a tick, which may + // then freeze and discard its hot DB — no writer may hold it then). + if cerr := hotDB.Close(); cerr != nil { + hotDB = nil // closed (failed) — do not double-close in defer + return fmt.Errorf("close hot DB at boundary chunk %s: %w", closed, cerr) + } + hotDB = nil // released; reopen below republishes it for the defer - nextDB, oerr := openHotDBForChunk(cfg.Catalog, next, cfg.Logger) - if oerr != nil { - return fmt.Errorf("open hot DB for chunk %s at boundary: %w", next, oerr) + nextDB, oerr := openHotDBForChunk(cfg.Catalog, next, cfg.Logger) + if oerr != nil { + return fmt.Errorf("open hot DB for chunk %s at boundary: %w", next, oerr) + } + hotDB = nextDB + hotService = ingest.NewHotService(hotDB, cfg.Sink) + // next's key (created inside openHotDBForChunk) moved the partition; only + // now publish the completed chunk to the lifecycle. + cfg.Boundary.Publish(closed) + + // Boundary observability (the woken tick reports the freeze/discard/prune). + metrics.ChunkBoundary() + cfg.Logger.WithField("closed_chunk", closed.String()). + WithField("next_chunk", next.String()). + WithField("last_ledger", seq). + Info("streaming: ingestion chunk boundary — handed off to lifecycle") } - hotDB = nextDB - hotService = ingest.NewHotService(hotDB, cfg.Sink) - // next's key (created inside openHotDBForChunk) moved the partition; only now - // publish the completed chunk to the lifecycle. - cfg.Boundary.Publish(closed) - - // Boundary observability (the woken tick reports the freeze/discard/prune). - metrics.ChunkBoundary() - cfg.Logger.WithField("closed_chunk", closed.String()). - WithField("next_chunk", next.String()). - WithField("last_ledger", vl.Seq). - Info("streaming: ingestion chunk boundary — handed off to lifecycle") + seq++ } // The unbounded production stream ends only on ctx cancellation or a source // error, both surfaced as the cursor's error element above. Falling through here diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go index e8734937e..7c73ad0f3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/driver.go @@ -27,73 +27,32 @@ func closeColdAll(ings []ColdIngester, err error) error { return err } -// ValidatedLedger is one sequence-validated ledger from a raw stream: its -// verified sequence and the borrowed view (valid only until the next iteration -// step, per the LedgerStream contract). -type ValidatedLedger struct { - Seq uint32 - View xdr.LedgerCloseMetaView -} - -// SeqValidatedCursor adapts a raw ledger stream into contiguous, sequence-checked -// ledgers starting at `from`: for each yielded frame it reads the view's own -// LedgerSequence() and rejects a gap, duplicate, or out-of-order ledger before -// handing it on. Both the cold drain and the hot ingestion loop consume it, so the -// sole writer of recent history never trusts an injected source blindly (the SDK -// backend also validates its own output — this is defense-in-depth, a zero-copy -// header read). A source error, a decode error, or a non-contiguous sequence is -// yielded as the error element and ends iteration; the view is borrowed. -func SeqValidatedCursor( - ledgers iter.Seq2[[]byte, error], from uint32, -) iter.Seq2[ValidatedLedger, error] { - return func(yield func(ValidatedLedger, error) bool) { - seq := from - for raw, serr := range ledgers { - if serr != nil { - yield(ValidatedLedger{Seq: seq}, fmt.Errorf("RawLedgers(%d): %w", seq, serr)) - return - } - lcm := xdr.LedgerCloseMetaView(raw) - actual, aerr := lcm.LedgerSequence() - if aerr != nil { - yield(ValidatedLedger{Seq: seq}, fmt.Errorf("ledger sequence at expected %d: %w", seq, aerr)) - return - } - if actual != seq { - yield(ValidatedLedger{Seq: seq}, fmt.Errorf("yielded ledger %d, expected %d", actual, seq)) - return - } - if !yield(ValidatedLedger{Seq: seq, View: lcm}, nil) { - return - } - seq++ - } - } -} - -// drain feeds each of the chunk's raw ledgers (as a validated view) to the -// service, then verifies the full [first,last] range was consumed — for cold this -// runs before Finalize, so a short stream never finalizes a truncated artifact. -// Cancellation is the iterator's job (RawLedgers errors on canceled ctx), so no -// ctx poll here. The per-ledger sequence guard lives in the shared cursor. +// drain feeds each of the chunk's raw ledgers (as a borrowed view) to the +// service on a local sequence counter, then verifies the full [first,last] range +// was consumed — for cold this runs before Finalize, so a short stream never +// finalizes a truncated artifact. The in-order contract is enforced at the SOURCE +// (packStream reads positionally by key; hotLedgerStream key-checks its own +// keyspace; the SDK backends validate their own output), so drain trusts the +// counter rather than re-parsing every view's sequence. Cancellation is the +// iterator's job (RawLedgers errors on a canceled ctx), so there is no ctx poll +// here. func drain(ctx context.Context, ledgers iter.Seq2[[]byte, error], chunkID chunk.ID, svc *ColdService) error { first, last := chunkID.FirstLedger(), chunkID.LastLedger() seq := first - for vl, verr := range SeqValidatedCursor(ledgers, first) { - if verr != nil { - return fmt.Errorf("ingest: stream for chunk %d: %w", uint32(chunkID), verr) + for raw, serr := range ledgers { + if serr != nil { + return fmt.Errorf("ingest: stream for chunk %d: %w", uint32(chunkID), serr) } // Reject a stream that runs PAST the chunk before ingesting out-of-chunk. - // The cursor already validated vl.Seq is contiguous; this bounds it above. - // All in-repo sources bound themselves; this guards custom iterators. - if vl.Seq > last { + // All in-repo sources self-bound; this guards a custom iterator. + if seq > last { return fmt.Errorf("ingest: stream for chunk %d yielded a ledger past %d (chunk overrun)", uint32(chunkID), last) } - if err := svc.Ingest(ctx, vl.Seq, vl.View); err != nil { + if err := svc.Ingest(ctx, seq, xdr.LedgerCloseMetaView(raw)); err != nil { return err } - seq = vl.Seq + 1 + seq++ } if seq != last+1 { return fmt.Errorf("ingest: stream for chunk %d ended at %d, expected through %d", uint32(chunkID), seq-1, last) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go index 903a1c257..033ea5f45 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingest_test.go @@ -7,7 +7,6 @@ import ( "iter" "os" "path/filepath" - "strconv" "sync" "testing" "time" @@ -394,29 +393,6 @@ func marshalV0LCM(t *testing.T, seq uint32) []byte { return raw } -// seqStream is a ledgerbackend.LedgerStream that yields LCMs for an explicit -// list of ledger sequences (in order), regardless of the requested range. It -// models a backend that hands back a duplicate / out-of-order / wrong-but- -// right-count sequence, exercising the drain seq guard. -type seqStream struct { - t *testing.T - seqs []uint32 -} - -var _ ledgerbackend.LedgerStream = (*seqStream)(nil) - -func (s *seqStream) RawLedgers( - _ context.Context, _ ledgerbackend.Range, _ ...ledgerbackend.StreamOption, -) iter.Seq2[[]byte, error] { - return func(yield func([]byte, error) bool) { - for _, seq := range s.seqs { - if !yield(marshalLCM(s.t, seq), nil) { - return - } - } - } -} - // errAtSeqStream yields valid LCMs until it reaches errAtSeq, where it yields // (nil, err) — modeling a backend that fails mid-stream. Used to exercise the // drain RawLedgers error path. @@ -947,77 +923,17 @@ func TestWriteColdChunk_EventsCold_Readback(t *testing.T) { require.Equal(t, uint64(len(evSeqs)), bm.GetCardinality()) } -// ───────────────────────── drain seq guard (P0-1) ───────────────────────── - -// TestWriteColdChunk_OutOfOrderSeq_NoArtifact feeds a stream that yields a ledger out -// of expected order (the second ledger repeats the first's seq — right total -// count, wrong sequence). drain must reject it with the mismatch error before -// any Finalize, and leave no cold artifact behind. -func TestWriteColdChunk_OutOfOrderSeq_NoArtifact(t *testing.T) { - chunkID := chunk.ID(0) - first := chunkID.FirstLedger() - last := chunkID.LastLedger() - coldDir := t.TempDir() - logger := testLogger() - - // Build a full-length seq list, then corrupt the second entry to a - // duplicate of the first: same count as a valid stream, wrong order. - seqs := make([]uint32, 0, last-first+1) - for s := first; s <= last; s++ { - seqs = append(seqs, s) - } - require.GreaterOrEqual(t, len(seqs), 2) - seqs[1] = seqs[0] // duplicate/out-of-order while keeping the count intact - - stream := &seqStream{t: t, seqs: seqs} - err := WriteColdChunk( - context.Background(), logger, chunkID, rawChunk(stream, chunkID), - coldDirsAt(coldDir, chunkID), nil, Config{Ledgers: true}, - ) - require.Error(t, err) - require.Contains(t, err.Error(), "yielded ledger") - require.Contains(t, err.Error(), "expected") - - // No finalized artifact: the deferred Close dropped the partial pack. - path := packPath(filepath.Join(coldDir, dataTypeLedgers), chunkID) - _, statErr := os.Stat(path) - require.True(t, os.IsNotExist(statErr), "expected no cold artifact at %s, stat err: %v", path, statErr) -} - -// TestDrain_TxhashSeqGuard asserts the guard also fires on the txhash path, -// where a wrong-but-right-count sequence would otherwise be silently absorbed -// (each ledger keys on its own LCM seq). -func TestDrain_TxhashSeqGuard(t *testing.T) { - chunkID := chunk.ID(0) - first := chunkID.FirstLedger() - last := chunkID.LastLedger() - coldDir := t.TempDir() - logger := testLogger() - - seqs := make([]uint32, 0, last-first+1) - for s := first; s <= last; s++ { - seqs = append(seqs, s) - } - require.GreaterOrEqual(t, len(seqs), 2) - // Corrupt the SECOND ledger so at least one valid ledger is ingested - // before the guard fires. - seqs[1] += 100 - - err := WriteColdChunk( - context.Background(), logger, chunkID, rawChunk(&seqStream{t: t, seqs: seqs}, chunkID), - coldDirsAt(coldDir, chunkID), nil, Config{Txhash: true}, - ) - require.Error(t, err) - require.Contains(t, err.Error(), "yielded ledger") - - binPath := txhashBinPath(filepath.Join(coldDir, dataTypeTxhash)) - _, statErr := os.Stat(binPath) - require.True(t, os.IsNotExist(statErr), "expected no .bin at %s, stat err: %v", binPath, statErr) -} +// ───────────────────────── drain stream errors ───────────────────────── +// +// The per-seq order guard the shared cursor used to run in drain moved to the +// SOURCE (packStream reads positionally; hotLedgerStream key-checks its keyspace, +// see TestSource_RejectsGap; the SDK backends validate their own output), so drain +// keeps only its overrun + completeness checks on a local counter. The tests that +// fed an artificially mis-ordered stream to drain were deleted with the cursor. // TestWriteColdChunk_DrainStreamError_NoArtifact exercises the drain mid-stream error // path: the backend yields valid ledgers, then hands back (nil, err) at a seq in -// the middle of the chunk. drain must wrap the error with RawLedgers + the seq, +// the middle of the chunk. drain must propagate the error (wrapped with the chunk), // short-circuit before Finalize (so no cold artifact is committed), and the // deferred Close must drop the partial. func TestWriteColdChunk_DrainStreamError_NoArtifact(t *testing.T) { @@ -1036,8 +952,7 @@ func TestWriteColdChunk_DrainStreamError_NoArtifact(t *testing.T) { ) require.Error(t, err) require.ErrorIs(t, err, wantErr, "the backend error must propagate") - require.Contains(t, err.Error(), "RawLedgers", "error wraps RawLedgers") - require.Contains(t, err.Error(), strconv.FormatUint(uint64(failAt), 10), "error names the failing seq") + require.Contains(t, err.Error(), "stream for chunk", "error wraps the drained chunk") // Finalize never ran → no finalized artifact; deferred Close dropped the partial. path := packPath(filepath.Join(coldDir, dataTypeLedgers), chunkID) @@ -1454,7 +1369,7 @@ func TestWriteColdChunk_LazySourceFirstReadError(t *testing.T) { ) require.Error(t, err) require.ErrorIs(t, err, wantErr) - require.Contains(t, err.Error(), "RawLedgers", "the error surfaces from drain's stream pull") + require.Contains(t, err.Error(), "stream for chunk", "the error surfaces from drain's stream pull") // Finalize never committed → no finalized pack (Close dropped the partial). path := packPath(filepath.Join(coldDir, dataTypeLedgers), chunkID) diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go b/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go index a70069f4d..ad312520d 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/ingester.go @@ -21,8 +21,8 @@ import ( // artifact; implementations are encouraged to latch the failure and refuse // (eventsCold does). // -// Input: seq is the cursor-validated ledger sequence of lcm (the shared -// SeqValidatedCursor has already checked contiguity), and lcm is a zero-copy +// Input: seq is the ledger sequence of lcm on drain's contiguous counter (the +// in-order contract is enforced at the source), and lcm is a zero-copy // xdr.LedgerCloseMetaView over the source stream's BORROWED buffer, valid only for // the current iteration step — an implementation must copy any bytes it retains. // ColdService drives the per-ledger Ingest calls sequentially, so each view is diff --git a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go index 3e4e60933..8b9952e6b 100644 --- a/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go +++ b/cmd/stellar-rpc/internal/fullhistory/ingest/metrics.go @@ -17,10 +17,6 @@ const ( dataTypeEvents = "events" ) -// Tier label reported to a MetricSink (cold stages; the hot phases have their own -// enum-keyed family). -const tierCold = "cold" - // Cold stage labels reported via MetricSink.IngestStage. These sit at the seams // the rpc-hack bench collectors measured (per-stage extract / term-index / // store-write samples plus a per-chunk finish), so a CSV sink can reproduce @@ -38,10 +34,14 @@ const ( // //nolint:gochecknoglobals // fixed label set, read-only var coldStagePairs = []struct{ dataType, stage string }{ - {dataTypeLedgers, stageWrite}, {dataTypeLedgers, stageFinalize}, - {dataTypeTxhash, stageExtract}, {dataTypeTxhash, stageFinalize}, - {dataTypeEvents, stageExtract}, {dataTypeEvents, stageTermIndex}, - {dataTypeEvents, stageWrite}, {dataTypeEvents, stageFinalize}, + {dataTypeLedgers, stageWrite}, + {dataTypeLedgers, stageFinalize}, + {dataTypeTxhash, stageExtract}, + {dataTypeTxhash, stageFinalize}, + {dataTypeEvents, stageExtract}, + {dataTypeEvents, stageTermIndex}, + {dataTypeEvents, stageWrite}, + {dataTypeEvents, stageFinalize}, } // MetricSink receives ingest timing and volume signals. Ingesters report their @@ -229,7 +229,7 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh hotPhaseDurVec := prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: metricsSubsystem, Name: "hot_phase_duration_seconds", - Help: "per-ledger phase wall-clock (extract, ledgers, txhash, events, commit; the phases sum to the per-ledger total)", + Help: "per-ledger phase wall-clock (extract/ledgers/txhash/events/commit; phases sum to the per-ledger total)", Buckets: hotBuckets, }, []string{"phase"}) @@ -288,7 +288,7 @@ func NewPrometheusSink(registry *prometheus.Registry, namespace string) *Prometh coldChunkTotal: coldChunkTotal, } // Hot phases: one child per phase, indexed by the phase value. - for p := hotchunk.Phase(0); p < hotchunk.NumPhases; p++ { + for p := range hotchunk.NumPhases { sink.hotPhaseDur[p] = hotPhaseDurVec.WithLabelValues(p.String()) sink.hotPhaseItems[p] = hotPhaseItemsVec.WithLabelValues(p.String()) sink.hotPhaseErrs[p] = hotPhaseErrsVec.WithLabelValues(p.String()) diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index 7aa8d7d06..bfb24f2a4 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -329,16 +329,30 @@ var _ ledgerbackend.LedgerStream = (*hotLedgerStream)(nil) // yields BORROWED buffers (valid only to the next step); the drain loop consumes // each fully before the next yield, so the borrow is safe. ctx cancellation is // observed between ledgers (the LedgerStream contract drain relies on). +// +// It enforces the LedgerStream in-order contract at the source (so the shared +// cursor could be deleted): the hot store is the SOLE writer of recent history, so +// a gap in its keyspace is a real defect, caught here by a key-derived seq check +// (no XDR parse). An unbounded range self-bounds at the store's committed frontier +// (LastSeq), mirroring packStream, so callers can pass UnboundedRange(from). func (st *hotLedgerStream) RawLedgers( ctx context.Context, r ledgerbackend.Range, _ ...ledgerbackend.StreamOption, ) iter.Seq2[[]byte, error] { return func(yield func([]byte, error) bool) { - // The freeze always passes a bounded chunk range; assert it. + to := r.To() if !r.Bounded() { - yield(nil, fmt.Errorf("hotLedgerStream requires a bounded range, got unbounded from %d", r.From())) - return + maxSeq, ok, err := st.store.LastSeq() + if err != nil { + yield(nil, fmt.Errorf("hotLedgerStream: read committed frontier: %w", err)) + return + } + if !ok { + return // empty store: nothing to yield + } + to = maxSeq } - for e, ierr := range st.store.IterateLedgers(r.From(), r.To()) { + expected := r.From() + for e, ierr := range st.store.IterateLedgers(r.From(), to) { if cerr := ctx.Err(); cerr != nil { yield(nil, cerr) return @@ -347,9 +361,14 @@ func (st *hotLedgerStream) RawLedgers( yield(nil, ierr) return } + if e.Seq != expected { + yield(nil, fmt.Errorf("hotLedgerStream: gap at seq %d, expected %d", e.Seq, expected)) + return + } if !yield(e.Bytes, nil) { return } + expected++ } } } diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go index d2ca307c4..7485979e8 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" "github.com/stellar/go-stellar-sdk/keypair" "github.com/stellar/go-stellar-sdk/network" supportlog "github.com/stellar/go-stellar-sdk/support/log" @@ -40,12 +41,14 @@ func openTestDB(t *testing.T) *DB { } // assertWriteItems checks the per-type write volume the report carries on the -// write phases (the item counts that used to be LedgerCounts). -func assertWriteItems(t *testing.T, rep LedgerReport, ledgers, txhash, events int) { +// write phases (the item counts that used to be LedgerCounts). Every fixture +// commits exactly one ledger with one event, so only the txhash count (one per +// applied tx) varies across callers. +func assertWriteItems(t *testing.T, rep LedgerReport, txhash int) { t.Helper() - assert.Equal(t, ledgers, rep.Phases[PhaseLedgers].Items, "ledgers items") + assert.Equal(t, 1, rep.Phases[PhaseLedgers].Items, "ledgers items") assert.Equal(t, txhash, rep.Phases[PhaseTxhash].Items, "txhash items") - assert.Equal(t, events, rep.Phases[PhaseEvents].Items, "events items") + assert.Equal(t, 1, rep.Phases[PhaseEvents].Items, "events items") } func TestOpen_ValidatesInputs(t *testing.T) { @@ -93,11 +96,11 @@ func TestIngestLedger_AllCFsAdvanceTogether(t *testing.T) { repA, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(rawA)) require.NoError(t, err) - assertWriteItems(t, repA, 1, 1, 1) + assertWriteItems(t, repA, 1) repB, err := db.IngestLedger(first+1, xdr.LedgerCloseMetaView(rawB)) require.NoError(t, err) - assertWriteItems(t, repB, 1, 1, 1) + assertWriteItems(t, repB, 1) // ledgers CF. gotA, err := db.Ledgers().GetLedgerRaw(first) @@ -261,6 +264,56 @@ func TestSharedBatch_DirectRocksAbortAcrossCFs(t *testing.T) { // package, so no production accessor is needed). func storeOf(db *DB) *rocksdb.Store { return db.store } +// TestSource_SelfBoundsUnboundedRange confirms the freeze source (hotLedgerStream) +// yields the store's committed ledgers in order and self-bounds an UNBOUNDED range +// at the committed frontier (mirroring packStream), so drain can pass +// UnboundedRange(from) rather than a pre-computed bound. +func TestSource_SelfBoundsUnboundedRange(t *testing.T) { + db := openTestDB(t) + first := chunk.ID(0).FirstLedger() + for i := range uint32(3) { + _, err := db.IngestLedger(first+i, xdr.LedgerCloseMetaView(zeroTxLCM(t, first+i))) + require.NoError(t, err) + } + + var got []uint32 + for raw, err := range db.Source().RawLedgers(context.Background(), ledgerbackend.UnboundedRange(first)) { + require.NoError(t, err) + seq, serr := xdr.LedgerCloseMetaView(raw).LedgerSequence() + require.NoError(t, serr) + got = append(got, seq) + } + require.Equal(t, []uint32{first, first + 1, first + 2}, got, "self-bounds at the frontier, in order") +} + +// TestSource_RejectsGap pins the source-side in-order guard that replaced the +// shared cursor: a gap in the hot store's keyspace (the sole writer of recent +// history) is a real defect and must surface as an error, not a silent skip. +func TestSource_RejectsGap(t *testing.T) { + db := openTestDB(t) + first := chunk.ID(0).FirstLedger() + // Seed the ledgers CF directly with a GAP (first, first+2), bypassing + // IngestLedger's contiguity so the source-level guard is what's exercised. + require.NoError(t, storeOf(db).Batch(func(b *rocksdb.BatchWriter) error { + for _, s := range []uint32{first, first + 2} { + if err := db.Ledgers().AddLedgerToBatch(b, ledger.Entry{Seq: s, Bytes: []byte("x")}); err != nil { + return err + } + } + return nil + })) + + var lastErr error + for _, err := range db.Source().RawLedgers(context.Background(), ledgerbackend.BoundedRange(first, first+2)) { + if err != nil { + lastErr = err + break + } + } + require.Error(t, lastErr) + require.Contains(t, lastErr.Error(), "gap") +} + // TestIngestLedger_WritesEveryHotType confirms the hot tier always writes all // three hot data types; per-type disabling is not a supported hot DB mode. func TestIngestLedger_WritesEveryHotType(t *testing.T) { @@ -271,7 +324,7 @@ func TestIngestLedger_WritesEveryHotType(t *testing.T) { raw, hash, term := lcmWithEvent(t, first) rep, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) require.NoError(t, err) - assertWriteItems(t, rep, 1, 1, 1) + assertWriteItems(t, rep, 1) got, err := db.Ledgers().GetLedgerRaw(first) require.NoError(t, err) @@ -312,7 +365,7 @@ func TestIngestLedger_EventlessTxStillIndexesHash(t *testing.T) { rep, err := db.IngestLedger(first, xdr.LedgerCloseMetaView(raw)) require.NoError(t, err) - assertWriteItems(t, rep, 1, 2, 1) // both hashes indexed (event-less included); one event + assertWriteItems(t, rep, 2) // both hashes indexed (event-less included); one event // Both hashes resolve in the txhash CF to this ledger. for _, h := range hashes { From 3dcd6914af3cda5bb9f88928be95614741148fe9 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Fri, 3 Jul 2026 11:48:32 -0400 Subject: [PATCH 54/55] fullhistory: partial-duration on failed hot phase + restore lifecycle op retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two round-4 follow-ups on this batch's own changes: - hotchunk.IngestLedger stamped the failed phase's Dur only on the success path, so an error was emitted with a zero-duration sample (a commit that blocks 30s then fails showed the error but not the 30s). Stamp time.Since(...) on the failed phase before every error return — partial duration is signal, matching RunBackfill's "reported even on failure" (#3517613508). - Restore the per-op retry the design carried (3 attempts / 5s pause) that was lost when the lifecycle loop joined run()'s errgroup: a transient discard/prune failure (busy file, slow fsync) now retries in place instead of canceling ingestion and forcing a whole-daemon restart (relaunching captive core) for a retryable, idempotent file op. runOps wraps each op in a bounded constant backoff (ctx-abortable); Config gains OpRetryAttempts/OpRetryBackoff with defaults; a zero-value Config runs each op once (#3517561326). Tests: TestRunOps_{RetriesTransientThenSucceeds,GivesUpAfterAttempts, CtxCancelStopsBeforeOp,ZeroConfigRunsOnce}. lifecycle + hotchunk -short green; golangci --new-from-rev clean. --- .../fullhistory/lifecycle/lifecycle.go | 48 +++++++++++++--- .../fullhistory/lifecycle/runops_test.go | 56 +++++++++++++++++++ .../pkg/stores/hotchunk/hotchunk.go | 22 ++++++-- 3 files changed, 113 insertions(+), 13 deletions(-) create mode 100644 cmd/stellar-rpc/internal/fullhistory/lifecycle/runops_test.go diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go index c0d42ef81..3046a034c 100644 --- a/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/lifecycle.go @@ -6,6 +6,8 @@ import ( "sync/atomic" "time" + "github.com/cenkalti/backoff/v4" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/backfill" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/catalog" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/fullhistory/observability" @@ -33,24 +35,52 @@ type Config struct { // RetentionChunks bounds the sliding retention floor's width. 0 disables the // sliding floor (the fixed earliest-ledger floor alone applies). RetentionChunks uint32 + + // OpRetryAttempts / OpRetryBackoff bound the per-op retry the discard/prune + // sweeps use (see runOps). Zero values fall back to defaults in + // WithLifecycleDefaults. + OpRetryAttempts int + OpRetryBackoff time.Duration } -// WithLifecycleDefaults returns a copy with the embedded ExecConfig defaults -// applied. Called once at startup before launching the loop. +const ( + defaultOpRetryAttempts = 3 + defaultOpRetryBackoff = 5 * time.Second +) + +// WithLifecycleDefaults returns a copy with the embedded ExecConfig defaults and +// the op-retry defaults applied. Called once at startup before launching the loop. func (cfg Config) WithLifecycleDefaults() Config { cfg.ExecConfig = cfg.WithDefaults() + if cfg.OpRetryAttempts < 1 { + cfg.OpRetryAttempts = defaultOpRetryAttempts + } + if cfg.OpRetryBackoff <= 0 { + cfg.OpRetryBackoff = defaultOpRetryBackoff + } return cfg } -// runOps runs each op in order, returning the first error. It checks ctx between -// ops so a shutdown mid-scan stops promptly without starting the next storage op; -// the ctx error is surfaced up through Loop for supervise to classify as clean. -func runOps(ctx context.Context, ops []func() error) error { +// runOps runs each op in order, retrying a failed op a bounded number of times on +// a fixed pause before giving up. The discard/prune ops are idempotent file +// deletions, so a transient failure (a busy file, a slow fsync) is exactly the +// retryable kind — retrying in place avoids canceling ingestion through the shared +// errgroup and forcing a whole-daemon restart (which relaunches captive core) for +// a retryable file operation. It checks ctx between ops (and the backoff aborts on +// ctx cancellation) so a shutdown mid-scan stops promptly; the ctx error surfaces +// up through Loop for supervise to classify as clean. +func runOps(ctx context.Context, cfg Config, ops []func() error) error { + // A zero-value Config (no WithLifecycleDefaults, e.g. a test harness) runs each + // op exactly once. + attempts := max(cfg.OpRetryAttempts, 1) for _, op := range ops { if err := ctx.Err(); err != nil { return err } - if err := op(); err != nil { + // attempts total tries == 1 initial + (attempts-1) retries, fixed pause. + //nolint:gosec // attempts >= 1, so attempts-1 >= 0 + bo := backoff.WithMaxRetries(backoff.NewConstantBackOff(cfg.OpRetryBackoff), uint64(attempts-1)) + if err := backoff.Retry(op, backoff.WithContext(bo, ctx)); err != nil { return err } } @@ -109,7 +139,7 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu if err != nil { return fmt.Errorf("eligible discard ops: %w", err) } - if err := runOps(ctx, discardOps); err != nil { + if err := runOps(ctx, cfg, discardOps); err != nil { return fmt.Errorf("discard op: %w", err) } metrics.Discard(len(discardOps), time.Since(discardStart)) @@ -130,7 +160,7 @@ func runLifecycle(ctx context.Context, cfg Config, cat *catalog.Catalog, lastChu if err != nil { return fmt.Errorf("eligible prune ops: %w", err) } - if err := runOps(ctx, pruneOps); err != nil { + if err := runOps(ctx, cfg, pruneOps); err != nil { return fmt.Errorf("prune op: %w", err) } metrics.Prune(prunedArtifacts, time.Since(pruneStart)) diff --git a/cmd/stellar-rpc/internal/fullhistory/lifecycle/runops_test.go b/cmd/stellar-rpc/internal/fullhistory/lifecycle/runops_test.go new file mode 100644 index 000000000..a3f3eb8b1 --- /dev/null +++ b/cmd/stellar-rpc/internal/fullhistory/lifecycle/runops_test.go @@ -0,0 +1,56 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// runOps retries a failed (idempotent) op a bounded number of times on a fixed +// pause before giving up, so a transient sweep failure doesn't cancel ingestion +// and force a whole-daemon restart. + +func TestRunOps_RetriesTransientThenSucceeds(t *testing.T) { + cfg := Config{OpRetryAttempts: 3, OpRetryBackoff: time.Millisecond} + calls := 0 + op := func() error { + calls++ + if calls < 3 { + return errors.New("busy file") + } + return nil + } + require.NoError(t, runOps(context.Background(), cfg, []func() error{op})) + require.Equal(t, 3, calls, "two transient failures retried, third try succeeds") +} + +func TestRunOps_GivesUpAfterAttempts(t *testing.T) { + cfg := Config{OpRetryAttempts: 2, OpRetryBackoff: time.Millisecond} + calls := 0 + op := func() error { calls++; return errors.New("permanent") } + require.Error(t, runOps(context.Background(), cfg, []func() error{op})) + require.Equal(t, 2, calls, "attempts total tries (1 initial + 1 retry), then gives up") +} + +func TestRunOps_CtxCancelStopsBeforeOp(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + cfg := Config{OpRetryAttempts: 3, OpRetryBackoff: time.Hour} + calls := 0 + op := func() error { calls++; return errors.New("x") } + require.ErrorIs(t, runOps(ctx, cfg, []func() error{op}), context.Canceled) + require.Zero(t, calls, "a canceled ctx stops before running the op") +} + +// A zero-value Config (no WithLifecycleDefaults) runs each op exactly once — no +// retry, no panic on the zero backoff — so a test harness that builds Config +// directly keeps the pre-retry behavior. +func TestRunOps_ZeroConfigRunsOnce(t *testing.T) { + calls := 0 + op := func() error { calls++; return errors.New("boom") } + require.Error(t, runOps(context.Background(), Config{}, []func() error{op})) + require.Equal(t, 1, calls, "zero-config = single attempt") +} diff --git a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go index bfb24f2a4..477b31ecf 100644 --- a/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/pkg/stores/hotchunk/hotchunk.go @@ -237,9 +237,14 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerReport // LCMViewToPayloads. The atomic batch below serializes only the commit; the // extractors are independent and could run concurrently into the same batch if // catch-up profiling ever demands it — sequential is right at live cadence. + // Every failure below stamps the failed phase's PARTIAL duration before + // returning — a phase that blocked and then failed is signal (mirrors + // RunBackfill's "reported even on failure"), so the error is never emitted with + // a zero-duration sample. extractStart := time.Now() txEvents, err := sdkingest.ExtractLedgerEvents(lcm) if err != nil { + rep.Phases[PhaseExtract].Dur = time.Since(extractStart) rep.Failed = PhaseExtract return rep, fmt.Errorf("extract ledger events seq %d: %w", seq, err) } @@ -250,12 +255,14 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerReport closedAt, err := lcm.LedgerCloseTime() if err != nil { + rep.Phases[PhaseExtract].Dur = time.Since(extractStart) rep.Failed = PhaseExtract return rep, fmt.Errorf("ledger close time seq %d: %w", seq, err) } // A pre-Soroban ledger yields zero payloads, no error. payloads, err := events.PayloadsFromLedgerEvents(txEvents, seq, closedAt) if err != nil { + rep.Phases[PhaseExtract].Dur = time.Since(extractStart) rep.Failed = PhaseExtract return rep, fmt.Errorf("shape events seq %d: %w", seq, err) } @@ -279,6 +286,7 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerReport cerr := d.store.Batch(func(b *rocksdb.BatchWriter) error { ls := time.Now() if err := d.ledger.AddLedgerToBatch(b, ledger.Entry{Seq: seq, Bytes: []byte(lcm)}); err != nil { + rep.Phases[PhaseLedgers].Dur = time.Since(ls) failed = PhaseLedgers return fmt.Errorf("queue ledger seq %d: %w", seq, err) } @@ -287,6 +295,7 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerReport ts := time.Now() if len(txEntries) > 0 { if err := d.txhash.AddEntriesToBatch(b, txEntries); err != nil { + rep.Phases[PhaseTxhash].Dur = time.Since(ts) failed = PhaseTxhash return fmt.Errorf("queue tx hashes seq %d: %w", seq, err) } @@ -296,6 +305,7 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerReport es := time.Now() apply, err := d.events.IngestLedgerToBatch(b, seq, payloads) if err != nil { + rep.Phases[PhaseEvents].Dur = time.Since(es) failed = PhaseEvents return fmt.Errorf("queue events seq %d: %w", seq, err) } @@ -303,14 +313,18 @@ func (d *DB) IngestLedger(seq uint32, lcm xdr.LedgerCloseMetaView) (LedgerReport applyEvents = apply return nil }) + // Commit is the whole Batch call minus the three queue steps: the RocksDB write + // (WAL append + fsync + memtable). Stamp it whether the batch succeeded or the + // commit itself failed (all queue steps ran) — a slow-then-failed commit is + // signal. A queue-step failure already stamped its own partial above. + if failed == PhaseCommit { + rep.Phases[PhaseCommit].Dur = time.Since(batchStart) - + rep.Phases[PhaseLedgers].Dur - rep.Phases[PhaseTxhash].Dur - rep.Phases[PhaseEvents].Dur + } if cerr != nil { rep.Failed = failed return rep, fmt.Errorf("commit ledger %d to chunk %s: %w", seq, d.chunkID, cerr) } - // The three queue steps are strictly nested inside the Batch call (monotonic - // clock), so Commit is the non-negative remainder: the RocksDB write itself. - rep.Phases[PhaseCommit].Dur = time.Since(batchStart) - - rep.Phases[PhaseLedgers].Dur - rep.Phases[PhaseTxhash].Dur - rep.Phases[PhaseEvents].Dur // Batch is durable — now and only now apply the events mirror/offsets update. applyEvents() From e7eb46da9d0d26610a2704edc12979e08e01cbf4 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Fri, 3 Jul 2026 12:49:53 -0400 Subject: [PATCH 55/55] fullhistory: open resume hot DB before serving reads (#3517545236, part A) Restore the design's startup order for the hot tier: run() opens the resume chunk's hot DB BEFORE ServeReads and hands the handle to the ingestion loop, whose first deferred statement takes ownership of the close. So a broken hot tier fails startup instead of serving behind a crash-looping loop. The leak-safety the earlier fix bought is preserved: the deferred close still sits ahead of any early return. run() owns the close only until the errgroup launch (loopOwnsDB flips there); after that the loop's defer owns it and g.Wait joins before run returns, so there is no window where neither owns the handle. An error between the open and the launch (OpenCore / ServeReads) returns via run()'s defer, which closes it. Restarts re-enter run() from the top, so this stays the single initial-open site; the loop still reopens at each boundary. Part B (eager core start before serve) is NOT a fullhistory-side change: the SDK LedgerStream is RawLedgers-only and deliberately starts core on the first pull with no Start/PrepareRange hook, so eager start lands if the SDK stream grows one (or with #772's real core wiring). OpenCore is already called before ServeReads. Tests: TestRun_OpensHotDBAndCoreBeforeServe (ServeReads observes the resume chunk already "ready" + core opened); TestRun_ServeReadsErrorSurfaces restores the now-meaningful reopen assertion (run opened the DB, closed it on the error path). loopConfig opens the resume DB the way run() does. Root -short + non-short E2E (85s) + golangci --new-from-rev all green. --- .../internal/fullhistory/hotloop.go | 29 +++++++------ .../internal/fullhistory/hotloop_test.go | 26 +++++++----- .../internal/fullhistory/startup.go | 28 +++++++++++-- .../internal/fullhistory/startup_test.go | 42 +++++++++++++++++-- 4 files changed, 92 insertions(+), 33 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop.go b/cmd/stellar-rpc/internal/fullhistory/hotloop.go index 4d276fe12..3b1ac2487 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop.go @@ -90,12 +90,15 @@ type boundaryPublisher interface { Publish(c chunk.ID) } -// ingestionLoopConfig bundles the ingestion loop's dependencies. The loop opens -// the resume chunk's hot DB itself from Catalog + Resume, so there is no hot-DB -// handle to thread in (and no cross-call ownership gap to leak through). +// ingestionLoopConfig bundles the ingestion loop's dependencies. run() opens the +// resume chunk's hot DB (HotDB) BEFORE serving reads — so a broken hot tier fails +// startup instead of serving behind a crash-looping loop — and hands the open +// handle in; the loop's first deferred statement takes ownership of the close, and +// it reopens the DB itself at every boundary (Catalog + Logger). type ingestionLoopConfig struct { Stream ledgerbackend.LedgerStream Resume uint32 + HotDB *hotchunk.DB Catalog *catalog.Catalog Boundary boundaryPublisher Logger *supportlog.Entry @@ -126,18 +129,14 @@ type ingestionLoopConfig struct { func runIngestionLoop(ctx context.Context, cfg ingestionLoopConfig) (err error) { metrics := observability.MetricsOrNop(cfg.Metrics) - // Open the resume chunk's hot DB HERE, so the open and its deferred close are - // adjacent in one function — no cross-call ownership gap for a transient open - // failure to leak the handle (and its RocksDB LOCK) through. The loop trusts the - // resume point passed in (run() derived it from the same durable state); there is - // nothing to re-derive or assert. The loop is this DB's single writer and reopens - // it at every boundary; the defer closes whatever handle is live on any exit (the - // boundary handoff already closed every prior chunk's DB), and no writer races the - // close (the loop has stopped on every exit path). - hotDB, err := openHotDBForChunk(cfg.Catalog, chunk.IDFromLedger(cfg.Resume), cfg.Logger) - if err != nil { - return fmt.Errorf("open resume hot tier for ledger %d: %w", cfg.Resume, err) - } + // Take ownership of the resume hot DB run() opened (before serving reads) as the + // loop's FIRST statement, so the deferred close sits ahead of any early return — + // no ownership gap for a transient failure to leak the handle (and its RocksDB + // LOCK) through. The loop is this DB's single writer and reopens it at every + // boundary; the defer closes whatever handle is live on any exit (the boundary + // handoff already closed every prior chunk's DB), and no writer races the close + // (the loop has stopped on every exit path). + hotDB := cfg.HotDB defer func() { if hotDB != nil { if cerr := hotDB.Close(); cerr != nil && err == nil { diff --git a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go index e1b540f7d..6f7b62ec1 100644 --- a/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/hotloop_test.go @@ -115,16 +115,20 @@ func (r *recordingBoundary) list() []chunk.ID { } // loopConfig builds an ingestionLoopConfig for a test: the stream + resume point + -// a recording boundary. The loop opens the resume chunk's hot DB itself, so no DB -// handle is passed — and the test must hold none on that dir while the loop runs (a -// second read-write open would contend the RocksDB LOCK). +// a recording boundary, and opens the resume chunk's hot DB the way run() does now +// (the loop takes ownership and closes it). The test must hold no other handle on +// that dir while the loop runs (a second read-write open would contend the LOCK). func loopConfig( - stream ledgerbackend.LedgerStream, cat *catalog.Catalog, resume uint32, + t *testing.T, stream ledgerbackend.LedgerStream, cat *catalog.Catalog, resume uint32, ) (ingestionLoopConfig, *recordingBoundary) { + t.Helper() rec := &recordingBoundary{} + db, err := openHotDBForChunk(cat, chunk.IDFromLedger(resume), silentLogger()) + require.NoError(t, err) return ingestionLoopConfig{ Stream: stream, Resume: resume, + HotDB: db, Catalog: cat, Boundary: rec, Logger: silentLogger(), @@ -243,7 +247,7 @@ func TestRunIngestionLoop_LedgerLandsAcrossAllCFs(t *testing.T) { // loop opens the empty chunk 0 itself and resumes at its first ledger. stream := streamForSeqs(t, first, first+2) stream.endErr = errors.New("backend crashed") - cfg, _ := loopConfig(stream, cat, first) + cfg, _ := loopConfig(t, stream, cat, first) err := runIngestionLoop(context.Background(), cfg) require.Error(t, err, "stream ran past the prefix and errored") @@ -282,7 +286,7 @@ func TestRunIngestionLoop_BoundaryNotifiesCompletedChunk(t *testing.T) { c.LastLedger(): zeroTxLCMBytes(t, c.LastLedger()), // boundary 0->1 c1.FirstLedger(): zeroTxLCMBytes(t, c1.FirstLedger()), // a ledger in chunk 1 }, endErr: errors.New("end")} - cfg, rec := loopConfig(stream, cat, resume) + cfg, rec := loopConfig(t, stream, cat, resume) done := make(chan error, 1) go func() { @@ -314,7 +318,7 @@ func TestRunIngestionLoop_CtxCancelReturnsCtxErr(t *testing.T) { stream := streamForSeqs(t, first, first+1) stream.blockOnCtx = true // after the frames, behave like a live tip stream - cfg, _ := loopConfig(stream, cat, first) + cfg, _ := loopConfig(t, stream, cat, first) ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) @@ -347,7 +351,7 @@ func TestRunIngestionLoop_StreamErrorReturnsError(t *testing.T) { stream := streamForSeqs(t, first, first) stream.yieldErrAt = first + 1 stream.errAt = boom - cfg, _ := loopConfig(stream, cat, first) + cfg, _ := loopConfig(t, stream, cat, first) err := runIngestionLoop(context.Background(), cfg) require.Error(t, err) @@ -368,11 +372,11 @@ func TestRunIngestionLoop_RestartResumesFromWatermark(t *testing.T) { c := chunk.ID(0) first := c.FirstLedger() - // First run: the loop opens empty chunk 0 itself (resumes at first), commits + // First run: loopConfig opens empty chunk 0 (resumes at first), the loop commits // [first, first+2], then the stream errs. stream1 := streamForSeqs(t, first, first+2) stream1.endErr = errors.New("end") - cfg1, _ := loopConfig(stream1, cat, first) + cfg1, _ := loopConfig(t, stream1, cat, first) err := runIngestionLoop(context.Background(), cfg1) require.Error(t, err) assert.Equal(t, first, stream1.firstSeen.Load(), "first run resumed at the chunk's first ledger") @@ -388,7 +392,7 @@ func TestRunIngestionLoop_RestartResumesFromWatermark(t *testing.T) { // Second run resumes at the derived watermark and commits two more ledgers. stream2 := streamForSeqs(t, first+3, first+5) stream2.endErr = errors.New("end") - cfg2, _ := loopConfig(stream2, cat, resume) + cfg2, _ := loopConfig(t, stream2, cat, resume) err = runIngestionLoop(context.Background(), cfg2) require.Error(t, err) assert.Equal(t, first+3, stream2.firstSeen.Load(), "second run resumed at watermark+1") diff --git a/cmd/stellar-rpc/internal/fullhistory/startup.go b/cmd/stellar-rpc/internal/fullhistory/startup.go index cb6e5dda2..a38d16c20 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup.go @@ -75,16 +75,33 @@ func run(ctx context.Context, cfg StartConfig) error { Info("backfill complete — opening resume hot tier and ingesting") // Step 2: serve + ingest. resumeLedger is one past the last-committed ledger — - // the live chunk's next un-committed ledger. The ingestion loop opens that - // chunk's hot DB itself (open and deferred close in one function, no cross-call - // ownership gap) and consumes the stream from there. + // the live chunk's next un-committed ledger. resumeLedger := lastCommitted + 1 + // Open the resume chunk's hot DB BEFORE serving reads, so a broken hot tier (a + // "ready" key whose DB won't open) fails startup instead of serving behind a + // crash-looping ingestion loop. run() owns the close only until the loop takes + // over: loopOwnsDB flips true at the errgroup launch, after which the loop's + // deferred close owns it (and g.Wait joins before run returns, so there is no + // window where neither owns it). Restarts re-enter run() from the top, so this + // stays the single initial-open site; the loop still reopens at each boundary. + hotDB, err := openHotDBForChunk(cat, chunk.IDFromLedger(resumeLedger), logger) + if err != nil { + return fmt.Errorf("startup open resume hot tier for ledger %d: %w", resumeLedger, err) + } + loopOwnsDB := false + defer func() { + if !loopOwnsDB { + _ = hotDB.Close() // an error before the loop took ownership + } + }() + // The live ingestion stream. It owns the captive-core process (started on the // loop's first pull, torn down when the loop exits), so there is no eager // prepare and no closer to defer — the loop's ctx-scoped iteration is the // teardown. OpenCore only constructs, so a start failure surfaces as the loop's - // first stream error for the daemon to classify (and restart). + // first stream error for the daemon to classify (and restart). (Eager core start + // before serve would need a LedgerStream.Start hook the SDK deliberately omits.) stream, err := cfg.Core.OpenCore(ctx) if err != nil { return fmt.Errorf("startup open ingestion stream: %w", err) @@ -122,10 +139,13 @@ func run(ctx context.Context, cfg StartConfig) error { // supervise is the one clean-vs-restart decision point; a canceled parent ctx // classifies as clean. g, gctx := errgroup.WithContext(ctx) + // The loop's deferred close now owns hotDB; g.Wait joins it before run returns. + loopOwnsDB = true g.Go(func() error { err := runIngestionLoop(gctx, ingestionLoopConfig{ Stream: stream, Resume: resumeLedger, + HotDB: hotDB, Catalog: cat, Boundary: boundary, Logger: logger, diff --git a/cmd/stellar-rpc/internal/fullhistory/startup_test.go b/cmd/stellar-rpc/internal/fullhistory/startup_test.go index a44b130a8..a9ba55b12 100644 --- a/cmd/stellar-rpc/internal/fullhistory/startup_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/startup_test.go @@ -396,9 +396,9 @@ func TestRun_FirstStartServeIngestCleanShutdown(t *testing.T) { } // A ServeReads error is surfaced wrapped as a restartable failure (NOT clean). -// ServeReads runs after core starts but BEFORE the ingestion loop launches, so run -// returns without the loop ever opening the resume hot DB (the boundary-close -// fence that releases the write handle is pinned where the loop owns it). +// run() opens the resume hot DB and starts core BEFORE serving; a serve error +// after those returns via run()'s defer, which closes the DB (the loop never took +// ownership), so a restart can reopen it — asserted by the reopen below. func TestRun_ServeReadsErrorSurfaces(t *testing.T) { cat, _ := testCatalog(t) pinGenesis(t, cat) @@ -412,6 +412,42 @@ func TestRun_ServeReadsErrorSurfaces(t *testing.T) { require.Contains(t, err.Error(), "serve reads") require.NotErrorIs(t, err, context.Canceled, "a ServeReads error is restartable, not a clean shutdown") require.Equal(t, int32(1), core.openedCount.Load(), "core was started before serving") + + // run() opened the resume hot DB before serving and closed it on the error path + // (the loop never took ownership): reopening it succeeds (LOCK released). + db, err := openHotDBForChunk(cat, chunk.IDFromLedger(chunk.FirstLedgerSeq), silentLogger()) + require.NoError(t, err, "the resume hot DB is reopenable — run released its LOCK") + require.NoError(t, db.Close()) +} + +// The resume hot DB and core are opened BEFORE reads are served (the design's +// fail-fast order): by the time ServeReads runs, the resume chunk's hot key is +// already "ready" and core has started — so a broken hot tier / core fails startup +// instead of serving behind a crash-looping loop. Asserted from inside ServeReads, +// which then errors to avoid entering the blocking loop. +func TestRun_OpensHotDBAndCoreBeforeServe(t *testing.T) { + cat, _ := testCatalog(t) + pinGenesis(t, cat) + resumeChunk := chunk.IDFromLedger(chunk.FirstLedgerSeq) // fresh start ⇒ resume at genesis + core := &fakeCore{stream: &fakeCoreStream{frames: map[uint32][]byte{}, blockOnCtx: true}} + tip := &fakeTipBackend{tips: []uint32{chunk.FirstLedgerSeq + 10}} // young ⇒ no backfill + cfg := startTestConfig(t, cat, tip, core, nil) + + var stateAtServe geometry.HotState + var coreAtServe int32 + cfg.ServeReads = func(context.Context) error { + st, herr := cat.HotState(resumeChunk) + require.NoError(t, herr) + stateAtServe = st + coreAtServe = core.openedCount.Load() + return errors.New("stop before the blocking loop") + } + + err := run(context.Background(), cfg) + require.Error(t, err) + require.Contains(t, err.Error(), "serve reads") + assert.Equal(t, geometry.HotReady, stateAtServe, "resume hot DB is open+ready before serve") + assert.Equal(t, int32(1), coreAtServe, "core is opened before serve") } // run errors on a first start with an unavailable tip (restartable, no sentinel);