From 0c7df0f0e419180b1bb4f507e2be8e23af6d0f26 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 20:08:29 +0900 Subject: [PATCH 1/6] backup: add live point-in-time logical backup producer --- adapter/admin_backup.go | 18 +- cmd/elastickv-backup/main.go | 495 ++++++++++++ cmd/elastickv-backup/main_test.go | 64 ++ cmd/elastickv-snapshot-decode/main.go | 7 +- ... 2026_04_29_implemented_logical_backup.md} | 231 +++--- ...29_implemented_snapshot_logical_decoder.md | 4 +- ...25_implemented_snapshot_logical_encoder.md | 2 +- docs/operations/backup_restore.md | 188 +++++ docs/operations/snapshot_restore.md | 7 +- internal/backup/checksums.go | 40 +- internal/backup/decode.go | 6 + internal/backup/dynamodb.go | 162 +++- internal/backup/dynamodb_test.go | 80 +- internal/backup/filename.go | 2 +- internal/backup/finalize.go | 98 +++ internal/backup/finalize_test.go | 72 ++ internal/backup/live.go | 13 +- internal/backup/live_producer.go | 713 ++++++++++++++++++ internal/backup/live_producer_test.go | 456 +++++++++++ internal/backup/manifest.go | 32 +- main.go | 1 + proto/admin.pb.go | 29 +- proto/admin.proto | 1 + 23 files changed, 2544 insertions(+), 177 deletions(-) create mode 100644 cmd/elastickv-backup/main.go create mode 100644 cmd/elastickv-backup/main_test.go rename docs/design/{2026_04_29_proposed_logical_backup.md => 2026_04_29_implemented_logical_backup.md} (90%) create mode 100644 docs/operations/backup_restore.md create mode 100644 internal/backup/finalize.go create mode 100644 internal/backup/finalize_test.go create mode 100644 internal/backup/live_producer.go create mode 100644 internal/backup/live_producer_test.go diff --git a/adapter/admin_backup.go b/adapter/admin_backup.go index f88db35f4..b78efdace 100644 --- a/adapter/admin_backup.go +++ b/adapter/admin_backup.go @@ -39,6 +39,7 @@ const ( defaultLiveBackupRenewAttempts = 3 defaultLiveBackupRenewBackoff = 500 * time.Millisecond backupAppliedPollInterval = 10 * time.Millisecond + defaultLiveBackupMaxActivePins = 4 ) var ( @@ -83,6 +84,7 @@ type AdminBackupConfig struct { BeginDeadline time.Duration SnapshotHeadroomEntries uint64 ScanPageSize int + MaxActivePins int RenewAttempts int RenewBackoff time.Duration } @@ -94,6 +96,7 @@ type backupConfig struct { beginDeadline time.Duration snapshotHeadroomEntries uint64 scanPageSize int + maxActivePins int renewAttempts int renewBackoff time.Duration } @@ -106,6 +109,7 @@ func defaultBackupConfig() backupConfig { beginDeadline: defaultLiveBackupBeginDeadline, snapshotHeadroomEntries: defaultLiveBackupHeadroom, scanPageSize: defaultLiveBackupScanPageSize, + maxActivePins: defaultLiveBackupMaxActivePins, renewAttempts: defaultLiveBackupRenewAttempts, renewBackoff: defaultLiveBackupRenewBackoff, } @@ -151,6 +155,9 @@ func WithAdminBackupConfig(cfg AdminBackupConfig) AdminOption { if cfg.ScanPageSize > 0 { s.backupConfig.scanPageSize = cfg.ScanPageSize } + if cfg.MaxActivePins > 0 { + s.backupConfig.maxActivePins = cfg.MaxActivePins + } if cfg.RenewAttempts > 0 { s.backupConfig.renewAttempts = cfg.RenewAttempts } @@ -245,11 +252,12 @@ func (s *AdminServer) BeginBackup(ctx context.Context, req *pb.BeginBackupReques s.rememberBackupSession(tok, prepared.routes) return &pb.BeginBackupResponse{ - ReadTs: prepared.readTS, - PinToken: encodedToken, - TtlMsEffective: uint64(prepared.ttl / time.Millisecond), //nolint:gosec // validated positive. - Shards: backupShardResponses(prepared.groups, prepared.commits), - ExpectedKeys: backupExpectedResponses(counts, appliedAtCount), + ReadTs: prepared.readTS, + PinToken: encodedToken, + TtlMsEffective: uint64(prepared.ttl / time.Millisecond), //nolint:gosec // validated positive. + Shards: backupShardResponses(prepared.groups, prepared.commits), + ExpectedKeys: backupExpectedResponses(counts, appliedAtCount), + MaxActiveBackupPins: uint32(s.backupConfig.maxActivePins), //nolint:gosec // startup validation requires a positive int. }, nil } diff --git a/cmd/elastickv-backup/main.go b/cmd/elastickv-backup/main.go new file mode 100644 index 000000000..191559e9c --- /dev/null +++ b/cmd/elastickv-backup/main.go @@ -0,0 +1,495 @@ +// Command elastickv-backup creates a point-in-time logical dump from a running +// elastickv cluster. +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "io" + "log/slog" + "os" + "os/signal" + "sort" + "strconv" + "strings" + "syscall" + "time" + + internalutil "github.com/bootjp/elastickv/internal" + "github.com/bootjp/elastickv/internal/backup" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" +) + +const ( + maxAdminTokenFileBytes = 4 << 10 + defaultBackupTTLMS = 1_800_000 + defaultRPCDeadline = 30 * time.Second +) + +var version = "dev" + +func main() { + if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintf(os.Stderr, "elastickv-backup: %v\n", err) + os.Exit(1) + } +} + +type config struct { + address string + outputRoot string + format string + tokenFile string + clusterID string + + adapters backup.AdapterSet + scopes []backup.Scope + ttl time.Duration + + includeIncompleteUploads bool + includeOrphans bool + preserveSQSVisibility bool + includeSQSSideRecords bool + renameCollisions bool + bundleJSONL bool + bundleSizeBytes int64 + + beginTimeout time.Duration + endTimeout time.Duration + tlsCAFile string + tlsServer string + tlsSkip bool +} + +func run(argv []string, stdout, stderr io.Writer) (retErr error) { + cfg, err := parseFlags(argv) + if err != nil { + return errors.Wrap(err, "run live backup") + } + token, err := loadAdminToken(cfg.tokenFile) + if err != nil { + return err + } + creds, err := loadTransportCredentials(cfg.tlsCAFile, cfg.tlsServer, cfg.tlsSkip) + if err != nil { + return err + } + conn, err := grpc.NewClient(cfg.address, + grpc.WithTransportCredentials(creds), + internalutil.GRPCCallOptions(), + ) + if err != nil { + return errors.Wrap(err, "dial backup admin endpoint") + } + defer func() { + retErr = errors.CombineErrors(retErr, errors.WithStack(conn.Close())) + }() + + logger := slog.New(slog.NewTextHandler(stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + result, err := backup.RunLiveBackup(ctx, &grpcLiveBackupRPC{client: pb.NewAdminClient(conn), token: token}, backup.LiveBackupOptions{ + OutputRoot: cfg.outputRoot, + Adapters: cfg.adapters, + Scopes: cfg.scopes, + TTL: cfg.ttl, + IncludeIncompleteUploads: cfg.includeIncompleteUploads, + IncludeOrphans: cfg.includeOrphans, + PreserveSQSVisibility: cfg.preserveSQSVisibility, + IncludeSQSSideRecords: cfg.includeSQSSideRecords, + RenameS3Collisions: cfg.renameCollisions, + DynamoDBBundleJSONL: cfg.bundleJSONL, + DynamoDBBundleSizeBytes: cfg.bundleSizeBytes, + ElastickvVersion: version, + ClusterID: cfg.clusterID, + BeginTimeout: cfg.beginTimeout, + EndTimeout: cfg.endTimeout, + WarnSink: warningSink(logger), + }) + if err != nil { + return errors.Wrap(err, "create live backup") + } + if err := writeArchive(cfg, stdout); err != nil { + return err + } + logger.Info("live backup complete", + "output", cfg.outputRoot, + "read_ts", result.ReadTS, + "scopes", len(result.Scopes), + "records", result.Counters.Total, + "pin_renewals_total", result.PinRenewals, + "format", cfg.format, + ) + return nil +} + +func parseFlags(argv []string) (*config, error) { + if len(argv) == 0 || argv[0] != "dump" { + return nil, errors.New("usage: elastickv-backup dump [flags]") + } + values, err := parseDumpFlagValues(argv[1:]) + if err != nil { + return nil, err + } + return buildDumpConfig(values) +} + +type dumpFlagValues struct { + cfg config + adapterCSV string + bundleMode string + bundleSize string + checksums string + ttlMS uint64 + scopes stringListFlag +} + +func parseDumpFlagValues(argv []string) (*dumpFlagValues, error) { + fs := flag.NewFlagSet("elastickv-backup dump", flag.ContinueOnError) + fs.SetOutput(io.Discard) + values := &dumpFlagValues{} + bindDumpFlags(fs, values) + if err := fs.Parse(argv); err != nil { + return nil, errors.WithStack(err) + } + if len(fs.Args()) != 0 { + return nil, errors.Errorf("unexpected arguments: %s", strings.Join(fs.Args(), " ")) + } + return values, nil +} + +func bindDumpFlags(fs *flag.FlagSet, values *dumpFlagValues) { + cfg := &values.cfg + fs.StringVar(&cfg.address, "address", "", "Admin gRPC address") + fs.StringVar(&cfg.outputRoot, "output-dir", "", "New destination directory") + fs.StringVar(&cfg.format, "output-format", "directory", "directory, tar, or tar+zstd") + fs.StringVar(&cfg.tokenFile, "admin-token-file", "", "Bearer-token file") + fs.StringVar(&cfg.clusterID, "cluster-id", "", "Cluster identifier for MANIFEST.json") + fs.StringVar(&values.adapterCSV, "adapter", "dynamodb,s3,redis,sqs", "Comma-separated adapter set") + fs.Var(&values.scopes, "scope", "Adapter scope selection, for example dynamodb=orders,users") + fs.Uint64Var(&values.ttlMS, "ttl-ms", defaultBackupTTLMS, "Requested pin TTL in milliseconds") + fs.DurationVar(&cfg.beginTimeout, "begin-backup-deadline", defaultRPCDeadline, "Client deadline for BeginBackup") + fs.DurationVar(&cfg.endTimeout, "end-backup-deadline", defaultRPCDeadline, "Cleanup deadline for EndBackup") + fs.BoolVar(&cfg.includeIncompleteUploads, "include-incomplete-uploads", false, "Include in-flight S3 multipart uploads") + fs.BoolVar(&cfg.includeOrphans, "include-orphans", false, "Include S3 orphan blobs") + fs.BoolVar(&cfg.preserveSQSVisibility, "preserve-sqs-visibility", false, "Preserve SQS visibility state") + fs.BoolVar(&cfg.includeSQSSideRecords, "include-sqs-side-records", false, "Include SQS internal side records") + fs.BoolVar(&cfg.renameCollisions, "rename-collisions", false, "Rename colliding S3 logical paths") + fs.StringVar(&values.checksums, "checksums", "sha256", "Checksum algorithm (sha256)") + fs.StringVar(&values.bundleMode, "dynamodb-bundle-mode", "per-item", "per-item or jsonl") + fs.StringVar(&values.bundleSize, "dynamodb-bundle-size", "64MiB", "Maximum JSONL part size") + fs.StringVar(&cfg.tlsCAFile, "tls-ca-cert-file", "", "PEM CA file for the Admin endpoint") + fs.StringVar(&cfg.tlsServer, "tls-server-name", "", "Expected TLS server name") + fs.BoolVar(&cfg.tlsSkip, "tls-insecure-skip-verify", false, "Use TLS without certificate verification") +} + +func buildDumpConfig(values *dumpFlagValues) (*config, error) { + cfg := values.cfg + var err error + if err := validateDumpFlagValues(values); err != nil { + return nil, err + } + cfg.ttl, err = dumpTTL(values.ttlMS) + if err != nil { + return nil, err + } + cfg.adapters, err = parseAdapterSet(values.adapterCSV) + if err != nil { + return nil, err + } + cfg.scopes, err = parseScopes(values.scopes) + if err != nil { + return nil, err + } + if err := validateRequestedScopeAdapters(cfg.adapters, cfg.scopes); err != nil { + return nil, err + } + if err := populateDynamoDBBundleConfig(&cfg, values); err != nil { + return nil, err + } + return &cfg, nil +} + +func validateDumpFlagValues(values *dumpFlagValues) error { + if values.cfg.address == "" || values.cfg.outputRoot == "" { + return errors.New("--address and --output-dir are required") + } + if values.checksums != backup.ChecksumAlgorithmSHA256 { + return errors.Errorf("unsupported --checksums %q", values.checksums) + } + return validateOutputFormat(values.cfg.format) +} + +func dumpTTL(ttlMS uint64) (time.Duration, error) { + if ttlMS == 0 || ttlMS > uint64((time.Duration(1<<63-1))/time.Millisecond) { + return 0, errors.New("--ttl-ms is outside the supported duration range") + } + return time.Duration(ttlMS) * time.Millisecond, nil //nolint:gosec // upper bound checked above. +} + +func populateDynamoDBBundleConfig(cfg *config, values *dumpFlagValues) error { + var err error + cfg.bundleJSONL, err = parseBundleMode(values.bundleMode) + if err != nil { + return err + } + cfg.bundleSizeBytes, err = parseByteSize(values.bundleSize) + return errors.Wrap(err, "--dynamodb-bundle-size") +} + +type stringListFlag []string + +func (f *stringListFlag) String() string { return strings.Join(*f, ";") } +func (f *stringListFlag) Set(value string) error { + *f = append(*f, value) + return nil +} + +func parseAdapterSet(csv string) (backup.AdapterSet, error) { + if csv == "" || csv == "all" { + return backup.AllAdapters(), nil + } + var set backup.AdapterSet + for _, raw := range strings.Split(csv, ",") { + name := strings.TrimSpace(raw) + if name == "" { + continue + } + if err := enableAdapter(&set, name); err != nil { + return backup.AdapterSet{}, err + } + } + if set == (backup.AdapterSet{}) { + return backup.AdapterSet{}, errors.New("--adapter selected zero adapters") + } + return set, nil +} + +func enableAdapter(set *backup.AdapterSet, name string) error { + switch name { + case "dynamodb": + set.DynamoDB = true + case "s3": + set.S3 = true + case "redis": + set.Redis = true + case "sqs": + set.SQS = true + default: + return errors.Errorf("unknown adapter %q", name) + } + return nil +} + +func parseScopes(flags []string) ([]backup.Scope, error) { + seen := make(map[backup.Scope]struct{}) + for _, raw := range flags { + adapter, names, ok := strings.Cut(raw, "=") + adapter = strings.TrimSpace(adapter) + if !ok || adapter == "" || names == "" { + return nil, errors.Errorf("invalid --scope %q", raw) + } + for _, name := range strings.Split(names, ",") { + name = strings.TrimSpace(name) + if name == "" { + return nil, errors.Errorf("invalid empty scope in %q", raw) + } + seen[backup.Scope{Adapter: adapter, Name: name}] = struct{}{} + } + } + out := make([]backup.Scope, 0, len(seen)) + for scope := range seen { + out = append(out, scope) + } + sortScopes(out) + return out, nil +} + +func sortScopes(scopes []backup.Scope) { + sort.Slice(scopes, func(i, j int) bool { + if scopes[i].Adapter != scopes[j].Adapter { + return scopes[i].Adapter < scopes[j].Adapter + } + return scopes[i].Name < scopes[j].Name + }) +} + +func validateRequestedScopeAdapters(set backup.AdapterSet, scopes []backup.Scope) error { + for _, scope := range scopes { + enabled := (scope.Adapter == "dynamodb" && set.DynamoDB) || + (scope.Adapter == "s3" && set.S3) || + (scope.Adapter == "redis" && set.Redis) || + (scope.Adapter == "sqs" && set.SQS) + if !enabled { + return errors.Errorf("--scope %s is not enabled by --adapter", scope) + } + } + return nil +} + +func parseBundleMode(mode string) (bool, error) { + switch mode { + case "", "per-item": + return false, nil + case "jsonl": + return true, nil + default: + return false, errors.Errorf("invalid --dynamodb-bundle-mode %q", mode) + } +} + +func parseByteSize(raw string) (int64, error) { + raw = strings.TrimSpace(raw) + multiplier := int64(1) + for _, unit := range []struct { + suffix string + scale int64 + }{{"GiB", 1 << 30}, {"MiB", 1 << 20}, {"KiB", 1 << 10}, {"B", 1}} { + if strings.HasSuffix(raw, unit.suffix) { + raw = strings.TrimSpace(strings.TrimSuffix(raw, unit.suffix)) + multiplier = unit.scale + break + } + } + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil || value <= 0 || value > (1<<63-1)/multiplier { + return 0, errors.Errorf("invalid byte size %q", raw) + } + return value * multiplier, nil +} + +func validateOutputFormat(format string) error { + switch format { + case "directory", "tar", "tar+zstd": + return nil + default: + return errors.Errorf("unsupported --output-format %q", format) + } +} + +func writeArchive(cfg *config, stdout io.Writer) error { + switch cfg.format { + case "directory": + return nil + case "tar": + return errors.Wrap(backup.PackDumpTree(cfg.outputRoot, stdout, backup.ArchiveCompressionNone), "write tar backup") + case "tar+zstd": + return errors.Wrap(backup.PackDumpTree(cfg.outputRoot, stdout, backup.ArchiveCompressionZstd), "write zstd tar backup") + default: + return errors.Errorf("unsupported output format %q", cfg.format) + } +} + +func loadAdminToken(path string) (string, error) { + if path == "" { + return "", nil + } + f, err := os.Open(path) //nolint:gosec // operator-selected token file + if err != nil { + return "", errors.WithStack(err) + } + body, err := io.ReadAll(io.LimitReader(f, maxAdminTokenFileBytes+1)) + if err != nil { + _ = f.Close() + return "", errors.WithStack(err) + } + if err := f.Close(); err != nil { + return "", errors.WithStack(err) + } + if len(body) > maxAdminTokenFileBytes { + return "", errors.New("admin token file exceeds 4 KiB") + } + token := strings.TrimSpace(string(body)) + if token == "" { + return "", errors.New("admin token file is empty") + } + return token, nil +} + +func loadTransportCredentials(caFile, serverName string, skipVerify bool) (credentials.TransportCredentials, error) { + if caFile == "" && !skipVerify { + return insecure.NewCredentials(), nil + } + conf := &tls.Config{MinVersion: tls.VersionTLS12, ServerName: serverName, InsecureSkipVerify: skipVerify} //nolint:gosec // explicit development flag. + if caFile != "" { + pem, err := os.ReadFile(caFile) //nolint:gosec // operator-selected CA file + if err != nil { + return nil, errors.WithStack(err) + } + pool, err := x509.SystemCertPool() + if err != nil || pool == nil { + pool = x509.NewCertPool() + } + if !pool.AppendCertsFromPEM(pem) { + return nil, errors.New("TLS CA file contains no certificates") + } + conf.RootCAs = pool + } + return credentials.NewTLS(conf), nil +} + +func warningSink(logger *slog.Logger) func(string, ...any) { + return func(event string, fields ...any) { + args := append([]any{"event", event}, fields...) + logger.Warn("backup warning", args...) + } +} + +type grpcLiveBackupRPC struct { + client pb.AdminClient + token string +} + +func (r *grpcLiveBackupRPC) outgoing(ctx context.Context) context.Context { + if r.token == "" { + return ctx + } + return metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+r.token) +} + +func (r *grpcLiveBackupRPC) BeginBackup(ctx context.Context, req *pb.BeginBackupRequest) (*pb.BeginBackupResponse, error) { + resp, err := r.client.BeginBackup(r.outgoing(ctx), req) + return resp, errors.Wrap(err, "BeginBackup RPC") +} + +func (r *grpcLiveBackupRPC) RenewBackup(ctx context.Context, req *pb.RenewBackupRequest) (*pb.RenewBackupResponse, error) { + resp, err := r.client.RenewBackup(r.outgoing(ctx), req) + return resp, errors.Wrap(err, "RenewBackup RPC") +} + +func (r *grpcLiveBackupRPC) EndBackup(ctx context.Context, req *pb.EndBackupRequest) (*pb.EndBackupResponse, error) { + resp, err := r.client.EndBackup(r.outgoing(ctx), req) + return resp, errors.Wrap(err, "EndBackup RPC") +} + +func (r *grpcLiveBackupRPC) ListAdaptersAndScopes(ctx context.Context, req *pb.ListAdaptersAndScopesRequest) (*pb.ListAdaptersAndScopesResponse, error) { + resp, err := r.client.ListAdaptersAndScopes(r.outgoing(ctx), req) + return resp, errors.Wrap(err, "ListAdaptersAndScopes RPC") +} + +func (r *grpcLiveBackupRPC) StreamBackup(ctx context.Context, req *pb.StreamBackupRequest, consume func(*pb.BackupKV) error) error { + stream, err := r.client.StreamBackup(r.outgoing(ctx), req) + if err != nil { + return errors.Wrap(err, "StreamBackup RPC") + } + for { + record, err := stream.Recv() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return errors.Wrap(err, "receive StreamBackup record") + } + if err := consume(record); err != nil { + return err + } + } +} diff --git a/cmd/elastickv-backup/main_test.go b/cmd/elastickv-backup/main_test.go new file mode 100644 index 000000000..f2cda3d96 --- /dev/null +++ b/cmd/elastickv-backup/main_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "testing" + + "github.com/bootjp/elastickv/internal/backup" +) + +func TestParseFlagsScopesBundleAndArchive(t *testing.T) { + t.Parallel() + cfg, err := parseFlags([]string{ + "dump", + "--address", "127.0.0.1:50051", + "--output-dir", "/tmp/dump", + "--output-format", "tar+zstd", + "--adapter", "dynamodb,s3", + "--scope", "dynamodb=users,orders", + "--scope", "s3=photos", + "--dynamodb-bundle-mode", "jsonl", + "--dynamodb-bundle-size", "2MiB", + }) + if err != nil { + t.Fatalf("parseFlags: %v", err) + } + if cfg.format != "tar+zstd" || !cfg.bundleJSONL || cfg.bundleSizeBytes != 2<<20 { + t.Fatalf("cfg=%+v", cfg) + } + want := []backup.Scope{ + {Adapter: "dynamodb", Name: "orders"}, + {Adapter: "dynamodb", Name: "users"}, + {Adapter: "s3", Name: "photos"}, + } + if len(cfg.scopes) != len(want) { + t.Fatalf("scopes=%v", cfg.scopes) + } + for i := range want { + if cfg.scopes[i] != want[i] { + t.Fatalf("scope[%d]=%v, want %v", i, cfg.scopes[i], want[i]) + } + } +} + +func TestParseFlagsRejectsScopeExcludedByAdapter(t *testing.T) { + t.Parallel() + _, err := parseFlags([]string{ + "dump", "--address", "127.0.0.1:50051", "--output-dir", "/tmp/dump", + "--adapter", "redis", "--scope", "s3=photos", + }) + if err == nil { + t.Fatal("parseFlags accepted excluded scope") + } +} + +func TestParseByteSize(t *testing.T) { + t.Parallel() + for input, want := range map[string]int64{ + "1": 1, "1B": 1, "2KiB": 2 << 10, "3MiB": 3 << 20, "4GiB": 4 << 30, + } { + got, err := parseByteSize(input) + if err != nil || got != want { + t.Fatalf("parseByteSize(%q)=(%d,%v), want %d", input, got, err, want) + } + } +} diff --git a/cmd/elastickv-snapshot-decode/main.go b/cmd/elastickv-snapshot-decode/main.go index 3359e2eaf..3c697584d 100644 --- a/cmd/elastickv-snapshot-decode/main.go +++ b/cmd/elastickv-snapshot-decode/main.go @@ -193,10 +193,9 @@ var ErrUnknownAdapter = errors.New("unknown adapter name") // flag value is neither per-item nor jsonl. var ErrBundleModeInvalid = errors.New("invalid --dynamodb-bundle-mode") -// parseBundleMode validates the --dynamodb-bundle-mode value. The -// underlying encoder rejects bundleJSONL=true at Finalize time -// (the JSONL path is a Phase 0b follow-up), but we still accept -// the flag now so the CLI surface is stable across that change. +// parseBundleMode validates the --dynamodb-bundle-mode value. JSONL uses the +// encoder's default 64 MiB part size; the live producer additionally exposes a +// per-run size override. func parseBundleMode(mode string) (bool, error) { switch mode { case "per-item", "": diff --git a/docs/design/2026_04_29_proposed_logical_backup.md b/docs/design/2026_04_29_implemented_logical_backup.md similarity index 90% rename from docs/design/2026_04_29_proposed_logical_backup.md rename to docs/design/2026_04_29_implemented_logical_backup.md index c5e97c6b7..5fd5654e0 100644 --- a/docs/design/2026_04_29_proposed_logical_backup.md +++ b/docs/design/2026_04_29_implemented_logical_backup.md @@ -1,6 +1,6 @@ # Logical Backup: Live Cluster, PIT-Consistent Extraction (Phase 1) -Status: Proposed +Status: Implemented Author: bootjp Date: 2026-04-29 @@ -11,10 +11,14 @@ design doc; both produce dumps in the **same on-disk format** so restorers and external tools do not care which phase produced a given dump. +The Phase 1 central scope in this document is implemented. Phase 2 restore +replay and Phase 3 incremental backup remain intentional future extensions; +they are not prerequisites for Phase 1's live PIT producer status. + | Phase | Doc | Scope | |-------|-----|-------| -| **Phase 0** | [`2026_04_29_proposed_snapshot_logical_decoder.md`](./2026_04_29_proposed_snapshot_logical_decoder.md) | Offline `.fsm` ↔ logical-format directory tree converter. No live cluster, no admin RPCs, no FSM/Raft changes. **Owns the format definition.** Sufficient for single-shard clusters, one-time exports off elastickv, and any use case where the latest available snapshot is a good-enough recovery point. | -| **Phase 1 (this doc)** | This file | Live, running-cluster extraction with cluster-wide point-in-time consistency across multiple Raft groups. Adds `BeginBackup` / `RenewBackup` / `EndBackup` admin RPCs, replicated `BackupPin` / `Extend` / `Release` Raft FSM commands, version-gated rolling-upgrade safety, expected-keys baseline. Required only when cross-shard PIT consistency or "snapshot now" cadence is needed. | +| **Phase 0** | [`2026_04_29_implemented_snapshot_logical_decoder.md`](./2026_04_29_implemented_snapshot_logical_decoder.md) | Offline `.fsm` ↔ logical-format directory tree converter. No live cluster, no admin RPCs, no FSM/Raft changes. **Owns the format definition.** Sufficient for single-shard clusters, one-time exports off elastickv, and any use case where the latest available snapshot is a good-enough recovery point. | +| **Phase 1 (this doc)** | This file | Live, running-cluster extraction with cluster-wide point-in-time consistency across multiple Raft groups. Adds `BeginBackup` / `RenewBackup` / `EndBackup` / `ListAdaptersAndScopes` / `StreamBackup` admin RPCs, replicated capacity reservation and pin lifecycle commands, version-gated rolling-upgrade safety, and an expected-keys baseline. Required only when cross-shard PIT consistency or "snapshot now" cadence is needed. | The format details (per-adapter directory layout, filename encoding, `MANIFEST.json` schema, per-adapter record shapes) are defined in the @@ -720,8 +724,9 @@ in `main.go` (listed in Phase 1 scope below). > review of the Phase 1 producer will flag the `ScanAt` use as > contradicting the warning. -`pageSize` is the same `ScanAt` `limit` parameter; the producer's -default is 1024 and is exposed as a `--scan-page-size` CLI flag. +`pageSize` is the same `ScanAt` `limit` parameter. It defaults to 1024 +and is configured on each server with `--backupScanPageSize`; the +producer cannot weaken the server's paging or snapshot-headroom policy. Per-adapter encoders consume `BackupScanner` and emit one record per `Next` (or batch records into the same JSONL file for SQS / streams). @@ -740,7 +745,7 @@ service Admin { message BeginBackupRequest { // Time-to-live for the read_ts pin on the active timestamp tracker. // If neither EndBackup nor RenewBackup is called within this window - // the pin is auto-released. Range: 60s–24h. Default: 30m. + // the pin is auto-released. Default range: 60s–1h. Default: 30m. uint64 ttl_ms = 1; } message BeginBackupResponse { @@ -753,26 +758,19 @@ message BeginBackupResponse { // own scope-level traversal count against expected_keys[i].key_count // and aborts with ErrCompactionDuringDump on shortfall. repeated ScopeKeyCount expected_keys = 5; + uint32 max_active_backup_pins = 6; } message ShardApplied { - string group_id = 1; // distribution.GroupID stringified + uint64 group_id = 1; uint64 applied_index = 2; } message ScopeKeyCount { - // scope_id is the canonical "/" string, used - // by the producer to match each baseline against the directory it - // is about to write. Mapping per adapter: - // - DynamoDB: "dynamodb/" e.g. "dynamodb/orders" - // - S3 : "s3/" e.g. "s3/photos" - // - Redis : "redis/db_" e.g. "redis/db_0" - // (n is the uint32 from - // ListAdaptersAndScopesResponse.redis_databases) - // - SQS : "sqs/" e.g. "sqs/orders-fifo.fifo" - string scope_id = 1; - uint64 key_count = 2; - uint64 applied_index_at_count = 3; + string adapter = 1; + string scope = 2; + uint64 key_count = 3; + uint64 applied_index_at_count = 4; } // RenewBackup extends the deadline for an existing pin. A long-running @@ -782,11 +780,14 @@ message ScopeKeyCount { message RenewBackupRequest { bytes pin_token = 1; // Same range constraint as BeginBackupRequest.ttl_ms: - // 60s–24h, bounded above by backup_max_ttl_ms. Out-of-range values + // 60s–backupMaxTTL (1h by default). Out-of-range values // are rejected with InvalidArgument. uint64 ttl_ms = 2; } -message RenewBackupResponse { uint64 ttl_ms_effective = 1; } +message RenewBackupResponse { + uint64 ttl_ms_effective = 1; + bytes pin_token = 2; // replacement token with the committed hard deadline +} message EndBackupRequest { bytes pin_token = 1; } message EndBackupResponse {} @@ -797,12 +798,13 @@ message EndBackupResponse {} // created by a concurrent client between BeginBackup and this call is // invisible at the pinned read_ts and therefore not listed. message ListAdaptersAndScopesRequest { bytes pin_token = 1; } -message ListAdaptersAndScopesResponse { - repeated string dynamodb_tables = 1; // from !ddb|meta|table| scan at read_ts - repeated string s3_buckets = 2; // from !s3|bucket|meta| scan at read_ts - repeated uint32 redis_databases = 3; // {0} until multi-db lands - repeated string sqs_queues = 4; // from !sqs|queue|meta| scan at read_ts +message BackupScope { string adapter = 1; string scope = 2; } +message ListAdaptersAndScopesResponse { repeated BackupScope scopes = 1; } +message StreamBackupRequest { + bytes pin_token = 1; + repeated BackupScope scopes = 2; // empty means every scope at read_ts } +message BackupKV { bytes key = 1; bytes value = 2; } ``` `ListAdaptersAndScopes` is a thin wrapper over per-adapter metadata @@ -836,8 +838,8 @@ deployment, a pin recorded on the node receiving the admin RPC is not sufficient — the producer's `BackupScanner` reads from group leaders that may live on different nodes whose compactors are oblivious to the local pin. Pins are therefore **propagated through each Raft group's -log** as `BackupPin{pin_id, read_ts, deadline}` / `BackupExtend` / -`BackupRelease` FSM commands. Every replica applies these to its +log** as capacity reservation plus `BackupPin{pin_id, read_ts, +deadline}` / `BackupExtend` / `BackupRelease` FSM commands. Every replica applies these to its local tracker on log apply, so the pin set is replicated and durable across leader changes (a newly-elected leader inherits the pin from the same log it just applied). Compaction on each replica continues to @@ -860,15 +862,18 @@ follower catching up via snapshot will never replay it. The replica's compactor (now bounded only by other live pins, or unbounded) becomes free to retire MVCC versions at `read_ts`. -This bounds when the design is safe: +Every renewal replays the complete pin rather than only extending a +deadline, so the next successful renewal repairs this replica-local +loss. The remaining safety bound is the repair window: > **Safe-bound invariant**: -> `backup_duration + worst_case_compaction_lag < mvcc_retention_horizon`. +> `renewal_interval + worst_case_compaction_lag < mvcc_retention_horizon`. -Concretely, with a 1-hour MVCC retention and a 10-minute Raft -snapshot interval, the design tolerates dumps up to ~50 minutes -without hitting the corner case. Beyond that, a snapshot triggered -by a freshly-restarted follower can quietly unprotect a replica. +With the default 30-minute TTL, renewal runs every 10 minutes. A +snapshot-installed replica is therefore unprotected for at most one +successful-renewal interval; renewal failure aborts the producer, and +the expected-key check remains the final defense before manifest +publication. `BeginBackup` enforces a soft form of this invariant: it reads each group's `Status.AppliedIndex` and `Status.LastSnapshotIndex` @@ -880,7 +885,7 @@ configurable margin: ```text entries_since_last_snapshot = AppliedIndex - LastSnapshotIndex remaining_headroom = SnapshotEvery - entries_since_last_snapshot -refuse if: remaining_headroom < --snapshot-headroom-entries +refuse if: remaining_headroom < --backupSnapshotHeadroomEntries ``` `SnapshotEvery` is the per-engine snapshot trigger (default @@ -907,7 +912,7 @@ place, `BeginBackup` reads each group's `SnapshotEvery` rather than hardcoding `defaultSnapshotEvery`, so an operator who tuned `ELASTICKV_RAFT_SNAPSHOT_COUNT` sees consistent behavior. With `SnapshotEvery = 10000` and -`--snapshot-headroom-entries = 1000` (default; one-tenth of +`--backupSnapshotHeadroomEntries = 1000` (default; one-tenth of SnapshotEvery), the check refuses backups when fewer than 1000 entries remain before the next snapshot fires — i.e. when an in-flight backup is at risk of triggering the snapshot-installation corner case. A @@ -923,8 +928,8 @@ become eventually-consistent (versions at `read_ts` may already have been compacted on that replica), and the producer surfaces the inconsistency via a per-scope **expected-keys baseline**: -> **Expected-keys baseline.** **After step 2 (every shard has applied -> through `read_ts`) and before step 3 (the pin is installed)**, the +> **Expected-keys baseline.** **After every shard has applied through +> `read_ts` and after the pin is installed**, the > admin server runs a *count-only* scan of each adapter scope at > `read_ts` (`ShardStore.ScanKeysAt`, returning > per-scope `(scope_id, key_count, applied_index_at_count)`). The @@ -947,8 +952,9 @@ inconsistency via a per-scope **expected-keys baseline**: > cheaper than the data-bearing scan that follows. For a 100M-key > scope the baseline pass adds seconds, not minutes. > -> If the producer's actual key count for a scope is `< 99% × baseline -> ± sqrt(baseline)` (binomial-noise tolerance for legitimate +> If the producer's actual key count for a scope is below +> `baseline - ceil(baseline/100) - floor(sqrt(baseline))` (integer-only +> binomial-noise tolerance for legitimate > compaction of TTL'd keys between baseline and dump), the dump fails > with `ErrCompactionDuringDump` and the partial directory tree is > rejected by the restore tool (no `MANIFEST.json` is written). The @@ -1046,14 +1052,16 @@ dumps with retention pressure. `BackupScanner.Next(at_ts=read_ts)`. 5. **Renew on long dumps**: the producer calls `RenewBackup(pin_token, ttl_ms)` every `ttl_ms / 3`. The admin - server proposes `BackupExtend{pin_id, deadline}` on every group - recorded in `pin_token`. The `read_ts` is preserved across - renewals; only the deadline shifts. A multi-hour dump never relies + server replays a complete `BackupPin{pin_id, read_ts, deadline}` on + every group recorded in `pin_token`. Replaying the complete pin + restores a replica-local fence lost during snapshot installation; + `read_ts` is preserved and the response rotates the HMAC token to + carry the newly committed hard deadline. A multi-hour dump never relies on a single 30-minute pin. Renewals are cheap (one Raft entry per group per renewal — at `ttl_ms/3 = 10 min`, that is 6 entries per hour per group, negligible alongside production traffic). - **Per-group renewal retry**: `BackupExtend` proposes through + **Per-group renewal retry**: the replacement `BackupPin` proposes through `ShardGroup.Propose`, which fails transiently during a leader election (typically <1 s under etcd/raft defaults). The admin server retries each per-group proposal **up to 3 times with 500 ms @@ -1064,8 +1072,15 @@ dumps with retention pressure. compactor would have already retired versions the in-flight scan still depends on). If renewal succeeds on some groups but fails on others after retries, the producer aborts and issues `EndBackup` - (which itself tolerates partial state — see step 6). -6. **`EndBackup(pin_token)`** proposes `BackupRelease{pin_id}` on + (which itself tolerates partial state — see step 7). +6. **Publish the completion marker only after renewal quiesces.** After the + stream and all adapter finalizers complete, the producer stops the renewal + goroutine and waits for it to exit. Any renewal error or parent-context + cancellation aborts publication. Only a clean stop may write `CHECKSUMS` + and atomically publish `MANIFEST.json`. This ordering prevents a renewal + failure racing with manifest publication and leaving a completed-looking + artifact whose pin had already become unsafe. +7. **`EndBackup(pin_token)`** proposes `BackupRelease{pin_id}` on every group recorded in `pin_token`. The release is idempotent: a group that has already swept the pin via deadline expiry treats the release as a no-op. A producer crash before EndBackup leaves @@ -1113,6 +1128,8 @@ piping straight to S3 / GCS / a tape device. elastickv-backup dump \ --address 127.0.0.1:50051 \ --output-dir /backups/2026-04-29 \ + [--output-format directory|tar|tar+zstd] \ + [--admin-token-file /run/secrets/elastickv-admin-token] \ [--adapter dynamodb,s3,redis,sqs] \ [--scope dynamodb=orders,users] \ [--scope s3=photos] \ @@ -1122,21 +1139,27 @@ elastickv-backup dump \ [--include-sqs-side-records] \ [--checksums sha256] \ [--ttl-ms 1800000] \ - [--begin-backup-deadline 5s] \ - [--snapshot-headroom-entries 1000] \ - [--scan-page-size 1024] \ + [--begin-backup-deadline 30s] \ + [--end-backup-deadline 30s] \ [--dynamodb-bundle-mode per-item|jsonl] \ [--dynamodb-bundle-size 64MiB] \ [--rename-collisions] ``` +`--begin-backup-deadline` is the producer-side RPC deadline. The +server-side pin fan-out deadline, snapshot headroom, and scan page size +remain operator-controlled server flags (`--backupBeginDeadline`, +`--backupSnapshotHeadroomEntries`, and `--backupScanPageSize`) so a +backup client cannot weaken cluster safety policy. + Internally it runs: ```text BeginBackup(ttl_ms=1800000) → ListAdaptersAndScopes → BackupScanner.Next* (per scope, at read_ts) → encode-and-write per adapter - → CHECKSUMS → MANIFEST.json + → CHECKSUMS (including the prepared manifest digest) + → atomically publish MANIFEST.json last → EndBackup(pin_token) ``` @@ -1294,7 +1317,7 @@ trustworthy off-cluster artifact even before the restore tool is fully written. - New admin RPCs on `proto/admin.proto`: `BeginBackup` (with `ttl_ms`), - `EndBackup`, `RenewBackup`, `ListAdaptersAndScopes`, + `EndBackup`, `RenewBackup`, `ListAdaptersAndScopes`, `StreamBackup`, `GetNodeVersion` (signatures in "Read-Side Consistency" and the "Two-version rolling-upgrade plan" subsection). Plus a one-field extension to the existing `RaftGroupState` message @@ -1346,46 +1369,18 @@ written. - Extend `kv/active_timestamp_tracker.go` with `PinWithDeadline`, `Extend`, and the per-second sweeper goroutine that reaps expired pins and emits the `backup_pin_expired` structured warning. -- **FSM plumbing for cluster-wide pins** (the propagation described - in "Cluster-wide propagation" only works if these land): - - New byte tag constants in `kv/fsm.go` alongside the existing - `raftEncodeHLCLease = 0x02` (`kv/fsm.go:116`): - `raftEncodeBackupPin = 0x03`, `raftEncodeBackupExtend = 0x04`, - `raftEncodeBackupRelease = 0x05`. New - `applyBackupPin` / `applyBackupExtend` / `applyBackupRelease` - handlers on `kvFSM`, dispatched from `kvFSM.Apply` - (`kv/fsm.go:60`) in the same shape as `applyHLCLease`. - - - **Two-version rolling-upgrade plan.** Today, an unknown leading - tag falls through `decodeRaftRequests` (`kv/fsm.go:140`) into - `decodeLegacyRaftRequest` (`kv/fsm.go:145`), which calls - `proto.Unmarshal` on the raw bytes. A `BackupPin` entry with - leading byte `0x03` would fail the unmarshal and surface as an - error from `Apply`, stalling the apply loop on any node that - hasn't been upgraded. Introducing the new tags in a single - release would therefore corrupt apply on old replicas the moment - the first `BeginBackup` lands during a rolling upgrade. - - Phase 1 ships in **two releases** to avoid this: - - 1. **Release N (forward-compat handler only).** `kvFSM.Apply` is - extended so unknown tags in the reserved range - `[0x03, 0x0F]` are treated as no-ops with a structured - `unknown_fsm_tag` warning — no error returned. The new check - is added in `kvFSM.Apply` (`kv/fsm.go:60`) **immediately after - the `raftEncodeHLCLease` guard and before the call to - `decodeRaftRequests`**, so all leading-byte dispatch lives in - one place rather than splitting the table between `Apply` and - `decodeRaftRequests`'s `default:` case. Tags `0x10` and above - still fall through to `decodeRaftRequests` and error as - before, bounding the forward-compat door. No new admin RPCs, - no producer, no `BackupPin` entries are ever written at this - version. Operators upgrade their clusters fleet-wide to - release N before the next step is enabled. - 2. **Release N+1 (BackupPin enabled).** The actual `applyBackupPin` - / `Extend` / `Release` handlers ship and the admin RPCs are - activated. `BeginBackup` itself **gates on every cluster - member reporting `node_version ≥ N`**. The naïve approach of +- **FSM plumbing for cluster-wide pins.** `kv/backup_codec.go` uses + envelope opcode `0x0e` and subtypes for reserve, pin, extend, + release, and unreserve. `kvFSM.Apply` dispatches that envelope to + `applyBackupEntry`; malformed lengths and unknown subtypes fail + closed instead of entering protobuf decoding. The reserve/unreserve + pair is committed through one deterministic control group so the + configured active-backup cap is cluster-wide rather than per admin + process. + + - **Rolling-upgrade capability gate.** `BeginBackup` is disabled + until every live member reports `backup_protocol_version >= 1`. + The naïve approach of reading versions from `Admin.GetRaftGroups` does **not** work: `GetRaftGroups` (`adapter/admin_grpc.go:382-416`) iterates the *local* `AdminServer`'s registered groups and populates each @@ -1403,7 +1398,10 @@ written. returns (GetNodeVersionResponse) {} } message GetNodeVersionRequest {} - message GetNodeVersionResponse { string node_version = 1; } + message GetNodeVersionResponse { + string node_version = 1; + uint32 backup_protocol_version = 2; + } ``` The admin server handling `BeginBackup` issues `GetNodeVersion` to **the live cluster member set, not the static @@ -1486,10 +1484,9 @@ written. Releases N and N+1 may be the same calendar release if the operator is willing to enforce a fleet-wide upgrade window before the first backup; the gate check still runs and - short-circuits if upgrade is incomplete. The reserved tag range - `[0x03, 0x0F]` lets future entry types (e.g. CDC tags for the - Phase 3 incremental backup) follow the same forward-compat - path without re-running the full upgrade dance. + short-circuits if upgrade is incomplete. Backup maintenance + entries use the existing FSM envelope opcode `0x0e` with bounded + subtypes, avoiding collisions with encrypted/protobuf wire starts. - New `*ActiveTimestampTracker` field on `kvFSM` (`kv/fsm.go:26`), wired via a new `NewKvFSMWithHLCAndTracker(store, hlc, tracker)` constructor — analogous to how `*HLC` is shared today via @@ -1501,20 +1498,23 @@ written. types. Hand-coded fixed-layout binary (matching the HLC lease style): ``` - BackupPin : [tag:1][pin_id:16][read_ts:8][deadline_ms:8] = 33 bytes - BackupExtend : [tag:1][pin_id:16][deadline_ms:8] = 25 bytes - BackupRelease : [tag:1][pin_id:16] = 17 bytes + BackupPin : [0x0e][subtype:1][pin_id:16][read_ts:8][deadline_ms:8] = 34 bytes + BackupExtend : [0x0e][subtype:2][pin_id:16][deadline_ms:8] = 26 bytes + BackupRelease : [0x0e][subtype:3][pin_id:16] = 18 bytes + BackupReserve : [0x0e][subtype:4][pin_id:16][read_ts:8][deadline_ms:8] = 34 bytes + BackupUnreserve : [0x0e][subtype:5][pin_id:16] = 18 bytes ``` `pin_id` is a UUIDv4 generated by the admin server at - `BeginBackup` time and echoed in every subsequent `BackupExtend` - / `BackupRelease` so the FSM can target the right tracker entry. + `BeginBackup` time and echoed in every subsequent pin lifecycle + entry so the FSM can target the right tracker record. Hand-coded binary (vs. proto) keeps the entry small enough to stay well within the `MaxSizePerMsg` limit (default 1 MiB, `internal/raftengine/etcd/engine.go:55`) and avoids pulling proto codegen into the FSM apply hot path. - New `kv/backup_scan.go` — `BackupScanner` iterator wrapping the existing `ShardStore.ScanAt` (`kv/shard_store.go:106`) so multi- - million-key ranges page through `ScanAt` calls of `--scan-page-size` + million-key ranges page through `ScanAt` calls sized by + `--backupScanPageSize` rather than materializing in one call. - New `MVCCStore.ScanKeysAt` / `ShardStore.ScanKeysAt` overload (returning `[][]byte` rather than `[]*KVPair`) so the expected-keys @@ -1528,12 +1528,13 @@ written. helper that takes a `keysOnly` flag, sharing the route-resolution and merge logic with the existing path. - Add `BeginBackupResponse.expected_keys` per scope (the count - produced by the baseline pass at `read_ts` before the pin is + produced by the baseline pass at `read_ts` after the pin is installed) and `ErrCompactionDuringDump` in the producer when the actual count falls below `99% × baseline ± sqrt(baseline)`. - New tool `cmd/elastickv-backup/` performing - `BeginBackup → ListAdaptersAndScopes → BackupScanner.Next* (per scope) - → encode → write directory tree → CHECKSUMS → MANIFEST.json + `BeginBackup → ListAdaptersAndScopes → StreamBackup + → encode → stop/verify renewal → write directory tree + → CHECKSUMS → MANIFEST.json (last) → EndBackup`. - Per-adapter encoders: - `internal/backup/dynamodb.go` — items + `_schema.json`. @@ -1544,8 +1545,7 @@ written. - `internal/backup/sqs.go` — `_queue.json` + `messages.jsonl`. - Filename encoding lives in `internal/backup/filename.go` with shared unit tests for round-trip safety. -- Documentation: `docs/operations/backup_restore.md` runbook (separate - PR after this design lands). +- Documentation: `docs/operations/backup_restore.md` runbook. ### Phase 2 — Restore consumer @@ -1575,6 +1575,12 @@ Scope: out of this proposal; mentioned only to draw the boundary. ## Required Tests +Phase 1 completion is gated by the control-plane, scanner, producer, format, +and manifest-publication rows below. Rows that require applying records to a +running destination (`TestRestoreReplaceMode`, `TestExternalToolReplay`, and +the Redis restore-policy tests) are Phase 2 acceptance criteria and are not +claimed by the Phase 1 implementation status at the top of this document. + ### P0 | Test | Verifies | @@ -1589,13 +1595,13 @@ Scope: out of this proposal; mentioned only to draw the boundary. | `TestBeginBackupPinFanOutAllNodes` | A 3-node cluster: `BeginBackup` issued to node A; verify nodes B and C have applied the `BackupPin` Raft entry and their compactors retain MVCC versions at `read_ts`. Compactor on B forced to run mid-dump must not retire pinned versions | | `TestBeginBackupPinSurvivesLeaderChange` | After `BeginBackup` on node A, force a leadership change on a group; the new leader still honors the pin (its FSM applied the same entry); subsequent `BackupScanner.Next` calls succeed | | `TestBeginBackupGroupUnreachable` | If one group cannot commit `BackupPin` within `--begin-backup-deadline`, `BeginBackup` returns `Unavailable` and proposes `BackupRelease` on every group that did commit; no stranded pins remain | -| `TestBackupPinFSMCodecRoundTrip` | `BackupPin` / `BackupExtend` / `BackupRelease` byte layouts (33 / 25 / 17 bytes) round-trip through the FSM apply path; unknown tag bytes return `ErrUnknownRequestType` rather than panicking | -| `TestRestoreWipesLocalPins` | A replica that installs a Raft snapshot during a backup loses its `BackupPin`; the producer's per-scope expected-keys baseline detects the resulting `ScanAt` shortfall (count below `99% × baseline ± sqrt(baseline)`) and fails the dump with `ErrCompactionDuringDump` rather than emitting a corrupted artifact | -| `TestBeginBackupRefusesNearSnapshotThreshold` | When any group's `SnapshotEvery - (AppliedIndex - LastSnapshotIndex) < --snapshot-headroom-entries`, `BeginBackup` returns `FailedPrecondition` rather than starting a dump that risks the snapshot-installation path. Verify a freshly-snapshotted cluster (largest remaining headroom) is allowed | +| `TestBackupPinFSMCodecRoundTrip` | The `0x0e` envelope and reserve/pin/extend/release/unreserve subtypes round-trip through the FSM apply path; malformed payloads and unknown subtypes fail closed | +| `TestRestoreWipesLocalPins` | A replica that installs a Raft snapshot during a backup loses its local pin; the next complete-pin renewal restores the fence, while any resulting count shortfall still fails with `ErrCompactionDuringDump` before manifest publication | +| `TestBeginBackupRefusesNearSnapshotThreshold` | When any group's `SnapshotEvery - (AppliedIndex - LastSnapshotIndex) < --backupSnapshotHeadroomEntries`, `BeginBackup` returns `FailedPrecondition` rather than starting a dump that risks the snapshot-installation path. Verify a freshly-snapshotted cluster (largest remaining headroom) is allowed | | `TestExpectedKeysBaselineToleratesTTLExpiry` | Routine TTL expiry between baseline and dump (1% of keys gone) does NOT trigger `ErrCompactionDuringDump`; a 5% drop DOES | | `TestExpectedKeysBaselineCountOnlyScan` | The baseline pass uses the new `ShardStore.ScanKeysAt`; verify per-key allocation is bounded (no value materialization) on a 1M-key scope and that existing `ScanAt` callers are bytewise-unchanged | | `TestExpectedKeysBaselineRunsAfterShardCatchup` | Force shard B to lag at `BeginBackup` time; verify the baseline scan blocks until step 2 (catch-up) completes for B, so B's `expected_keys[i].key_count` is the true count at `read_ts` rather than the partial-pre-catch-up count | -| `TestForwardCompatUnknownFSMTag` | A release-N FSM receiving a synthetic `0x03`-tagged entry (representing a future `BackupPin` from a release-N+1 leader) returns nil from `Apply` and emits an `unknown_fsm_tag` warning, with no apply-loop stall and no FSM mutation. Tags `0x10+` (outside the reserved range) still error so the forward-compat door is bounded | +| `TestBackupFSMEnvelopeDoesNotCollideWithProto` | `0x0e` remains invalid as a protobuf wire start and is dispatched only by the explicit backup envelope handler | | `TestBeginBackupGatesOnMinVersion` | The `BeginBackup` handler derives its fan-out target from `snapshotMembers` (live `Configuration()` union, not the static `AdminServer.members` seed) and issues `GetNodeVersion` per peer concurrently. If any peer returns `node_version < N` or is unreachable within `--begin-backup-deadline`, the call returns `FailedPrecondition` with an error message that distinguishes the two cases ("reports version vX" vs. "did not respond within Ns") and proposes no `BackupPin` entries on any group; once every live member reports `node_version ≥ N`, the same call succeeds | | `TestBeginBackupGatesIncludesDynamicallyAddedNodes` | Start a 3-node cluster, add a 4th node at `v(N-1)` via `AddVoter` after `AdminServer` startup (so it is NOT in the bootstrap `members` seed), then call `BeginBackup`; the gate fails with the new node identified in the error. The previous seed-only design would have falsely passed | | `TestGetRaftGroupsLeaderVersionAsync` | `GetRaftGroups` returns within local-snapshot latency even when a leader is unreachable; `leader_node_version` is empty string in that case. Verify the 500 ms per-leader timeout and the 10 s response cache prevent repeated fan-outs across rapid polls | @@ -1603,7 +1609,7 @@ Scope: out of this proposal; mentioned only to draw the boundary. | `TestBeginBackupPropagatesAdminAuthToken` | A cluster booted with `--adminToken` accepts a `BeginBackup` call carrying `authorization: Bearer `; the handler propagates the same metadata via `metadata.NewOutgoingContext` to every `GetNodeVersion` fan-out dial, so peers return their version (not `Unauthenticated`). A `BeginBackup` call without the token is itself rejected before any fan-out happens | | `TestVersionCacheRaceUnderLoad` | `go test -race` with 50 concurrent `GetRaftGroups` callers and async `GetNodeVersion` probe goroutines writing the cache simultaneously emits no data-race report; the `sync.Map` choice is enforced by the lack of a separate `versionCacheMu` field on `AdminServer` | | `TestSnapshotEveryReadsFromEngine` | A node started with `ELASTICKV_RAFT_SNAPSHOT_COUNT=5000` reports `Engine.SnapshotEvery() == 5000`; `BeginBackup` uses 5000 (not the default 10000) when computing remaining headroom | -| `TestRenewBackupRetriesLeaderElection` | Force a leader election mid-`RenewBackup`; the admin server retries `BackupExtend` up to 3 times with 500ms backoff and succeeds once the new leader is established, without aborting the dump | +| `TestRenewBackupRetriesLeaderElection` | Force a leader election mid-`RenewBackup`; the admin server retries the complete replacement pin up to 3 times with 500ms backoff and succeeds once the new leader is established, without aborting the dump | | `TestPinWithDeadlineExpiry` | `PinWithDeadline(ts, now+100ms)` is auto-released by the sweeper after the deadline; compactor unblocked; `backup_pin_expired` log emitted | | `TestBeginBackupWaitsForLaggingShard` | Force shard B's `applied_index` to lag; `BeginBackup` polls until it catches up or times out with `FailedPrecondition`; no scan starts in the timeout case | | `TestBackupScannerPaging` | A range with > pageSize keys is returned across multiple `ScanAt` pages with no overlap, no gaps; iteration tolerates concurrent writes by completing at the pinned `read_ts` | @@ -1611,6 +1617,7 @@ Scope: out of this proposal; mentioned only to draw the boundary. | `TestS3PathFileVsDirectoryCollision` | Bucket holds both `path/to` (object) and `path/to/obj`; producer renames the shorter key to `path/to.elastickv-leaf-data` and records it in `KEYMAP.jsonl`; restore tool reverses it via `MANIFEST.s3_collision_strategy` | | `TestBeginBackupTooManyActiveBackups` | Reaching `max_active_backup_pins` returns `ResourceExhausted`; releasing one pin frees a slot for the next request | | `TestRenewBackupExtendsDeadline` | `RenewBackup` shifts the deadline; producer's failed-renewal path aborts the dump with a critical log line rather than continuing past the TTL | +| `TestRunLiveBackupStopsRenewalBeforeManifestPublication` | The producer waits for an in-flight renewal to quiesce before `MANIFEST.json` publication, closing the completion-marker race | | `TestRenewBackupTTLRangeValidation` | `RenewBackup` with `ttl_ms < 60s` or `ttl_ms > backup_max_ttl_ms` returns `InvalidArgument`; in-range values succeed | | `TestListAdaptersAndScopesAtPinTS` | A scope created (e.g. CreateTable) after `BeginBackup` is not surfaced by `ListAdaptersAndScopes(pin_token)`; pre-existing scopes are | diff --git a/docs/design/2026_04_29_implemented_snapshot_logical_decoder.md b/docs/design/2026_04_29_implemented_snapshot_logical_decoder.md index 07205fc7c..ae67b5f69 100644 --- a/docs/design/2026_04_29_implemented_snapshot_logical_decoder.md +++ b/docs/design/2026_04_29_implemented_snapshot_logical_decoder.md @@ -62,7 +62,7 @@ backup: **vendor-independent recovery**. The full live-cluster, point-in-time-consistent extraction pipeline described in -`2026_04_29_proposed_logical_backup.md` (the Phase 1 design) +`2026_04_29_implemented_logical_backup.md` (the Phase 1 design) introduces non-trivial machinery: replicated `BackupPin` Raft FSM commands, version-gated RPC fan-outs, expected-keys baselines, admin-side connection caches, etc. That machinery exists to chase a @@ -731,7 +731,7 @@ bespoke parser, the format has failed its goal. - `2026_04_14_implemented_etcd_snapshot_disk_offload.md` — the `.fsm` file format Phase 0 reads and writes. -- `2026_04_29_proposed_logical_backup.md` — Phase 1 (live PIT +- `2026_04_29_implemented_logical_backup.md` — Phase 1 (live PIT extraction). Phase 1 produces dumps in the same format defined here. - `internal/s3keys/keys.go`, `kv/shard_key.go`, `adapter/sqs_keys.go`, diff --git a/docs/design/2026_05_25_implemented_snapshot_logical_encoder.md b/docs/design/2026_05_25_implemented_snapshot_logical_encoder.md index 6727a0186..aaf229fdc 100644 --- a/docs/design/2026_05_25_implemented_snapshot_logical_encoder.md +++ b/docs/design/2026_05_25_implemented_snapshot_logical_encoder.md @@ -463,7 +463,7 @@ P2: - `2026_04_29_implemented_snapshot_logical_decoder.md` — Phase 0 format owner; Phase 0a (decoder), Phase 0b (encoder), and Phase 0c operator integration are shipped. -- `2026_04_29_proposed_logical_backup.md` — Phase 1 (live PIT +- `2026_04_29_implemented_logical_backup.md` — Phase 1 (live PIT extraction); produces the same directory format this encoder reads. - `2026_04_14_implemented_etcd_snapshot_disk_offload.md` — the `.fsm` / `.snap` EKVT token format the restore runbook seeds. diff --git a/docs/operations/backup_restore.md b/docs/operations/backup_restore.md new file mode 100644 index 000000000..874294f47 --- /dev/null +++ b/docs/operations/backup_restore.md @@ -0,0 +1,188 @@ +# Live Logical Backup Runbook + +This runbook covers Phase 1 point-in-time logical backups from a running +elastickv cluster. The backup tree is vendor-independent and can also be +converted into a native snapshot for the existing stop-replace-restart restore +path. + +## Preconditions + +- Every cluster member runs a build with the same live-backup protocol. The + server rejects `BeginBackup` when a current member is unreachable or lacks + the required protocol. +- The Admin gRPC endpoint is reachable from the backup host. Use the existing + Admin bearer token and TLS configuration for non-loopback endpoints. +- MVCC retention exceeds the expected backup duration plus compaction lag. +- Every Raft group has enough snapshot headroom. A rejected `BeginBackup` is a + safety result; retry after the affected group takes a snapshot instead of + weakening the server policy without a retention review. +- The destination directory does not exist. The producer never merges with or + overwrites an earlier dump. + +Relevant server controls are: + +```text +--backupDefaultTTL +--backupMaxTTL +--backupBeginDeadline +--backupSnapshotHeadroomEntries +--backupScanPageSize +--backupMaxActivePins +``` + +The defaults are a 30-minute pin, a 1-hour maximum renewal TTL, a 5-second +server deadline, 1000 snapshot-headroom entries, 1024 keys per baseline page, +and four concurrent backup pins. + +## Create A Directory Backup + +```sh +elastickv-backup dump \ + --address 192.168.0.210:50051 \ + --output-dir /backups/elastickv-2026-07-18 \ + --cluster-id production \ + --admin-token-file /run/secrets/elastickv-admin-token \ + --ttl-ms 1800000 +``` + +For a TLS Admin endpoint, add the CA and expected server name: + +```text +--tls-ca-cert-file /etc/elastickv/admin-ca.pem +--tls-server-name elastickv-admin.internal +``` + +`--tls-insecure-skip-verify` is intended only for controlled development +environments. + +Limit the dump with `--adapter` and repeatable `--scope` flags: + +```sh +elastickv-backup dump \ + --address 192.168.0.210:50051 \ + --output-dir /backups/orders-and-photos \ + --cluster-id production \ + --admin-token-file /run/secrets/elastickv-admin-token \ + --adapter dynamodb,s3 \ + --scope dynamodb=orders \ + --scope s3=photos +``` + +For large DynamoDB tables, `--dynamodb-bundle-mode jsonl` avoids one inode per +item. `--dynamodb-bundle-size` defaults to `64MiB` in that mode. + +## Stream An Archive + +The producer always builds and validates the directory tree first. It can then +write a tar or zstd-compressed tar stream to stdout: + +```sh +elastickv-backup dump \ + --address 192.168.0.210:50051 \ + --output-dir /var/tmp/elastickv-live-backup \ + --cluster-id production \ + --admin-token-file /run/secrets/elastickv-admin-token \ + --output-format tar+zstd \ + > /backups/elastickv-live-backup.tar.zst +``` + +The archive is emitted only after `MANIFEST.json` and `CHECKSUMS` validate. +Keep stderr separate from stdout because stdout carries the archive bytes. + +## Verify Completion + +A dump is complete only when `MANIFEST.json` exists. The producer writes it +atomically after every adapter has finalized and after `CHECKSUMS` includes the +prepared manifest digest. + +```sh +test -f /backups/elastickv-2026-07-18/MANIFEST.json +cd /backups/elastickv-2026-07-18 +sha256sum -c CHECKSUMS +jq '{phase, cluster_id, last_commit_ts, live, adapters}' MANIFEST.json +``` + +On macOS, use `shasum -a 256 -c CHECKSUMS`. + +Expected properties include: + +- `phase` is `phase1-live-pinned`; +- `live.read_ts` is non-zero; +- every selected adapter and scope is present; +- checksum verification reports success for every entry, including + `MANIFEST.json`. + +The completion log reports `read_ts`, scope and record counts, +`pin_renewals_total`, and the selected output format. For a long-running dump, +the renewal count should increase. A completed backup with zero renewals is +normal when the dump finishes within the first third of its effective TTL. + +## Monitor A Running Backup + +Monitor the backup process together with every cluster member. The server-side +signals that require investigation are `backup_pin_expired`, failed backup-pin +proposals, snapshot-headroom refusal, and repeated leader-election errors during +renewal. A renewal failure is terminal for that run; the producer cancels the +stream and never publishes `MANIFEST.json`. + +Track free space and inode use on the backup host while the dump runs: + +```sh +df -h /backups +df -i /backups +du -sh /backups/elastickv-2026-07-18 +``` + +Per-item DynamoDB output can consume one inode per item. Switch future runs to +`--dynamodb-bundle-mode jsonl` when inode pressure, rather than bytes, is the +limiting resource. + +## Failure Handling + +- No `MANIFEST.json`: the tree is incomplete and must not be restored or + archived as a valid backup. Retain it only for diagnosis, then choose a new + output directory for the retry. +- `ResourceExhausted`: the cluster reached `--backupMaxActivePins`. Find the + active backup jobs or wait for their TTL cleanup before retrying. +- `FailedPrecondition` at begin: inspect the reported old or unreachable + member, snapshot headroom, or shard catch-up failure. No scan has started. +- Renewal failure: the producer cancels the stream, omits `MANIFEST.json`, and + calls `EndBackup` with the latest token. The replicated TTL remains the final + cleanup bound if the client is terminated. +- Expected-key shortfall: treat it as a consistency failure. Do not reuse the + partial tree; investigate retention, snapshot installation, and compaction. +- `EndBackup` failure after manifest publication: the artifact is complete, + but the command reports failure because cleanup was not confirmed. Verify + the dump, monitor pin expiry, and investigate Admin/Raft reachability. +- Archive-stream failure after manifest publication: the directory tree is a + complete backup, but stdout may contain a truncated archive. Discard the + archive stream, verify the directory tree again, and repack from that tree. + +## Restore Through A Native Snapshot + +Phase 1 produces the same logical tree accepted by the offline snapshot +encoder. To restore into a fresh or stopped cluster, first encode and self-test +the tree: + +```sh +elastickv-snapshot-encode \ + --input /backups/elastickv-2026-07-18 \ + --output /restore/elastickv-2026-07-18.fsm \ + --self-test +``` + +Then follow [Snapshot Restore Runbook](./snapshot_restore.md) to prepare a new +Raft data directory and start the replacement cluster. Do not merge the +prepared directory into an existing WAL or snapshot directory. + +Direct replay into a running cluster through public DynamoDB, S3, Redis, and +SQS endpoints is the separate Phase 2 restore consumer and is not part of the +Phase 1 producer. + +## Periodic Recovery Drill + +Treat checksum verification alone as artifact validation, not recovery proof. +On a schedule, encode a completed logical tree with `--self-test`, restore it +into an isolated replacement cluster, and compare adapter scope counts and +representative object/item/key/message contents at the recorded +`live.read_ts`. Do not run the drill against an existing Raft data directory. diff --git a/docs/operations/snapshot_restore.md b/docs/operations/snapshot_restore.md index 66d8a40ae..e82d02e28 100644 --- a/docs/operations/snapshot_restore.md +++ b/docs/operations/snapshot_restore.md @@ -98,6 +98,7 @@ It refuses to proceed when: 5. Add other nodes as fresh members so they catch up from the seeded node's snapshot. -Phase 0 restores one snapshot's state. It does not create a new live point-in- -time backup and does not provide cross-shard consistency. Use a Phase 1 live -extractor once that path exists when those properties are required. +This runbook restores one logical tree through a native snapshot. To create a +cluster-wide point-in-time tree from a running cluster, first follow the +[Live Logical Backup Runbook](./backup_restore.md), then return to the Encode +step above. diff --git a/internal/backup/checksums.go b/internal/backup/checksums.go index 17bc029eb..cc5463fd4 100644 --- a/internal/backup/checksums.go +++ b/internal/backup/checksums.go @@ -24,7 +24,10 @@ import ( // arbitrarily-long "line" — that is the OOM vector the gemini // security-high finding on PR #810 flagged when the previous // implementation slurped the whole file via os.ReadFile. -const maxChecksumLineLen = 8 * 1024 +const ( + maxChecksumLineLen = 8 * 1024 + checksumFileMode = 0o600 +) // CHECKSUMSFilename is the on-disk name of the dump-tree-wide // sha256sum(1)-compatible checksum file. Stored at the dump root so @@ -66,10 +69,43 @@ func WriteChecksums(root string) error { if err != nil { return err } + return writeChecksumEntries(root, entries, false) +} + +// WriteChecksumsWithVirtualFile includes one not-yet-published file in the +// checksum set. Live backup uses this to checksum MANIFEST.json before its +// final atomic publication, keeping the manifest both checksummed and last. +func WriteChecksumsWithVirtualFile(root, relPath string, content []byte) error { + if !filepath.IsLocal(relPath) || filepath.Clean(relPath) == "." { + return errors.Wrapf(ErrChecksumsPathTraversal, "%q", relPath) + } + relPath = filepath.ToSlash(filepath.Clean(relPath)) + if relPath == CHECKSUMSFilename { + return errors.Wrap(ErrChecksumsMalformedLine, "CHECKSUMS cannot list itself") + } + entries, err := collectChecksumEntries(root) + if err != nil { + return err + } + for _, entry := range entries { + if entry.relPath == relPath { + return errors.Wrapf(ErrChecksumsMalformedLine, "virtual file already exists: %s", relPath) + } + } + sum := sha256.Sum256(content) + entries = append(entries, checksumEntry{hex: hex.EncodeToString(sum[:]), relPath: relPath}) + return writeChecksumEntries(root, entries, true) +} + +func writeChecksumEntries(root string, entries []checksumEntry, exclusive bool) error { sort.Slice(entries, func(i, j int) bool { return entries[i].relPath < entries[j].relPath }) - out, err := os.Create(filepath.Join(root, CHECKSUMSFilename)) //nolint:gosec // operator-supplied output dir + flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC + if exclusive { + flags = os.O_WRONLY | os.O_CREATE | os.O_EXCL + } + out, err := os.OpenFile(filepath.Join(root, CHECKSUMSFilename), flags, checksumFileMode) //nolint:gosec // operator-supplied output dir if err != nil { return errors.WithStack(err) } diff --git a/internal/backup/decode.go b/internal/backup/decode.go index eb3db251a..4e906ff8b 100644 --- a/internal/backup/decode.go +++ b/internal/backup/decode.go @@ -87,6 +87,8 @@ type DecodeOptions struct { // DynamoDBBundleJSONL switches the DynamoDB encoder to the JSONL // bundle layout (`items/data-.jsonl`). DynamoDBBundleJSONL bool + // DynamoDBBundleSizeBytes caps each JSONL part. Zero uses 64 MiB. + DynamoDBBundleSizeBytes int64 // WarnSink, when non-nil, receives structured warnings from the // per-adapter encoders ("redis_orphan_ttl", "ddb_orphan_items", @@ -188,10 +190,14 @@ func newDispatcher(opts DecodeOptions) (*dispatcher, error) { if opts.OutRoot == "" { return nil, errors.Wrap(ErrDecodeOptionsInvalid, "OutRoot required") } + if opts.DynamoDBBundleSizeBytes < 0 { + return nil, errors.Wrap(ErrDecodeOptionsInvalid, "DynamoDBBundleSizeBytes must not be negative") + } d := &dispatcher{opts: opts} if opts.Adapters.DynamoDB { d.ddb = NewDDBEncoder(opts.OutRoot). WithBundleJSONL(opts.DynamoDBBundleJSONL). + WithBundleSizeBytes(opts.DynamoDBBundleSizeBytes). WithWarnSink(opts.WarnSink) } if opts.Adapters.S3 { diff --git a/internal/backup/dynamodb.go b/internal/backup/dynamodb.go index f2c384557..b9a69b2ff 100644 --- a/internal/backup/dynamodb.go +++ b/internal/backup/dynamodb.go @@ -25,6 +25,10 @@ const ( DDBTableGenPrefix = "!ddb|meta|gen|" DDBItemPrefix = "!ddb|item|" DDBGSIPrefix = "!ddb|gsi|" + + defaultDDBBundleSizeBytes = 64 << 20 + ddbUnbundledWarnItems = 1_000_000 + ddbGenerationSlots = 2 ) // Stored value magic prefixes. Mirror adapter/dynamodb_storage_codec.go:15-16. @@ -38,9 +42,10 @@ var ( // ErrDDBInvalidSchema, ErrDDBInvalidItem, ErrDDBMalformedKey are the // typed error classes for this encoder. Surface via errors.Is. var ( - ErrDDBInvalidSchema = errors.New("backup: invalid !ddb|meta|table value") - ErrDDBInvalidItem = errors.New("backup: invalid !ddb|item value") - ErrDDBMalformedKey = errors.New("backup: malformed DynamoDB key") + ErrDDBInvalidSchema = errors.New("backup: invalid !ddb|meta|table value") + ErrDDBInvalidItem = errors.New("backup: invalid !ddb|item value") + ErrDDBMalformedKey = errors.New("backup: malformed DynamoDB key") + ErrDDBBundleItemTooLarge = errors.New("backup: DynamoDB JSONL item exceeds bundle size") ) // DDBEncoder encodes the DynamoDB prefix family into the per-table layout @@ -59,6 +64,7 @@ var ( type DDBEncoder struct { outRoot string bundleJSONL bool + bundleSize int64 tables map[string]*ddbTableState @@ -82,8 +88,9 @@ func ensureItemsByGen(m map[uint64][]*pb.DynamoItem) map[uint64][]*pb.DynamoItem // NewDDBEncoder constructs an encoder rooted at /dynamodb/. func NewDDBEncoder(outRoot string) *DDBEncoder { return &DDBEncoder{ - outRoot: outRoot, - tables: make(map[string]*ddbTableState), + outRoot: outRoot, + bundleSize: defaultDDBBundleSizeBytes, + tables: make(map[string]*ddbTableState), } } @@ -91,15 +98,20 @@ func NewDDBEncoder(outRoot string) *DDBEncoder { // (one item per line). Default is per-item files. The choice is recorded // in MANIFEST.json (`dynamodb_layout`) by the master pipeline; the // encoder itself only needs the flag to pick the on-disk shape. -// -// Bundle mode is a follow-up: this PR ships per-item only. Calling -// WithBundleJSONL(true) returns an error from Finalize until the bundle -// path lands. func (d *DDBEncoder) WithBundleJSONL(on bool) *DDBEncoder { d.bundleJSONL = on return d } +// WithBundleSizeBytes sets the maximum size of one data-.jsonl file. +// Zero keeps the 64 MiB default. +func (d *DDBEncoder) WithBundleSizeBytes(size int64) *DDBEncoder { + if size > 0 { + d.bundleSize = size + } + return d +} + // WithWarnSink wires structured-warning emission (orphan items, // schema-less tables, etc.). func (d *DDBEncoder) WithWarnSink(fn func(event string, fields ...any)) *DDBEncoder { @@ -184,9 +196,6 @@ func (d *DDBEncoder) HandleTableGen(_, _ []byte) error { return nil } // immediately rather than being attributed to a later table (Gemini // MEDIUM #182). func (d *DDBEncoder) Finalize() error { - if d.bundleJSONL { - return errors.New("backup: dynamodb_layout=jsonl not implemented in this PR") - } for _, st := range d.tables { if st.schema == nil { d.emitWarn("ddb_orphan_items", @@ -232,27 +241,142 @@ func (d *DDBEncoder) flushTable(st *ddbTableState) error { // authoritative one (the live code prefers it on read). We emit // migration-source first, then active gen LAST, so writeFileAtomic's // tmp+rename leaves the active-gen content on disk per (pk,sk). - emitOrder := []uint64{} + emitOrder := ddbGenerationEmitOrder(activeGen, migrationSourceGen) + d.warnStaleGenerationItems(st, emitOrder) + items, err := effectiveDDBItems(st.itemsByGen, emitOrder, hashKey, rangeKey) + if err != nil { + return err + } + return d.writeEffectiveDDBItems(itemsDir, st.name, hashKey, rangeKey, items) +} + +func ddbGenerationEmitOrder(activeGen, migrationSourceGen uint64) []uint64 { + emitOrder := make([]uint64, 0, ddbGenerationSlots) if migrationSourceGen != 0 && migrationSourceGen != activeGen { emitOrder = append(emitOrder, migrationSourceGen) } - emitOrder = append(emitOrder, activeGen) + return append(emitOrder, activeGen) +} + +func (d *DDBEncoder) warnStaleGenerationItems(st *ddbTableState, emitOrder []uint64) { if stale := totalStaleItemsExcluding(st.itemsByGen, emitOrder); stale > 0 { d.emitWarn("ddb_stale_generation_items", "table", st.name, - "active_generation", activeGen, - "migration_source_generation", migrationSourceGen, + "active_generation", st.schema.GetGeneration(), + "migration_source_generation", st.schema.GetMigratingFromGeneration(), "stale_count", stale, "hint", "stale-gen rows excluded; restore would otherwise emit them under the new schema") } +} + +func (d *DDBEncoder) writeEffectiveDDBItems( + itemsDir string, + tableName string, + hashKey string, + rangeKey string, + items []*pb.DynamoItem, +) error { + if d.bundleJSONL { + return writeDDBJSONLBundles(itemsDir, items, d.bundleSize) + } + if len(items) > ddbUnbundledWarnItems { + d.emitWarn("ddb_unbundled_large_scope", "table", tableName, "items", len(items)) + } + for _, item := range items { + if err := writeDDBItem(itemsDir, hashKey, rangeKey, item); err != nil { + return err + } + } + return nil +} + +func effectiveDDBItems( + itemsByGen map[uint64][]*pb.DynamoItem, + emitOrder []uint64, + hashKey string, + rangeKey string, +) ([]*pb.DynamoItem, error) { + byKey := make(map[string]*pb.DynamoItem) for _, gen := range emitOrder { - for _, item := range st.itemsByGen[gen] { - if err := writeDDBItem(itemsDir, hashKey, rangeKey, item); err != nil { + for _, item := range itemsByGen[gen] { + key, err := ddbItemIdentity(hashKey, rangeKey, item) + if err != nil { + return nil, err + } + byKey[key] = item + } + } + keys := make([]string, 0, len(byKey)) + for key := range byKey { + keys = append(keys, key) + } + sort.Strings(keys) + out := make([]*pb.DynamoItem, 0, len(keys)) + for _, key := range keys { + out = append(out, byKey[key]) + } + return out, nil +} + +func ddbItemIdentity(hashKey, rangeKey string, item *pb.DynamoItem) (string, error) { + attrs := item.GetAttributes() + hashVal, ok := attrs[hashKey] + if !ok { + return "", errors.Wrapf(ErrDDBInvalidItem, "item missing hash-key attribute %q", hashKey) + } + hashSegment, err := ddbKeyAttrToSegment(hashVal) + if err != nil { + return "", err + } + if rangeKey == "" { + return hashSegment, nil + } + rangeVal, ok := attrs[rangeKey] + if !ok { + return "", errors.Wrapf(ErrDDBInvalidItem, "item missing range-key attribute %q", rangeKey) + } + rangeSegment, err := ddbKeyAttrToSegment(rangeVal) + if err != nil { + return "", err + } + return hashSegment + "\x00" + rangeSegment, nil +} + +func writeDDBJSONLBundles(itemsDir string, items []*pb.DynamoItem, maxBytes int64) error { + if maxBytes <= 0 { + return errors.New("backup: DynamoDB JSONL bundle size must be positive") + } + part := 1 + var body []byte + flush := func() error { + if len(body) == 0 { + return nil + } + path := filepath.Join(itemsDir, fmt.Sprintf("data-%05d.jsonl", part)) + if err := writeFileAtomic(path, body); err != nil { + return err + } + part++ + body = body[:0] + return nil + } + for _, item := range items { + line, err := json.Marshal(itemToPublic(item)) + if err != nil { + return errors.WithStack(err) + } + line = append(line, '\n') + if int64(len(line)) > maxBytes { + return errors.Wrapf(ErrDDBBundleItemTooLarge, "item line is %d bytes, bundle limit is %d", len(line), maxBytes) + } + if int64(len(body)+len(line)) > maxBytes { + if err := flush(); err != nil { return err } } + body = append(body, line...) } - return nil + return flush() } func totalStaleItemsExcluding(m map[uint64][]*pb.DynamoItem, included []uint64) int { diff --git a/internal/backup/dynamodb_test.go b/internal/backup/dynamodb_test.go index fdc772e4d..f580da9cf 100644 --- a/internal/backup/dynamodb_test.go +++ b/internal/backup/dynamodb_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "testing" pb "github.com/bootjp/elastickv/proto" @@ -325,14 +326,81 @@ func TestDDB_AttributeValueToPublic_EmptyOneofSurfacedAsNull(t *testing.T) { } } -func TestDDB_BundleJSONLNotImplementedYet(t *testing.T) { +func TestDDB_BundleJSONLSplitsWithinLimitAndUsesActiveGeneration(t *testing.T) { t.Parallel() - enc, _ := newDDBEncoder(t) - enc.WithBundleJSONL(true) - err := enc.Finalize() - if err == nil { - t.Fatalf("expected not-implemented error from Finalize on bundle mode") + enc, root := newDDBEncoder(t) + enc.WithBundleJSONL(true).WithBundleSizeBytes(60) + schema := &pb.DynamoTableSchema{ + TableName: "t", PrimaryKey: &pb.DynamoKeySchema{HashKey: "pk"}, + AttributeDefinitions: map[string]string{"pk": "S"}, Generation: 2, MigratingFromGeneration: 1, + } + addDDBBundleTestItems(t, enc) + if err := enc.HandleTableMeta(EncodeDDBTableMetaKey("t"), encodeSchemaValue(t, schema)); err != nil { + t.Fatalf("HandleTableMeta: %v", err) + } + if err := enc.Finalize(); err != nil { + t.Fatalf("Finalize: %v", err) + } + rows := readDDBBundleRows(t, root, 60) + assertDDBBundleRows(t, rows) +} + +func addDDBBundleTestItems(t *testing.T, enc *DDBEncoder) { + t.Helper() + for gen, value := range map[uint64]string{1: "old", 2: "new"} { + item := &pb.DynamoItem{Attributes: map[string]*pb.DynamoAttributeValue{"pk": sAttr("same"), "value": sAttr(value)}} + if err := enc.HandleItem(EncodeDDBItemKey("t", gen, "same", ""), encodeItemValue(t, item)); err != nil { + t.Fatalf("HandleItem: %v", err) + } + } + for _, key := range []string{"a", "b"} { + item := &pb.DynamoItem{Attributes: map[string]*pb.DynamoAttributeValue{"pk": sAttr(key)}} + if err := enc.HandleItem(EncodeDDBItemKey("t", 2, key, ""), encodeItemValue(t, item)); err != nil { + t.Fatalf("HandleItem: %v", err) + } + } +} + +func assertDDBBundleRows(t *testing.T, rows []map[string]any) { + t.Helper() + if len(rows) != 3 { + t.Fatalf("rows=%d, want 3 deduplicated rows", len(rows)) + } + for _, row := range rows { + pk := mustSubMap(t, row, "pk")["S"] + if pk == "same" && mustSubMap(t, row, "value")["S"] != "new" { + t.Fatalf("migration source won over active row: %v", row) + } + } +} + +func readDDBBundleRows(t *testing.T, root string, maxPartBytes int) []map[string]any { + t.Helper() + parts, err := filepath.Glob(filepath.Join(root, "dynamodb", "t", "items", "data-*.jsonl")) + if err != nil { + t.Fatalf("Glob: %v", err) + } + if len(parts) < 2 { + t.Fatalf("parts=%v, want split output", parts) + } + var rows []map[string]any + for _, part := range parts { + body, err := os.ReadFile(part) //nolint:gosec // test path + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if len(body) > maxPartBytes { + t.Fatalf("part %s is %d bytes, limit %d", part, len(body), maxPartBytes) + } + for _, line := range strings.Split(strings.TrimSpace(string(body)), "\n") { + var row map[string]any + if err := json.Unmarshal([]byte(line), &row); err != nil { + t.Fatalf("Unmarshal row: %v", err) + } + rows = append(rows, row) + } } + return rows } func TestDDB_MigrationSourceGenerationItemsAreEmitted(t *testing.T) { diff --git a/internal/backup/filename.go b/internal/backup/filename.go index 04d0fbc27..cf0793f83 100644 --- a/internal/backup/filename.go +++ b/internal/backup/filename.go @@ -1,6 +1,6 @@ // Package backup implements the per-adapter logical-backup format defined in // docs/design/2026_04_29_proposed_snapshot_logical_decoder.md (Phase 0) and -// reused by docs/design/2026_04_29_proposed_logical_backup.md (Phase 1). +// reused by docs/design/2026_04_29_implemented_logical_backup.md (Phase 1). // // This file owns the filename encoding rules for non-S3 segments. S3 object // keys preserve their `/` separators (and so are not transformed by EncodeSegment); diff --git a/internal/backup/finalize.go b/internal/backup/finalize.go new file mode 100644 index 000000000..50e846f84 --- /dev/null +++ b/internal/backup/finalize.go @@ -0,0 +1,98 @@ +package backup + +import ( + "bytes" + "io" + "os" + "path/filepath" + + "github.com/cockroachdb/errors" +) + +// FinalizeDump writes CHECKSUMS and then atomically publishes MANIFEST.json as +// the final dump-tree entry. A failure or process crash before publication +// leaves no manifest, so restore tooling cannot mistake a partial dump for a +// complete artifact. +func FinalizeDump(root string, manifest Manifest) error { + payload, err := marshalManifestPayload(manifest) + if err != nil { + return err + } + manifestPath := filepath.Join(root, ManifestFilename) + if err := requireManifestAbsent(manifestPath); err != nil { + return err + } + if err := WriteChecksumsWithVirtualFile(root, ManifestFilename, payload); err != nil { + return errors.Wrap(err, "write CHECKSUMS") + } + return publishManifest(root, manifestPath, payload) +} + +func marshalManifestPayload(manifest Manifest) ([]byte, error) { + var payload bytes.Buffer + if err := WriteManifest(&payload, manifest); err != nil { + return nil, err + } + return payload.Bytes(), nil +} + +func requireManifestAbsent(manifestPath string) error { + if _, err := os.Lstat(manifestPath); err == nil { + return errors.New("backup: MANIFEST.json already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return errors.WithStack(err) + } + return nil +} + +func publishManifest(root, manifestPath string, payload []byte) error { + tmpPath, err := writeManifestTemp(root, payload) + if err != nil { + return err + } + defer func() { _ = os.Remove(tmpPath) }() + if err := os.Rename(tmpPath, manifestPath); err != nil { + return errors.WithStack(err) + } + if err := syncBackupDirectory(root); err != nil { + _ = os.Remove(manifestPath) + return err + } + return nil +} + +func writeManifestTemp(root string, payload []byte) (path string, retErr error) { + tmp, err := os.CreateTemp(root, ".MANIFEST.json.*.tmp") + if err != nil { + return "", errors.WithStack(err) + } + path = tmp.Name() + defer func() { + if retErr != nil { + _ = tmp.Close() + _ = os.Remove(path) + } + }() + if _, err := io.Copy(tmp, bytes.NewReader(payload)); err != nil { + return path, errors.WithStack(err) + } + if err := tmp.Sync(); err != nil { + return path, errors.WithStack(err) + } + if err := tmp.Close(); err != nil { + return path, errors.WithStack(err) + } + return path, nil +} + +func syncBackupDirectory(path string) error { + dir, err := os.Open(path) //nolint:gosec // operator-selected dump root + if err != nil { + return errors.WithStack(err) + } + if err := dir.Sync(); err != nil { + _ = dir.Close() + return errors.WithStack(err) + } + return errors.WithStack(dir.Close()) +} diff --git a/internal/backup/finalize_test.go b/internal/backup/finalize_test.go new file mode 100644 index 000000000..f68d6e673 --- /dev/null +++ b/internal/backup/finalize_test.go @@ -0,0 +1,72 @@ +package backup + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestFinalizeDumpChecksumsManifestAndPublishesItLast(t *testing.T) { + t.Parallel() + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "data.txt"), []byte("value"), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + manifest := NewPhase0SnapshotManifest(time.Now()) + manifest.Phase = PhasePhase1LivePinned + manifest.Live = &Live{ReadTS: 42} + + if err := FinalizeDump(root, manifest); err != nil { + t.Fatalf("FinalizeDump: %v", err) + } + if err := VerifyChecksums(root); err != nil { + t.Fatalf("VerifyChecksums: %v", err) + } + checksums, err := os.ReadFile(filepath.Join(root, CHECKSUMSFilename)) //nolint:gosec // test path + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !strings.Contains(string(checksums), " "+ManifestFilename+"\n") { + t.Fatalf("CHECKSUMS does not include manifest: %s", checksums) + } +} + +func TestFinalizeDumpInvalidManifestLeavesNoCommitMarker(t *testing.T) { + t.Parallel() + root := t.TempDir() + manifest := NewPhase0SnapshotManifest(time.Now()) + manifest.Phase = PhasePhase1LivePinned + + if err := FinalizeDump(root, manifest); err == nil { + t.Fatal("FinalizeDump accepted phase1 manifest without live metadata") + } + if _, err := os.Lstat(filepath.Join(root, ManifestFilename)); !os.IsNotExist(err) { + t.Fatalf("MANIFEST exists after failure: %v", err) + } +} + +func TestFinalizeDumpRefusesExistingManifest(t *testing.T) { + t.Parallel() + root := t.TempDir() + manifestPath := filepath.Join(root, ManifestFilename) + existing := []byte("existing manifest") + if err := os.WriteFile(manifestPath, existing, 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + manifest := NewPhase0SnapshotManifest(time.Now()) + manifest.Phase = PhasePhase1LivePinned + manifest.Live = &Live{ReadTS: 42} + if err := FinalizeDump(root, manifest); err == nil { + t.Fatal("FinalizeDump replaced an existing manifest") + } + got, err := os.ReadFile(manifestPath) //nolint:gosec // test path + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(got) != string(existing) { + t.Fatalf("manifest content = %q, want %q", got, existing) + } +} diff --git a/internal/backup/live.go b/internal/backup/live.go index 7f6256aa6..7b5e1aacb 100644 --- a/internal/backup/live.go +++ b/internal/backup/live.go @@ -17,9 +17,16 @@ type Scope struct { Name string } +const ( + adapterDynamoDB = "dynamodb" + adapterS3 = "s3" + adapterRedis = "redis" + adapterSQS = "sqs" +) + func (s Scope) ID() string { - if s.Adapter == "redis" { - return "redis/" + s.Name + if s.Adapter == adapterRedis { + return adapterRedis + "/" + s.Name } return s.Adapter + "/" + s.Name } @@ -46,7 +53,7 @@ func ScopeForKey(key []byte) (Scope, bool, error) { ): return scopeForSQSKey(key) case isRedisBackupKey(key): - return Scope{Adapter: "redis", Name: "db_0"}, true, nil + return Scope{Adapter: adapterRedis, Name: "db_0"}, true, nil default: return Scope{}, false, nil } diff --git a/internal/backup/live_producer.go b/internal/backup/live_producer.go new file mode 100644 index 000000000..94b97d316 --- /dev/null +++ b/internal/backup/live_producer.go @@ -0,0 +1,713 @@ +package backup + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "math/bits" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" +) + +const ( + defaultLiveBackupEndTimeout = 30 * time.Second + defaultLiveBackupBeginTimeout = 30 * time.Second + liveBackupOutputDirMode = 0o755 + liveBackupRenewalDivisor = 3 + liveBackupPercentDivisor = 100 + integerSqrtRadix = 2 +) + +var ( + ErrCompactionDuringDump = errors.New("backup: key shortfall while live pin was active") + ErrLiveBackupOutputExists = errors.New("backup: live output already exists") + ErrLiveBackupRenewal = errors.New("backup: live pin renewal failed") + ErrLiveBackupScope = errors.New("backup: requested live scope is unavailable") +) + +// LiveBackupRPC is the transport-neutral control plane used by RunLiveBackup. +// The command package adapts the generated gRPC client to this narrow surface. +type LiveBackupRPC interface { + BeginBackup(context.Context, *pb.BeginBackupRequest) (*pb.BeginBackupResponse, error) + RenewBackup(context.Context, *pb.RenewBackupRequest) (*pb.RenewBackupResponse, error) + EndBackup(context.Context, *pb.EndBackupRequest) (*pb.EndBackupResponse, error) + ListAdaptersAndScopes(context.Context, *pb.ListAdaptersAndScopesRequest) (*pb.ListAdaptersAndScopesResponse, error) + StreamBackup(context.Context, *pb.StreamBackupRequest, func(*pb.BackupKV) error) error +} + +// LiveBackupOptions controls one pinned live dump. +type LiveBackupOptions struct { + OutputRoot string + Adapters AdapterSet + Scopes []Scope + TTL time.Duration + + IncludeIncompleteUploads bool + IncludeOrphans bool + PreserveSQSVisibility bool + IncludeSQSSideRecords bool + RenameS3Collisions bool + DynamoDBBundleJSONL bool + DynamoDBBundleSizeBytes int64 + + ElastickvVersion string + ClusterID string + BeginTimeout time.Duration + EndTimeout time.Duration + Now func() time.Time + WarnSink func(event string, fields ...any) +} + +// LiveBackupResult reports the consistency point and work completed. +type LiveBackupResult struct { + ReadTS uint64 + Scopes []Scope + Counters DecodeCounters + PinRenewals uint64 +} + +// RunLiveBackup owns the complete Begin -> renew -> stream -> finalize -> End +// lifecycle. The output root is created exclusively and intentionally retained +// on failure without MANIFEST.json for forensic inspection and safe cleanup. +func RunLiveBackup(ctx context.Context, rpc LiveBackupRPC, opts LiveBackupOptions) (LiveBackupResult, error) { + if err := validateLiveBackupOptions(rpc, opts); err != nil { + return LiveBackupResult{}, err + } + if err := createLiveBackupOutputRoot(opts.OutputRoot); err != nil { + return LiveBackupResult{}, err + } + + begin, err := beginLiveBackup(ctx, rpc, opts) + if err != nil { + return LiveBackupResult{}, err + } + + dumpCtx, cancelDump := context.WithCancelCause(ctx) + defer cancelDump(nil) + ttl, err := durationFromMillis(begin.GetTtlMsEffective()) + if err != nil { + endErr := endLiveBackup(rpc, begin.GetPinToken(), opts.EndTimeout) + return LiveBackupResult{}, combineLiveBackupErrors(err, endErr) + } + lease := newLiveBackupLease(rpc, begin, ttl, cancelDump) + lease.start(dumpCtx) + result, dumpErr := dumpPinnedBackup(dumpCtx, rpc, lease, begin, opts) + return finishLiveBackup(ctx, rpc, lease, begin, opts, result, dumpErr) +} + +func finishLiveBackup( + ctx context.Context, + rpc LiveBackupRPC, + lease *liveBackupLease, + begin *pb.BeginBackupResponse, + opts LiveBackupOptions, + result LiveBackupResult, + dumpErr error, +) (LiveBackupResult, error) { + renewErr := lease.stop() + if renewErr != nil { + if opts.WarnSink != nil { + opts.WarnSink("backup_pin_renewal_failed", "error", renewErr) + } + } + dumpErr = combineLiveBackupPhaseErrors(dumpErr, renewErr) + result.PinRenewals = lease.renewalCount() + dumpErr = publishCompletedLiveBackup(ctx, lease, begin, opts, result, dumpErr) + endErr := endLiveBackup(rpc, lease.tokenSnapshot(), opts.EndTimeout) + if err := combineLiveBackupErrors(dumpErr, endErr); err != nil { + return result, err + } + return result, nil +} + +func combineLiveBackupPhaseErrors(primary, secondary error) error { + if primary == nil { + return secondary + } + if secondary == nil { + return primary + } + return errors.Wrap(errors.WithSecondaryError(primary, secondary), "live backup and pin renewal failed") +} + +func publishCompletedLiveBackup( + ctx context.Context, + lease *liveBackupLease, + begin *pb.BeginBackupResponse, + opts LiveBackupOptions, + result LiveBackupResult, + priorErr error, +) error { + if priorErr != nil { + return priorErr + } + if ctx.Err() != nil { + return errors.Wrap(context.Cause(ctx), "live backup canceled before manifest publication") + } + manifest := liveBackupManifest(begin, lease.tokenSnapshot(), scopeSet(result.Scopes), opts) + return FinalizeDump(opts.OutputRoot, manifest) +} + +func beginLiveBackup(ctx context.Context, rpc LiveBackupRPC, opts LiveBackupOptions) (*pb.BeginBackupResponse, error) { + timeout := effectiveTimeout(opts.BeginTimeout, defaultLiveBackupBeginTimeout) + beginCtx, cancel := context.WithTimeout(ctx, timeout) + begin, err := rpc.BeginBackup(beginCtx, &pb.BeginBackupRequest{TtlMs: durationMillis(opts.TTL)}) + cancel() + if err != nil { + return nil, errors.Wrap(err, "begin live backup") + } + if err := validateBeginBackupResponse(begin); err != nil { + return nil, cleanupInvalidBeginResponse(rpc, begin, err) + } + return begin, nil +} + +func cleanupInvalidBeginResponse(rpc LiveBackupRPC, begin *pb.BeginBackupResponse, beginErr error) error { + if begin == nil || len(begin.GetPinToken()) == 0 { + return beginErr + } + cleanupCtx, cancel := context.WithTimeout(context.Background(), defaultLiveBackupEndTimeout) + _, cleanupErr := rpc.EndBackup(cleanupCtx, &pb.EndBackupRequest{PinToken: begin.GetPinToken()}) + cancel() + if cleanupErr == nil { + return beginErr + } + combined := errors.WithSecondaryError(beginErr, errors.Wrap(cleanupErr, "clean up invalid BeginBackup response")) + return errors.Wrap(combined, "invalid BeginBackup response") +} + +func endLiveBackup(rpc LiveBackupRPC, token []byte, timeout time.Duration) error { + endCtx, cancel := context.WithTimeout(context.Background(), effectiveTimeout(timeout, defaultLiveBackupEndTimeout)) + defer cancel() + _, err := rpc.EndBackup(endCtx, &pb.EndBackupRequest{PinToken: token}) + return errors.Wrap(err, "end live backup") +} + +func combineLiveBackupErrors(dumpErr, endErr error) error { + if dumpErr == nil { + return endErr + } + if endErr == nil { + return dumpErr + } + return errors.Wrap(errors.WithSecondaryError(dumpErr, endErr), "live backup and cleanup failed") +} + +func effectiveTimeout(configured, fallback time.Duration) time.Duration { + if configured > 0 { + return configured + } + return fallback +} + +func validateLiveBackupOptions(rpc LiveBackupRPC, opts LiveBackupOptions) error { + switch { + case rpc == nil: + return errors.New("backup: live RPC client is required") + case opts.OutputRoot == "": + return errors.New("backup: live output root is required") + case opts.Adapters == (AdapterSet{}): + return errors.New("backup: at least one adapter is required") + case opts.TTL <= 0: + return errors.New("backup: live TTL must be positive") + case opts.DynamoDBBundleSizeBytes < 0: + return errors.New("backup: DynamoDB bundle size must not be negative") + default: + return nil + } +} + +func createLiveBackupOutputRoot(root string) error { + root = filepath.Clean(root) + parent := filepath.Dir(root) + if parent != "." && parent != root { + if err := os.MkdirAll(parent, liveBackupOutputDirMode); err != nil { + return errors.WithStack(err) + } + } + if err := os.Mkdir(root, liveBackupOutputDirMode); err != nil { + if errors.Is(err, os.ErrExist) { + return errors.Wrapf(ErrLiveBackupOutputExists, "%s", root) + } + return errors.WithStack(err) + } + return nil +} + +func validateBeginBackupResponse(resp *pb.BeginBackupResponse) error { + if resp == nil || resp.GetReadTs() == 0 || len(resp.GetPinToken()) == 0 || resp.GetTtlMsEffective() == 0 { + return errors.New("backup: BeginBackup returned incomplete pin metadata") + } + _, err := durationFromMillis(resp.GetTtlMsEffective()) + return err +} + +func dumpPinnedBackup( + ctx context.Context, + rpc LiveBackupRPC, + lease *liveBackupLease, + begin *pb.BeginBackupResponse, + opts LiveBackupOptions, +) (LiveBackupResult, error) { + listed, err := rpc.ListAdaptersAndScopes(ctx, &pb.ListAdaptersAndScopesRequest{PinToken: lease.tokenSnapshot()}) + if err != nil { + return LiveBackupResult{}, errors.Wrap(err, "list live backup scopes") + } + selected, err := selectLiveBackupScopes(opts.Adapters, opts.Scopes, listed.GetScopes()) + if err != nil { + return LiveBackupResult{}, err + } + decoder, err := NewLiveDecoder(DecodeOptions{ + OutRoot: opts.OutputRoot, + Adapters: opts.Adapters, + IncludeIncompleteUploads: opts.IncludeIncompleteUploads, + IncludeOrphans: opts.IncludeOrphans, + RenameS3Collisions: opts.RenameS3Collisions, + PreserveSQSVisibility: opts.PreserveSQSVisibility, + IncludeSQSSideRecords: opts.IncludeSQSSideRecords, + DynamoDBBundleJSONL: opts.DynamoDBBundleJSONL, + DynamoDBBundleSizeBytes: opts.DynamoDBBundleSizeBytes, + WarnSink: opts.WarnSink, + }) + if err != nil { + return LiveBackupResult{}, err + } + actual := make(map[Scope]uint64, len(selected)) + streamErr := streamSelectedBackup(ctx, rpc, lease, selected, actual, decoder) + counters, finalizeErr := decoder.Finalize() + if streamErr != nil { + return LiveBackupResult{ReadTS: begin.GetReadTs(), Scopes: sortedScopeSet(selected)}, streamErr + } + if finalizeErr != nil { + return LiveBackupResult{ReadTS: begin.GetReadTs(), Scopes: sortedScopeSet(selected)}, finalizeErr + } + if err := validateExpectedLiveCounts(selected, actual, begin.GetExpectedKeys()); err != nil { + return LiveBackupResult{ReadTS: begin.GetReadTs(), Scopes: sortedScopeSet(selected), Counters: counters}, err + } + return LiveBackupResult{ReadTS: begin.GetReadTs(), Scopes: sortedScopeSet(selected), Counters: counters}, nil +} + +func streamSelectedBackup( + ctx context.Context, + rpc LiveBackupRPC, + lease *liveBackupLease, + selected map[Scope]struct{}, + actual map[Scope]uint64, + decoder *LiveDecoder, +) error { + if len(selected) == 0 { + return nil + } + req := &pb.StreamBackupRequest{PinToken: lease.tokenSnapshot(), Scopes: protoScopes(selected)} + err := rpc.StreamBackup(ctx, req, func(pair *pb.BackupKV) error { + if pair == nil { + return errors.New("backup: stream returned a nil record") + } + scope, scoped, err := ScopeForKey(pair.GetKey()) + if err != nil { + return err + } + if !scoped { + return errors.New("backup: stream returned an unscoped internal record") + } + if _, ok := selected[scope]; !ok { + return errors.Wrapf(ErrLiveBackupScope, "stream returned unselected %s", scope) + } + actual[scope]++ + return decoder.Add(pair.GetKey(), pair.GetValue()) + }) + if err != nil { + if cause := context.Cause(ctx); errors.Is(cause, ErrLiveBackupRenewal) { + return errors.WithStack(cause) + } + return errors.Wrap(err, "stream pinned live backup") + } + return nil +} + +func selectLiveBackupScopes(adapters AdapterSet, requested []Scope, listed []*pb.BackupScope) (map[Scope]struct{}, error) { + available, err := availableLiveBackupScopes(adapters, listed) + if err != nil { + return nil, err + } + if len(requested) == 0 { + return available, nil + } + return requestedLiveBackupScopes(adapters, requested, available) +} + +func availableLiveBackupScopes(adapters AdapterSet, listed []*pb.BackupScope) (map[Scope]struct{}, error) { + available := make(map[Scope]struct{}, len(listed)) + for _, item := range listed { + if item == nil { + continue + } + scope := Scope{Adapter: item.GetAdapter(), Name: item.GetScope()} + if scope.Adapter == "" || scope.Name == "" { + return nil, errors.Wrap(ErrLiveBackupScope, "server returned an empty scope") + } + if err := validateLiveScope(scope); err != nil { + return nil, err + } + if adapterEnabled(adapters, scope.Adapter) { + available[scope] = struct{}{} + } + } + return available, nil +} + +func requestedLiveBackupScopes( + adapters AdapterSet, + requested []Scope, + available map[Scope]struct{}, +) (map[Scope]struct{}, error) { + selected := make(map[Scope]struct{}, len(requested)) + for _, scope := range requested { + if !adapterEnabled(adapters, scope.Adapter) { + return nil, errors.Wrapf(ErrLiveBackupScope, "%s is excluded by --adapter", scope) + } + if _, ok := available[scope]; !ok { + return nil, errors.Wrapf(ErrLiveBackupScope, "%s is absent at the pinned timestamp", scope) + } + selected[scope] = struct{}{} + } + return selected, nil +} + +func validateLiveScope(scope Scope) error { + if scope.Adapter != adapterRedis { + return nil + } + db, ok := strings.CutPrefix(scope.Name, "db_") + if !ok { + return errors.Wrapf(ErrLiveBackupScope, "invalid Redis scope %q", scope.Name) + } + if _, err := strconv.ParseUint(db, 10, 32); err != nil { + return errors.Wrapf(ErrLiveBackupScope, "invalid Redis scope %q", scope.Name) + } + return nil +} + +func adapterEnabled(set AdapterSet, name string) bool { + switch name { + case adapterDynamoDB: + return set.DynamoDB + case adapterS3: + return set.S3 + case adapterRedis: + return set.Redis + case adapterSQS: + return set.SQS + default: + return false + } +} + +func protoScopes(scopes map[Scope]struct{}) []*pb.BackupScope { + sorted := sortedScopeSet(scopes) + out := make([]*pb.BackupScope, 0, len(sorted)) + for _, scope := range sorted { + out = append(out, &pb.BackupScope{Adapter: scope.Adapter, Scope: scope.Name}) + } + return out +} + +func sortedScopeSet(scopes map[Scope]struct{}) []Scope { + counts := make(map[Scope]uint64, len(scopes)) + for scope := range scopes { + counts[scope] = 0 + } + return SortedScopes(counts) +} + +func scopeSet(scopes []Scope) map[Scope]struct{} { + out := make(map[Scope]struct{}, len(scopes)) + for _, scope := range scopes { + out[scope] = struct{}{} + } + return out +} + +func validateExpectedLiveCounts(selected map[Scope]struct{}, actual map[Scope]uint64, baseline []*pb.BackupExpectedKeys) error { + expected := make(map[Scope]uint64, len(baseline)) + for _, item := range baseline { + if item == nil { + continue + } + scope := Scope{Adapter: item.GetAdapter(), Name: item.GetScope()} + if _, ok := expected[scope]; ok { + return errors.Wrapf(ErrCompactionDuringDump, "duplicate baseline for %s", scope) + } + expected[scope] = item.GetKeyCount() + } + for scope := range selected { + want, ok := expected[scope] + if !ok { + return errors.Wrapf(ErrCompactionDuringDump, "missing baseline for %s", scope) + } + minimum := minimumAcceptedLiveCount(want) + if actual[scope] < minimum { + return errors.Wrapf(ErrCompactionDuringDump, + "%s: actual=%d baseline=%d minimum=%d", scope, actual[scope], want, minimum) + } + } + return nil +} + +func minimumAcceptedLiveCount(expected uint64) uint64 { + onePercent := expected / liveBackupPercentDivisor + if expected%liveBackupPercentDivisor != 0 { + onePercent++ + } + tolerance := onePercent + integerSqrt(expected) + if tolerance >= expected { + return 0 + } + return expected - tolerance +} + +func integerSqrt(n uint64) uint64 { + if n < integerSqrtRadix { + return n + } + x := uint64(1) << ((bits.Len64(n) + 1) / integerSqrtRadix) + for { + y := (x + n/x) / integerSqrtRadix + if y >= x { + return x + } + x = y + } +} + +func liveBackupManifest(begin *pb.BeginBackupResponse, token []byte, selected map[Scope]struct{}, opts LiveBackupOptions) Manifest { + now := time.Now + if opts.Now != nil { + now = opts.Now + } + m := NewPhase0SnapshotManifest(now()) + m.Phase = PhasePhase1LivePinned + m.Source = nil + m.ElastickvVersion = opts.ElastickvVersion + m.ClusterID = opts.ClusterID + m.LastCommitTS = begin.GetReadTs() + m.Live = &Live{ReadTS: begin.GetReadTs(), PinTokenSHA256: tokenDigest(token)} + m.BackupTSTTLMS = begin.GetTtlMsEffective() + m.MaxActiveBackupPins = begin.GetMaxActiveBackupPins() + m.Shards = make([]LiveShard, 0, len(begin.GetShards())) + for _, shard := range begin.GetShards() { + if shard != nil { + m.Shards = append(m.Shards, LiveShard{RaftGroupID: shard.GetRaftGroupId(), AppliedIndex: shard.GetAppliedIndex()}) + } + } + sort.Slice(m.Shards, func(i, j int) bool { return m.Shards[i].RaftGroupID < m.Shards[j].RaftGroupID }) + m.Adapters = liveManifestAdapters(opts.Adapters, selected) + m.Exclusions = &Exclusions{ + IncludeIncompleteUploads: opts.IncludeIncompleteUploads, + IncludeOrphans: opts.IncludeOrphans, + PreserveSQSVisibility: opts.PreserveSQSVisibility, + IncludeSQSSideRecords: opts.IncludeSQSSideRecords, + RenameS3Collisions: opts.RenameS3Collisions, + } + if opts.DynamoDBBundleJSONL { + m.DynamoDBLayout = DynamoDBLayoutJSONL + } + return m +} + +func liveManifestAdapters(enabled AdapterSet, selected map[Scope]struct{}) *Adapters { + out := &Adapters{} + if enabled.DynamoDB { + out.DynamoDB = &Adapter{} + } + if enabled.S3 { + out.S3 = &Adapter{} + } + if enabled.Redis { + out.Redis = &Adapter{} + } + if enabled.SQS { + out.SQS = &Adapter{} + } + for _, scope := range sortedScopeSet(selected) { + addLiveManifestScope(out, scope) + } + return out +} + +func addLiveManifestScope(out *Adapters, scope Scope) { + switch scope.Adapter { + case adapterDynamoDB: + out.DynamoDB.Tables = append(out.DynamoDB.Tables, scope.Name) + case adapterS3: + out.S3.Buckets = append(out.S3.Buckets, scope.Name) + case adapterRedis: + addLiveManifestRedisScope(out.Redis, scope.Name) + case adapterSQS: + out.SQS.Queues = append(out.SQS.Queues, scope.Name) + } +} + +func addLiveManifestRedisScope(out *Adapter, name string) { + db, ok := strings.CutPrefix(name, "db_") + if !ok { + return + } + index, err := strconv.ParseUint(db, 10, 32) + if err == nil { + out.Databases = append(out.Databases, uint32(index)) + } +} + +func tokenDigest(token []byte) string { + sum := sha256.Sum256(token) + return hex.EncodeToString(sum[:]) +} + +func durationMillis(d time.Duration) uint64 { + return uint64(d / time.Millisecond) //nolint:gosec // validated positive and bounded by time.Duration. +} + +func durationFromMillis(ms uint64) (time.Duration, error) { + if ms == 0 || ms > uint64((time.Duration(1<<63-1))/time.Millisecond) { + return 0, errors.New("backup: effective TTL is outside the supported duration range") + } + return time.Duration(ms) * time.Millisecond, nil //nolint:gosec // upper bound checked above. +} + +type liveBackupLease struct { + rpc LiveBackupRPC + ttl time.Duration + cancel context.CancelCauseFunc + + mu sync.RWMutex + token []byte + renewals uint64 + stopOnce sync.Once + stopCh chan struct{} + doneCh chan error +} + +func newLiveBackupLease( + rpc LiveBackupRPC, + begin *pb.BeginBackupResponse, + ttl time.Duration, + cancel context.CancelCauseFunc, +) *liveBackupLease { + return &liveBackupLease{ + rpc: rpc, ttl: ttl, + token: append([]byte(nil), begin.GetPinToken()...), cancel: cancel, + stopCh: make(chan struct{}), doneCh: make(chan error, 1), + } +} + +func (l *liveBackupLease) start(ctx context.Context) { + go func() { l.doneCh <- l.renewLoop(ctx) }() +} + +func (l *liveBackupLease) stop() error { + l.stopOnce.Do(func() { close(l.stopCh) }) + l.cancel(nil) + return <-l.doneCh +} + +func (l *liveBackupLease) renewLoop(ctx context.Context) error { + interval, err := liveBackupRenewalInterval(l.ttl) + if err != nil { + return l.failRenewal(err) + } + timer := time.NewTimer(interval) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-l.stopCh: + return nil + case <-timer.C: + nextInterval, err := l.renewOnce(ctx) + if err != nil { + if l.stopping(ctx) { + return nil + } + return l.failRenewal(err) + } + timer.Reset(nextInterval) + } + } +} + +func (l *liveBackupLease) renewOnce(ctx context.Context) (time.Duration, error) { + l.mu.RLock() + ttl := l.ttl + l.mu.RUnlock() + resp, err := l.rpc.RenewBackup(ctx, &pb.RenewBackupRequest{ + PinToken: l.tokenSnapshot(), TtlMs: durationMillis(ttl), + }) + if err != nil { + return 0, errors.Wrap(err, "renew live backup pin") + } + if resp == nil || len(resp.GetPinToken()) == 0 || resp.GetTtlMsEffective() == 0 { + return 0, errors.New("renew returned incomplete pin metadata") + } + renewedTTL, err := durationFromMillis(resp.GetTtlMsEffective()) + if err != nil { + return 0, err + } + interval, err := liveBackupRenewalInterval(renewedTTL) + if err != nil { + return 0, err + } + l.mu.Lock() + l.token = append(l.token[:0], resp.GetPinToken()...) + l.ttl = renewedTTL + l.renewals++ + l.mu.Unlock() + return interval, nil +} + +func liveBackupRenewalInterval(ttl time.Duration) (time.Duration, error) { + interval := ttl / liveBackupRenewalDivisor + if interval <= 0 { + return 0, errors.New("renewal interval is not positive") + } + return interval, nil +} + +func (l *liveBackupLease) failRenewal(err error) error { + renewErr := errors.Wrap(ErrLiveBackupRenewal, err.Error()) + l.cancel(renewErr) + return renewErr +} + +func (l *liveBackupLease) stopping(ctx context.Context) bool { + if ctx.Err() != nil { + return true + } + select { + case <-l.stopCh: + return true + default: + return false + } +} + +func (l *liveBackupLease) tokenSnapshot() []byte { + l.mu.RLock() + defer l.mu.RUnlock() + return append([]byte(nil), l.token...) +} + +func (l *liveBackupLease) renewalCount() uint64 { + l.mu.RLock() + defer l.mu.RUnlock() + return l.renewals +} diff --git a/internal/backup/live_producer_test.go b/internal/backup/live_producer_test.go new file mode 100644 index 000000000..7b333c43c --- /dev/null +++ b/internal/backup/live_producer_test.go @@ -0,0 +1,456 @@ +package backup + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/bootjp/elastickv/internal/s3keys" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" +) + +type fakeLiveBackupRPC struct { + mu sync.Mutex + + begin *pb.BeginBackupResponse + listed []*pb.BackupScope + records []*pb.BackupKV + + renewErr error + renewStarted chan struct{} + renewStartOnce sync.Once + renewWait bool + manifestPath string + manifestSeen bool + blockStream bool + streamRelease <-chan struct{} + streamStarted chan struct{} + streamStartOnce sync.Once + renewCalls int + endCalls int + endedWithToken []byte +} + +func (f *fakeLiveBackupRPC) BeginBackup(context.Context, *pb.BeginBackupRequest) (*pb.BeginBackupResponse, error) { + return f.begin, nil +} + +func (f *fakeLiveBackupRPC) RenewBackup(ctx context.Context, _ *pb.RenewBackupRequest) (*pb.RenewBackupResponse, error) { + f.mu.Lock() + f.renewCalls++ + renewErr := f.renewErr + renewWait := f.renewWait + manifestPath := f.manifestPath + f.mu.Unlock() + if f.renewStarted != nil { + f.renewStartOnce.Do(func() { close(f.renewStarted) }) + } + if renewWait { + <-ctx.Done() + _, statErr := os.Lstat(manifestPath) + f.mu.Lock() + f.manifestSeen = statErr == nil + f.mu.Unlock() + return nil, ctx.Err() + } + if renewErr != nil { + return nil, renewErr + } + return &pb.RenewBackupResponse{PinToken: []byte("renewed-token"), TtlMsEffective: f.begin.GetTtlMsEffective()}, nil +} + +func (f *fakeLiveBackupRPC) EndBackup(_ context.Context, req *pb.EndBackupRequest) (*pb.EndBackupResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.endCalls++ + f.endedWithToken = append([]byte(nil), req.GetPinToken()...) + return &pb.EndBackupResponse{}, nil +} + +func (f *fakeLiveBackupRPC) ListAdaptersAndScopes(context.Context, *pb.ListAdaptersAndScopesRequest) (*pb.ListAdaptersAndScopesResponse, error) { + return &pb.ListAdaptersAndScopesResponse{Scopes: f.listed}, nil +} + +func (f *fakeLiveBackupRPC) StreamBackup(ctx context.Context, _ *pb.StreamBackupRequest, consume func(*pb.BackupKV) error) error { + if f.streamStarted != nil { + f.streamStartOnce.Do(func() { close(f.streamStarted) }) + } + if f.streamRelease != nil { + select { + case <-ctx.Done(): + return ctx.Err() + case <-f.streamRelease: + } + } + if f.blockStream { + <-ctx.Done() + return ctx.Err() + } + for _, record := range f.records { + if err := consume(record); err != nil { + return err + } + } + return nil +} + +func successfulLiveBackupRPC() *fakeLiveBackupRPC { + return &fakeLiveBackupRPC{ + begin: &pb.BeginBackupResponse{ + ReadTs: 42, PinToken: []byte("pin-token"), TtlMsEffective: 60_000, MaxActiveBackupPins: 4, + Shards: []*pb.BackupShardApplied{{RaftGroupId: 1, AppliedIndex: 99}}, + ExpectedKeys: []*pb.BackupExpectedKeys{{Adapter: "redis", Scope: "db_0", KeyCount: 1}}, + }, + listed: []*pb.BackupScope{{Adapter: "redis", Scope: "db_0"}}, + records: []*pb.BackupKV{{Key: []byte(RedisStringPrefix + "hello"), Value: []byte("world")}}, + } +} + +func TestRunLiveBackupFinalizesAndReleasesPin(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + root := filepath.Join(t.TempDir(), "dump") + result, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, Adapters: AdapterSet{Redis: true}, TTL: time.Minute, + ElastickvVersion: "test", Now: func() time.Time { return time.Unix(1_700_000_000, 0) }, + }) + if err != nil { + t.Fatalf("RunLiveBackup: %v", err) + } + if result.ReadTS != 42 || len(result.Scopes) != 1 { + t.Fatalf("result=%+v", result) + } + assertLiveBackupManifest(t, root) + assertLiveBackupEnded(t, rpc, "pin-token") +} + +func assertLiveBackupManifest(t *testing.T, root string) { + t.Helper() + manifest := readLiveBackupManifest(t, root) + if manifest.Live == nil || manifest.Live.ReadTS != 42 || manifest.BackupTSTTLMS != 60_000 || manifest.MaxActiveBackupPins != 4 { + t.Fatalf("manifest live metadata=%+v", manifest) + } + if manifest.Adapters.Redis == nil || len(manifest.Adapters.Redis.Databases) != 1 || manifest.Adapters.Redis.Databases[0] != 0 { + t.Fatalf("manifest adapters=%+v", manifest.Adapters) + } +} + +func readLiveBackupManifest(t *testing.T, root string) Manifest { + t.Helper() + if err := VerifyChecksums(root); err != nil { + t.Fatalf("VerifyChecksums: %v", err) + } + manifestFile, err := os.Open(filepath.Join(root, ManifestFilename)) //nolint:gosec // test path + if err != nil { + t.Fatalf("Open manifest: %v", err) + } + manifest, readErr := ReadManifest(manifestFile) + closeErr := manifestFile.Close() + if readErr != nil || closeErr != nil { + t.Fatalf("ReadManifest=%v Close=%v", readErr, closeErr) + } + return manifest +} + +func assertLiveBackupEnded(t *testing.T, rpc *fakeLiveBackupRPC, token string) { + t.Helper() + rpc.mu.Lock() + defer rpc.mu.Unlock() + if rpc.endCalls != 1 || string(rpc.endedWithToken) != token { + t.Fatalf("endCalls=%d token=%q", rpc.endCalls, rpc.endedWithToken) + } +} + +func TestRunLiveBackupCountShortfallLeavesNoManifest(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + rpc.records = nil + rpc.begin.ExpectedKeys[0].KeyCount = 1000 + root := filepath.Join(t.TempDir(), "dump") + _, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, Adapters: AdapterSet{Redis: true}, TTL: time.Minute, + }) + if !errors.Is(err, ErrCompactionDuringDump) { + t.Fatalf("err=%v, want ErrCompactionDuringDump", err) + } + if _, statErr := os.Lstat(filepath.Join(root, ManifestFilename)); !os.IsNotExist(statErr) { + t.Fatalf("manifest exists after count shortfall: %v", statErr) + } + rpc.mu.Lock() + defer rpc.mu.Unlock() + if rpc.endCalls != 1 { + t.Fatalf("endCalls=%d, want 1", rpc.endCalls) + } +} + +func TestRunLiveBackupRenewalFailureCancelsStreamAndReleasesPin(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + rpc.begin.TtlMsEffective = 9 + rpc.renewErr = errors.New("leader election") + rpc.blockStream = true + root := filepath.Join(t.TempDir(), "dump") + warnings := make(chan string, 1) + _, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, Adapters: AdapterSet{Redis: true}, TTL: time.Minute, + WarnSink: func(event string, _ ...any) { + warnings <- event + }, + }) + if !errors.Is(err, ErrLiveBackupRenewal) { + t.Fatalf("err=%v, want ErrLiveBackupRenewal", err) + } + if _, statErr := os.Lstat(filepath.Join(root, ManifestFilename)); !os.IsNotExist(statErr) { + t.Fatalf("manifest exists after renewal failure: %v", statErr) + } + if event := <-warnings; event != "backup_pin_renewal_failed" { + t.Fatalf("warning event = %q", event) + } + rpc.mu.Lock() + defer rpc.mu.Unlock() + if rpc.renewCalls == 0 || rpc.endCalls != 1 { + t.Fatalf("renewCalls=%d endCalls=%d", rpc.renewCalls, rpc.endCalls) + } +} + +func TestRunLiveBackupStopsRenewalBeforeManifestPublication(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + rpc.begin.TtlMsEffective = 9 + rpc.renewStarted = make(chan struct{}) + rpc.renewWait = true + rpc.streamRelease = rpc.renewStarted + root := filepath.Join(t.TempDir(), "dump") + rpc.manifestPath = filepath.Join(root, ManifestFilename) + + _, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, Adapters: AdapterSet{Redis: true}, TTL: time.Minute, + }) + if err != nil { + t.Fatalf("RunLiveBackup: %v", err) + } + rpc.mu.Lock() + manifestSeen := rpc.manifestSeen + rpc.mu.Unlock() + if manifestSeen { + t.Fatal("MANIFEST.json was published while a pin renewal was still active") + } + manifest := readLiveBackupManifest(t, root) + if manifest.Live == nil || manifest.Live.ReadTS != 42 || manifest.BackupTSTTLMS != 9 { + t.Fatalf("manifest live metadata = %+v", manifest) + } +} + +func TestProducerCrashLeavesNoManifest(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + rpc.blockStream = true + rpc.streamStarted = make(chan struct{}) + root := filepath.Join(t.TempDir(), "dump") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + warnings := make(chan string, 1) + go func() { + _, err := RunLiveBackup(ctx, rpc, LiveBackupOptions{ + OutputRoot: root, Adapters: AdapterSet{Redis: true}, TTL: time.Minute, + WarnSink: func(event string, _ ...any) { warnings <- event }, + }) + done <- err + }() + <-rpc.streamStarted + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("RunLiveBackup error = %v, want context.Canceled", err) + } + assertLiveBackupHasNoManifest(t, root) + assertLiveBackupEnded(t, rpc, "pin-token") + select { + case event := <-warnings: + t.Fatalf("graceful cancellation emitted warning %q", event) + default: + } +} + +func TestRunLiveBackupUsesRenewedTokenForCleanup(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + rpc.begin.TtlMsEffective = 30 + releaseStream := make(chan struct{}) + rpc.streamRelease = releaseStream + root := filepath.Join(t.TempDir(), "dump") + done := make(chan error, 1) + go func() { + _, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, Adapters: AdapterSet{Redis: true}, TTL: time.Minute, + }) + done <- err + }() + waitForLiveBackupRenewal(t, rpc) + close(releaseStream) + if err := <-done; err != nil { + t.Fatalf("RunLiveBackup: %v", err) + } + assertLiveBackupEnded(t, rpc, "renewed-token") +} + +func TestRunLiveBackupInvalidBeginResponseReleasesPin(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + rpc.begin.ReadTs = 0 + root := filepath.Join(t.TempDir(), "dump") + _, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, Adapters: AdapterSet{Redis: true}, TTL: time.Minute, + }) + if err == nil { + t.Fatal("RunLiveBackup accepted an incomplete BeginBackup response") + } + assertLiveBackupEnded(t, rpc, "pin-token") + assertLiveBackupHasNoManifest(t, root) +} + +func TestRunLiveBackupUnavailableScopeLeavesNoManifest(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + root := filepath.Join(t.TempDir(), "dump") + _, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, + Adapters: AdapterSet{Redis: true}, + Scopes: []Scope{{Adapter: adapterRedis, Name: "db_1"}}, + TTL: time.Minute, + }) + if !errors.Is(err, ErrLiveBackupScope) { + t.Fatalf("err=%v, want ErrLiveBackupScope", err) + } + assertLiveBackupEnded(t, rpc, "pin-token") + assertLiveBackupHasNoManifest(t, root) +} + +func TestCrossAdapterConsistency(t *testing.T) { + t.Parallel() + rpc, body := crossAdapterLiveBackupRPC(t) + root := filepath.Join(t.TempDir(), "dump") + result, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, + Adapters: AdapterSet{DynamoDB: true, S3: true}, + TTL: time.Minute, + }) + if err != nil { + t.Fatalf("RunLiveBackup: %v", err) + } + assertCrossAdapterLiveBackup(t, root, body, result) +} + +func crossAdapterLiveBackupRPC(t *testing.T) (*fakeLiveBackupRPC, []byte) { + t.Helper() + rpc := successfulLiveBackupRPC() + dynamoSchema := &pb.DynamoTableSchema{ + TableName: "sessions", + PrimaryKey: &pb.DynamoKeySchema{HashKey: "session_id"}, + AttributeDefinitions: map[string]string{"session_id": "S"}, + Generation: 1, + } + dynamoItem := &pb.DynamoItem{Attributes: map[string]*pb.DynamoAttributeValue{ + "session_id": sAttr("sess-42"), + "owner": sAttr("alice"), + }} + const ( + bucket = "photos" + object = "2026/image.jpg" + uploadID = "upload-1" + ) + body := []byte("image-at-read-ts-42") + rpc.listed = []*pb.BackupScope{ + {Adapter: adapterDynamoDB, Scope: "sessions"}, + {Adapter: adapterS3, Scope: bucket}, + } + rpc.begin.ExpectedKeys = []*pb.BackupExpectedKeys{ + {Adapter: adapterDynamoDB, Scope: "sessions", KeyCount: 2}, + {Adapter: adapterS3, Scope: bucket, KeyCount: 3}, + } + rpc.records = []*pb.BackupKV{ + {Key: EncodeDDBTableMetaKey("sessions"), Value: encodeSchemaValue(t, dynamoSchema)}, + {Key: EncodeDDBItemKey("sessions", 1, "sess-42", ""), Value: encodeItemValue(t, dynamoItem)}, + {Key: s3keys.BucketMetaKey(bucket), Value: encodeS3BucketMetaValue(t, map[string]any{ + "bucket_name": bucket, "generation": 1, + })}, + {Key: s3keys.ObjectManifestKey(bucket, 1, object), Value: encodeS3ManifestValue(t, map[string]any{ + "upload_id": uploadID, "size_bytes": len(body), "parts": []map[string]any{ + {"part_no": 1, "size_bytes": len(body), "chunk_count": 1}, + }, + })}, + {Key: s3keys.BlobKey(bucket, 1, object, uploadID, 1, 0), Value: body}, + } + return rpc, body +} + +func assertCrossAdapterLiveBackup(t *testing.T, root string, body []byte, result LiveBackupResult) { + t.Helper() + if result.ReadTS != 42 || len(result.Scopes) != 2 { + t.Fatalf("result = %+v", result) + } + item := readItemMap(t, filepath.Join(root, "dynamodb", "sessions", "items", "sess-42.json")) + if mustSubMap(t, item, "owner")["S"] != "alice" { + t.Fatalf("DynamoDB item = %v", item) + } + gotBody, err := os.ReadFile(filepath.Join(root, "s3", "photos", "2026/image.jpg")) //nolint:gosec // test path + if err != nil { + t.Fatalf("read S3 object: %v", err) + } + if string(gotBody) != string(body) { + t.Fatalf("S3 body = %q, want %q", gotBody, body) + } + manifest := readLiveBackupManifest(t, root) + if manifest.Live == nil || manifest.Live.ReadTS != 42 || manifest.Adapters.DynamoDB == nil || manifest.Adapters.S3 == nil { + t.Fatalf("manifest = %+v", manifest) + } +} + +func TestExpectedKeysBaselineToleratesTTLExpiry(t *testing.T) { + t.Parallel() + scope := Scope{Adapter: adapterRedis, Name: "db_0"} + selected := map[Scope]struct{}{scope: {}} + baseline := []*pb.BackupExpectedKeys{{Adapter: scope.Adapter, Scope: scope.Name, KeyCount: 10_000}} + if err := validateExpectedLiveCounts(selected, map[Scope]uint64{scope: 9_900}, baseline); err != nil { + t.Fatalf("1%% expiry rejected: %v", err) + } + err := validateExpectedLiveCounts(selected, map[Scope]uint64{scope: 9_500}, baseline) + if !errors.Is(err, ErrCompactionDuringDump) { + t.Fatalf("5%% shortfall error = %v, want ErrCompactionDuringDump", err) + } +} + +func waitForLiveBackupRenewal(t *testing.T, rpc *fakeLiveBackupRPC) { + t.Helper() + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + rpc.mu.Lock() + renewed := rpc.renewCalls > 0 + rpc.mu.Unlock() + if renewed { + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("live backup pin was not renewed") +} + +func assertLiveBackupHasNoManifest(t *testing.T, root string) { + t.Helper() + if _, err := os.Lstat(filepath.Join(root, ManifestFilename)); !os.IsNotExist(err) { + t.Fatalf("manifest exists after failed backup: %v", err) + } +} + +func TestMinimumAcceptedLiveCount(t *testing.T) { + t.Parallel() + if got := minimumAcceptedLiveCount(10_000); got != 9_800 { + t.Fatalf("minimum=%d, want 9800", got) + } + if got := integerSqrt(^uint64(0)); got != uint64(^uint32(0)) { + t.Fatalf("sqrt(max uint64)=%d, want %d", got, uint64(^uint32(0))) + } +} diff --git a/internal/backup/manifest.go b/internal/backup/manifest.go index 84409eac7..4842a11a6 100644 --- a/internal/backup/manifest.go +++ b/internal/backup/manifest.go @@ -28,6 +28,9 @@ import ( // constant. const CurrentFormatVersion uint32 = 1 +// ManifestFilename is the final commit marker for a complete logical dump. +const ManifestFilename = "MANIFEST.json" + const ( // PhasePhase0SnapshotDecode marks dumps produced by Phase 0a (offline // snapshot decoder). @@ -88,6 +91,14 @@ type Live struct { PinTokenSHA256 string `json:"pin_token_sha256,omitempty"` } +// LiveShard records the Raft apply point acknowledged while the live pin was +// installed. It is audit metadata; readers still use Live.ReadTS as the +// consistency boundary. +type LiveShard struct { + RaftGroupID uint64 `json:"raft_group_id"` + AppliedIndex uint64 `json:"applied_index"` +} + // Adapters lists which scopes were dumped per adapter. The pointer // values express two distinguishable on-disk states: // @@ -141,15 +152,18 @@ type Exclusions struct { // Manifest is the on-disk MANIFEST.json structure. Field tags match the // spec in docs/design/2026_04_29_proposed_snapshot_logical_decoder.md. type Manifest struct { - FormatVersion uint32 `json:"format_version"` - Phase string `json:"phase"` - ElastickvVersion string `json:"elastickv_version,omitempty"` - ClusterID string `json:"cluster_id,omitempty"` - SnapshotIndex uint64 `json:"snapshot_index,omitempty"` - LastCommitTS uint64 `json:"last_commit_ts,omitempty"` - WallTimeISO string `json:"wall_time_iso"` - Source *Source `json:"source,omitempty"` - Live *Live `json:"live,omitempty"` + FormatVersion uint32 `json:"format_version"` + Phase string `json:"phase"` + ElastickvVersion string `json:"elastickv_version,omitempty"` + ClusterID string `json:"cluster_id,omitempty"` + SnapshotIndex uint64 `json:"snapshot_index,omitempty"` + LastCommitTS uint64 `json:"last_commit_ts,omitempty"` + Shards []LiveShard `json:"shards,omitempty"` + BackupTSTTLMS uint64 `json:"backup_ts_ttl_ms,omitempty"` + MaxActiveBackupPins uint32 `json:"max_active_backup_pins,omitempty"` + WallTimeISO string `json:"wall_time_iso"` + Source *Source `json:"source,omitempty"` + Live *Live `json:"live,omitempty"` // Adapters and Exclusions are pointer types so ReadManifest can // distinguish "section omitted entirely" (a corrupted or // truncated dump that should fail validation) from "section diff --git a/main.go b/main.go index 01966f0ab..0c3e7167f 100644 --- a/main.go +++ b/main.go @@ -2485,6 +2485,7 @@ func adminServerOptions( BeginDeadline: *backupBeginDeadline, SnapshotHeadroomEntries: *backupSnapshotHeadroomEntries, ScanPageSize: *backupScanPageSize, + MaxActivePins: *backupMaxActivePins, }), ) } diff --git a/proto/admin.pb.go b/proto/admin.pb.go index f6da3a84b..73f04c0b1 100644 --- a/proto/admin.pb.go +++ b/proto/admin.pb.go @@ -1290,14 +1290,15 @@ func (x *BeginBackupRequest) GetTtlMs() uint64 { } type BeginBackupResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - ReadTs uint64 `protobuf:"varint,1,opt,name=read_ts,json=readTs,proto3" json:"read_ts,omitempty"` - PinToken []byte `protobuf:"bytes,2,opt,name=pin_token,json=pinToken,proto3" json:"pin_token,omitempty"` - TtlMsEffective uint64 `protobuf:"varint,3,opt,name=ttl_ms_effective,json=ttlMsEffective,proto3" json:"ttl_ms_effective,omitempty"` - Shards []*BackupShardApplied `protobuf:"bytes,4,rep,name=shards,proto3" json:"shards,omitempty"` - ExpectedKeys []*BackupExpectedKeys `protobuf:"bytes,5,rep,name=expected_keys,json=expectedKeys,proto3" json:"expected_keys,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + ReadTs uint64 `protobuf:"varint,1,opt,name=read_ts,json=readTs,proto3" json:"read_ts,omitempty"` + PinToken []byte `protobuf:"bytes,2,opt,name=pin_token,json=pinToken,proto3" json:"pin_token,omitempty"` + TtlMsEffective uint64 `protobuf:"varint,3,opt,name=ttl_ms_effective,json=ttlMsEffective,proto3" json:"ttl_ms_effective,omitempty"` + Shards []*BackupShardApplied `protobuf:"bytes,4,rep,name=shards,proto3" json:"shards,omitempty"` + ExpectedKeys []*BackupExpectedKeys `protobuf:"bytes,5,rep,name=expected_keys,json=expectedKeys,proto3" json:"expected_keys,omitempty"` + MaxActiveBackupPins uint32 `protobuf:"varint,6,opt,name=max_active_backup_pins,json=maxActiveBackupPins,proto3" json:"max_active_backup_pins,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *BeginBackupResponse) Reset() { @@ -1365,6 +1366,13 @@ func (x *BeginBackupResponse) GetExpectedKeys() []*BackupExpectedKeys { return nil } +func (x *BeginBackupResponse) GetMaxActiveBackupPins() uint32 { + if x != nil { + return x.MaxActiveBackupPins + } + return 0 +} + type RenewBackupRequest struct { state protoimpl.MessageState `protogen:"open.v1"` PinToken []byte `protobuf:"bytes,1,opt,name=pin_token,json=pinToken,proto3" json:"pin_token,omitempty"` @@ -2230,13 +2238,14 @@ const file_admin_proto_rawDesc = "" + "\tkey_count\x18\x03 \x01(\x04R\bkeyCount\x123\n" + "\x16applied_index_at_count\x18\x04 \x01(\x04R\x13appliedIndexAtCount\"+\n" + "\x12BeginBackupRequest\x12\x15\n" + - "\x06ttl_ms\x18\x01 \x01(\x04R\x05ttlMs\"\xdc\x01\n" + + "\x06ttl_ms\x18\x01 \x01(\x04R\x05ttlMs\"\x91\x02\n" + "\x13BeginBackupResponse\x12\x17\n" + "\aread_ts\x18\x01 \x01(\x04R\x06readTs\x12\x1b\n" + "\tpin_token\x18\x02 \x01(\fR\bpinToken\x12(\n" + "\x10ttl_ms_effective\x18\x03 \x01(\x04R\x0ettlMsEffective\x12+\n" + "\x06shards\x18\x04 \x03(\v2\x13.BackupShardAppliedR\x06shards\x128\n" + - "\rexpected_keys\x18\x05 \x03(\v2\x13.BackupExpectedKeysR\fexpectedKeys\"H\n" + + "\rexpected_keys\x18\x05 \x03(\v2\x13.BackupExpectedKeysR\fexpectedKeys\x123\n" + + "\x16max_active_backup_pins\x18\x06 \x01(\rR\x13maxActiveBackupPins\"H\n" + "\x12RenewBackupRequest\x12\x1b\n" + "\tpin_token\x18\x01 \x01(\fR\bpinToken\x12\x15\n" + "\x06ttl_ms\x18\x02 \x01(\x04R\x05ttlMs\"\\\n" + diff --git a/proto/admin.proto b/proto/admin.proto index bcd8b55a7..d8fa1eee9 100644 --- a/proto/admin.proto +++ b/proto/admin.proto @@ -186,6 +186,7 @@ message BeginBackupResponse { uint64 ttl_ms_effective = 3; repeated BackupShardApplied shards = 4; repeated BackupExpectedKeys expected_keys = 5; + uint32 max_active_backup_pins = 6; } message RenewBackupRequest { From 3a86fa50fa4c6576eb03c2683a1d28fcd845d1ab Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 21:23:28 +0900 Subject: [PATCH 2/6] backup: keep live dumps restore-compatible --- cmd/elastickv-backup/main.go | 96 ++++++++++++------- cmd/elastickv-backup/main_test.go | 39 +++++++- .../2026_04_29_implemented_logical_backup.md | 66 ++++++------- docs/operations/backup_restore.md | 12 ++- internal/backup/live.go | 37 +------ internal/backup/live_producer.go | 42 ++++++-- internal/backup/live_producer_test.go | 34 +++++++ internal/backup/live_test.go | 5 +- 8 files changed, 217 insertions(+), 114 deletions(-) diff --git a/cmd/elastickv-backup/main.go b/cmd/elastickv-backup/main.go index 191559e9c..5f12f9d8e 100644 --- a/cmd/elastickv-backup/main.go +++ b/cmd/elastickv-backup/main.go @@ -96,24 +96,9 @@ func run(argv []string, stdout, stderr io.Writer) (retErr error) { logger := slog.New(slog.NewTextHandler(stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer cancel() - result, err := backup.RunLiveBackup(ctx, &grpcLiveBackupRPC{client: pb.NewAdminClient(conn), token: token}, backup.LiveBackupOptions{ - OutputRoot: cfg.outputRoot, - Adapters: cfg.adapters, - Scopes: cfg.scopes, - TTL: cfg.ttl, - IncludeIncompleteUploads: cfg.includeIncompleteUploads, - IncludeOrphans: cfg.includeOrphans, - PreserveSQSVisibility: cfg.preserveSQSVisibility, - IncludeSQSSideRecords: cfg.includeSQSSideRecords, - RenameS3Collisions: cfg.renameCollisions, - DynamoDBBundleJSONL: cfg.bundleJSONL, - DynamoDBBundleSizeBytes: cfg.bundleSizeBytes, - ElastickvVersion: version, - ClusterID: cfg.clusterID, - BeginTimeout: cfg.beginTimeout, - EndTimeout: cfg.endTimeout, - WarnSink: warningSink(logger), - }) + opts := liveBackupOptions(cfg) + opts.WarnSink = warningSink(logger) + result, err := backup.RunLiveBackup(ctx, &grpcLiveBackupRPC{client: pb.NewAdminClient(conn), token: token}, opts) if err != nil { return errors.Wrap(err, "create live backup") } @@ -215,9 +200,32 @@ func buildDumpConfig(values *dumpFlagValues) (*config, error) { if err := populateDynamoDBBundleConfig(&cfg, values); err != nil { return nil, err } + if err := backup.ValidateLiveBackupRestoreCompatibility(liveBackupOptions(&cfg)); err != nil { + return nil, errors.Wrap(err, "validate native restore compatibility") + } return &cfg, nil } +func liveBackupOptions(cfg *config) backup.LiveBackupOptions { + return backup.LiveBackupOptions{ + OutputRoot: cfg.outputRoot, + Adapters: cfg.adapters, + Scopes: cfg.scopes, + TTL: cfg.ttl, + IncludeIncompleteUploads: cfg.includeIncompleteUploads, + IncludeOrphans: cfg.includeOrphans, + PreserveSQSVisibility: cfg.preserveSQSVisibility, + IncludeSQSSideRecords: cfg.includeSQSSideRecords, + RenameS3Collisions: cfg.renameCollisions, + DynamoDBBundleJSONL: cfg.bundleJSONL, + DynamoDBBundleSizeBytes: cfg.bundleSizeBytes, + ElastickvVersion: version, + ClusterID: cfg.clusterID, + BeginTimeout: cfg.beginTimeout, + EndTimeout: cfg.endTimeout, + } +} + func validateDumpFlagValues(values *dumpFlagValues) error { if values.cfg.address == "" || values.cfg.outputRoot == "" { return errors.New("--address and --output-dir are required") @@ -415,27 +423,51 @@ func loadAdminToken(path string) (string, error) { } func loadTransportCredentials(caFile, serverName string, skipVerify bool) (credentials.TransportCredentials, error) { - if caFile == "" && !skipVerify { + tlsRequested := caFile != "" || skipVerify + if err := validateTLSFlagCombination(tlsRequested, caFile, serverName, skipVerify); err != nil { + return nil, err + } + if !tlsRequested { return insecure.NewCredentials(), nil } conf := &tls.Config{MinVersion: tls.VersionTLS12, ServerName: serverName, InsecureSkipVerify: skipVerify} //nolint:gosec // explicit development flag. - if caFile != "" { - pem, err := os.ReadFile(caFile) //nolint:gosec // operator-selected CA file - if err != nil { - return nil, errors.WithStack(err) - } - pool, err := x509.SystemCertPool() - if err != nil || pool == nil { - pool = x509.NewCertPool() - } - if !pool.AppendCertsFromPEM(pem) { - return nil, errors.New("TLS CA file contains no certificates") - } - conf.RootCAs = pool + if caFile == "" { + return credentials.NewTLS(conf), nil + } + pool, err := loadTLSCAPool(caFile) + if err != nil { + return nil, err } + conf.RootCAs = pool return credentials.NewTLS(conf), nil } +func validateTLSFlagCombination(tlsRequested bool, caFile, serverName string, skipVerify bool) error { + switch { + case !tlsRequested && serverName != "": + return errors.New("--tls-server-name requires --tls-ca-cert-file or --tls-insecure-skip-verify") + case caFile != "" && skipVerify: + return errors.New("--tls-ca-cert-file and --tls-insecure-skip-verify are mutually exclusive") + default: + return nil + } +} + +func loadTLSCAPool(caFile string) (*x509.CertPool, error) { + pem, err := os.ReadFile(caFile) //nolint:gosec // operator-selected CA file + if err != nil { + return nil, errors.WithStack(err) + } + pool, err := x509.SystemCertPool() + if err != nil || pool == nil { + pool = x509.NewCertPool() + } + if !pool.AppendCertsFromPEM(pem) { + return nil, errors.New("TLS CA file contains no certificates") + } + return pool, nil +} + func warningSink(logger *slog.Logger) func(string, ...any) { return func(event string, fields ...any) { args := append([]any{"event", event}, fields...) diff --git a/cmd/elastickv-backup/main_test.go b/cmd/elastickv-backup/main_test.go index f2cda3d96..d0294318d 100644 --- a/cmd/elastickv-backup/main_test.go +++ b/cmd/elastickv-backup/main_test.go @@ -1,6 +1,10 @@ package main import ( + "errors" + "os" + "path/filepath" + "strings" "testing" "github.com/bootjp/elastickv/internal/backup" @@ -16,13 +20,12 @@ func TestParseFlagsScopesBundleAndArchive(t *testing.T) { "--adapter", "dynamodb,s3", "--scope", "dynamodb=users,orders", "--scope", "s3=photos", - "--dynamodb-bundle-mode", "jsonl", "--dynamodb-bundle-size", "2MiB", }) if err != nil { t.Fatalf("parseFlags: %v", err) } - if cfg.format != "tar+zstd" || !cfg.bundleJSONL || cfg.bundleSizeBytes != 2<<20 { + if cfg.format != "tar+zstd" || cfg.bundleJSONL || cfg.bundleSizeBytes != 2<<20 { t.Fatalf("cfg=%+v", cfg) } want := []backup.Scope{ @@ -40,6 +43,24 @@ func TestParseFlagsScopesBundleAndArchive(t *testing.T) { } } +func TestParseFlagsRejectsModesUnsupportedByNativeRestore(t *testing.T) { + t.Parallel() + cases := [][]string{ + {"--adapter", "dynamodb", "--dynamodb-bundle-mode", "jsonl"}, + {"--adapter", "s3", "--include-incomplete-uploads"}, + {"--adapter", "s3", "--include-orphans"}, + {"--adapter", "sqs", "--preserve-sqs-visibility"}, + {"--adapter", "sqs", "--include-sqs-side-records"}, + } + for _, extra := range cases { + argv := []string{"dump", "--address", "127.0.0.1:50051", "--output-dir", "/tmp/dump"} + _, err := parseFlags(append(argv, extra...)) + if !errors.Is(err, backup.ErrLiveBackupRestoreUnsupported) { + t.Fatalf("flags=%v error=%v, want ErrLiveBackupRestoreUnsupported", extra, err) + } + } +} + func TestParseFlagsRejectsScopeExcludedByAdapter(t *testing.T) { t.Parallel() _, err := parseFlags([]string{ @@ -62,3 +83,17 @@ func TestParseByteSize(t *testing.T) { } } } + +func TestLoadTransportCredentialsRejectsIgnoredTLSFlags(t *testing.T) { + t.Parallel() + if _, err := loadTransportCredentials("", "backup.internal", false); err == nil || !strings.Contains(err.Error(), "requires") { + t.Fatalf("server name without TLS error = %v", err) + } + caPath := filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(caPath, []byte("not used"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := loadTransportCredentials(caPath, "", true); err == nil || !strings.Contains(err.Error(), "mutually exclusive") { + t.Fatalf("CA plus skip-verify error = %v", err) + } +} diff --git a/docs/design/2026_04_29_implemented_logical_backup.md b/docs/design/2026_04_29_implemented_logical_backup.md index 5fd5654e0..7b83fa1aa 100644 --- a/docs/design/2026_04_29_implemented_logical_backup.md +++ b/docs/design/2026_04_29_implemented_logical_backup.md @@ -361,11 +361,9 @@ item table emits 50 million inodes. On most modern filesystems (ext4/xfs/zfs/apfs) this is fine for write-once-and-tar dumps but slow for live filesystem operations. -For tables where the inode count is the binding constraint, the -producer accepts an opt-in `--dynamodb-bundle-mode jsonl` (paired -with `--dynamodb-bundle-size 64MiB`, defaulting to that value) that -emits items as `items/data-.jsonl` instead, packed up to the -configurable per-file size budget: +The logical decoder implements an opt-in JSONL layout (paired with a +configurable part-size budget) that emits items as +`items/data-.jsonl`: ```text dynamodb/orders/ @@ -377,17 +375,15 @@ dynamodb/orders/ ``` `MANIFEST.json` records the choice (`dynamodb_layout: "per-item" | -"jsonl"`) so a restore tool dispatches the right reader. The bundle -mode is an explicit operational decision; the default stays per-file -because that is the layout the user asked for and the layout that -makes one-line recovery scripts trivial. Operators are also free to -post-process a per-item dump into bundles (`find … | xargs cat`) -without losing any information, so the choice is not load-bearing on -the format itself. - -The producer emits a structured warning when an unbundled scope -exceeds 1 million items so operators can decide whether to switch -modes for that table. +"jsonl"`) so a restore tool can dispatch the right reader. The live +producer currently rejects `--dynamodb-bundle-mode jsonl` when DynamoDB is +selected because the documented native snapshot encoder cannot reverse that +layout yet. Shipping the producer mode and reverse encoder together is an +intentional future extension; completed live backups therefore remain +restorable by the current native path. + +The producer emits a structured warning when an unbundled scope exceeds one +million items so operators can provision inode capacity explicitly. GSIs are **not materialized** under the dump because they are derivable from `_schema.json` plus the base item set. Re-creating the table from @@ -445,9 +441,10 @@ Multipart parts are **flattened on dump**: the user gets the assembled object, not the per-part fragments. Tools like `aws s3 cp --recursive` work on the dump directory tree directly. In-flight multipart uploads (`!s3|upload|meta|`, `!s3|upload|part|`, blob chunks not yet committed by -`CompleteMultipartUpload`) are excluded by default; an -`--include-incomplete-uploads` dump flag emits them under -`s3//_incomplete_uploads//`. +`CompleteMultipartUpload`) are excluded. The decoder has an +`--include-incomplete-uploads` representation under +`s3//_incomplete_uploads//`, but the live producer rejects +that mode until the native snapshot encoder can reverse it. `_bucket.json`: @@ -466,8 +463,9 @@ Generation handling: only the live (latest) generation per bucket is included. Pre-generation orphans (objects under `!s3|blob|||...`) are deliberately omitted — they exist on disk only because GC has not run yet, and replaying them into a fresh -elastickv would resurrect deleted state. They land under -`_orphans//` only when `--include-orphans` is passed. +elastickv would resurrect deleted state. The decoder can place them under +`_orphans//`, but the live producer rejects `--include-orphans` until +the native snapshot encoder supports that subtree. ### Redis @@ -599,10 +597,10 @@ The schema is the dump-time projection of `sqsMessageRecord` (`adapter/sqs_messages.go:80`) with **all visibility-state fields present but zeroed**. A restored queue starts with every message visible — which matches what AWS SQS does when a queue is rehydrated from a backup. If the -operator explicitly requests `--preserve-visibility`, the live -`visible_at_millis` / `current_receipt_token` are kept, but this is opt-in -because resuming with a stale receipt token mid-flight is almost never -the right behavior. +decoder can preserve live `visible_at_millis` / `current_receipt_token`, but +the live producer rejects that opt-in until the native snapshot encoder can +round-trip those fields. Resuming with a stale receipt token mid-flight is +also rarely the desired restore behavior. JSONL was chosen over per-message files for two reasons: (1) production queues commonly hold tens of thousands of messages, and one file per @@ -614,8 +612,9 @@ In-flight side records (`!sqs|msg|dedup|`, `!sqs|msg|group|`, `!sqs|msg|byage|`, `!sqs|msg|vis|`, `!sqs|queue|tombstone|`) are derivable from the queue config + message records and are not dumped. The dedup window will reset on restore; this is documented as expected behavior. -Operators who need exactness pass `--include-sqs-side-records`, which -emits an `_internals/` subdirectory of newline-delimited records. +The decoder can emit an `_internals/` subdirectory of newline-delimited side +records, but the live producer rejects `--include-sqs-side-records` until the +native snapshot encoder can reverse them. ## MANIFEST.json @@ -1133,15 +1132,11 @@ elastickv-backup dump \ [--adapter dynamodb,s3,redis,sqs] \ [--scope dynamodb=orders,users] \ [--scope s3=photos] \ - [--include-incomplete-uploads] \ - [--include-orphans] \ - [--preserve-sqs-visibility] \ - [--include-sqs-side-records] \ [--checksums sha256] \ [--ttl-ms 1800000] \ [--begin-backup-deadline 30s] \ [--end-backup-deadline 30s] \ - [--dynamodb-bundle-mode per-item|jsonl] \ + [--dynamodb-bundle-mode per-item] \ [--dynamodb-bundle-size 64MiB] \ [--rename-collisions] ``` @@ -1257,10 +1252,9 @@ parser, the format has failed its goal. restore (the body hash drives dedup; replaying the dump produces the same hashes). For ID-based dedup, callers replaying messages from the dump *and* from a still-live source concurrently can produce - duplicates. Operators who depend on exact dedup state across a - restore use `--include-sqs-side-records` to opt in to the - `_internals/dedup.jsonl` artifact, then replay it through a - follow-up tool that re-seeds the dedup keys. + duplicates. Exact side-record backup and replay remains a future extension; + the live producer rejects `--include-sqs-side-records` until the native + reverse encoder and a supported replay path can preserve it. - **Redis TTL keys may already be expired by the time of restore.** TTLs are dumped as absolute Unix-millis (`expire_at_ms`). The default restore behavior is **skip-expired**: keys whose diff --git a/docs/operations/backup_restore.md b/docs/operations/backup_restore.md index 874294f47..57325ab28 100644 --- a/docs/operations/backup_restore.md +++ b/docs/operations/backup_restore.md @@ -68,8 +68,10 @@ elastickv-backup dump \ --scope s3=photos ``` -For large DynamoDB tables, `--dynamodb-bundle-mode jsonl` avoids one inode per -item. `--dynamodb-bundle-size` defaults to `64MiB` in that mode. +The producer currently requires DynamoDB per-item output. JSONL output remains +disabled until the native snapshot encoder can reverse that layout. The S3 +incomplete-upload/orphan and SQS visibility/side-record opt-ins are likewise +rejected while the documented native restore path cannot preserve them. ## Stream An Archive @@ -133,9 +135,9 @@ df -i /backups du -sh /backups/elastickv-2026-07-18 ``` -Per-item DynamoDB output can consume one inode per item. Switch future runs to -`--dynamodb-bundle-mode jsonl` when inode pressure, rather than bytes, is the -limiting resource. +Per-item DynamoDB output can consume one inode per item. Capacity planning must +include that inode cost until JSONL output and its native reverse encoder ship +together. ## Failure Handling diff --git a/internal/backup/live.go b/internal/backup/live.go index 7b5e1aacb..40626376f 100644 --- a/internal/backup/live.go +++ b/internal/backup/live.go @@ -81,15 +81,13 @@ func scopeForS3Key(key []byte) (Scope, bool, error) { case bytes.HasPrefix(key, []byte(S3BucketMetaPrefix)): bucket, ok := s3keys.ParseBucketMetaKey(key) return parsedS3Scope(bucket, ok, key) + case bytes.HasPrefix(key, []byte(S3BucketGenPrefix)): + return Scope{}, false, nil case bytes.HasPrefix(key, []byte(S3ObjectManifestPrefix)): bucket, _, _, ok := s3keys.ParseObjectManifestKey(key) return parsedS3Scope(bucket, ok, key) - case bytes.HasPrefix(key, []byte(S3UploadMetaPrefix)): - bucket, ok := parseUploadFamily(S3UploadMetaPrefix, key) - return parsedS3Scope(bucket, ok, key) - case bytes.HasPrefix(key, []byte(S3UploadPartPrefix)): - bucket, _, _, _, _, ok := s3keys.ParseUploadPartKey(key) - return parsedS3Scope(bucket, ok, key) + case bytes.HasPrefix(key, []byte(S3UploadMetaPrefix)), bytes.HasPrefix(key, []byte(S3UploadPartPrefix)): + return Scope{}, false, nil case bytes.HasPrefix(key, []byte(S3BlobPrefix)): bucket, _, _, _, _, _, _, ok := s3keys.ParseBlobKey(key) return parsedS3Scope(bucket, ok, key) @@ -112,26 +110,9 @@ func scopeForSQSKey(key []byte) (Scope, bool, error) { return Scope{}, false, err } return decodedScope("sqs", encoded) - case bytes.HasPrefix(key, []byte(SQSQueueSeqPrefix)): - return sqsScopeFromDirectSegment(key, SQSQueueSeqPrefix) default: - prefix, ok := sqsDerivedBackupPrefix(key) - if !ok { - return Scope{}, false, nil - } - return sqsScopeFromGenericKey(key, prefix) - } -} - -func sqsDerivedBackupPrefix(key []byte) (string, bool) { - for _, prefix := range [...]string{ - SQSQueueTombstonePrefix, SQSMsgVisPrefix, SQSMsgByAgePrefix, SQSMsgDedupPrefix, SQSMsgGroupPrefix, - } { - if bytes.HasPrefix(key, []byte(prefix)) { - return prefix, true - } + return Scope{}, false, nil } - return "", false } func hasAnyBackupPrefix(key []byte, prefixes ...string) bool { @@ -159,14 +140,6 @@ func sqsScopeFromDirectSegment(key []byte, prefix string) (Scope, bool, error) { return decodedScope("sqs", encoded) } -func sqsScopeFromGenericKey(key []byte, prefix string) (Scope, bool, error) { - encoded, err := parseSQSGenericKey(key, prefix) - if err != nil { - return Scope{}, false, err - } - return decodedScope("sqs", encoded) -} - func decodedScope(adapter, encoded string) (Scope, bool, error) { name, err := base64.RawURLEncoding.DecodeString(encoded) if err != nil || len(name) == 0 { diff --git a/internal/backup/live_producer.go b/internal/backup/live_producer.go index 94b97d316..0c42af2bf 100644 --- a/internal/backup/live_producer.go +++ b/internal/backup/live_producer.go @@ -27,10 +27,11 @@ const ( ) var ( - ErrCompactionDuringDump = errors.New("backup: key shortfall while live pin was active") - ErrLiveBackupOutputExists = errors.New("backup: live output already exists") - ErrLiveBackupRenewal = errors.New("backup: live pin renewal failed") - ErrLiveBackupScope = errors.New("backup: requested live scope is unavailable") + ErrCompactionDuringDump = errors.New("backup: key shortfall while live pin was active") + ErrLiveBackupOutputExists = errors.New("backup: live output already exists") + ErrLiveBackupRenewal = errors.New("backup: live pin renewal failed") + ErrLiveBackupRestoreUnsupported = errors.New("backup: live producer mode is unsupported by native restore") + ErrLiveBackupScope = errors.New("backup: requested live scope is unavailable") ) // LiveBackupRPC is the transport-neutral control plane used by RunLiveBackup. @@ -221,8 +222,31 @@ func validateLiveBackupOptions(rpc LiveBackupRPC, opts LiveBackupOptions) error case opts.DynamoDBBundleSizeBytes < 0: return errors.New("backup: DynamoDB bundle size must not be negative") default: - return nil + return ValidateLiveBackupRestoreCompatibility(opts) + } +} + +// ValidateLiveBackupRestoreCompatibility rejects producer modes that the +// documented native snapshot restore path cannot faithfully reverse yet. +// Keeping this check public lets the CLI fail before dialing the cluster while +// RunLiveBackup repeats it for in-process callers. +func ValidateLiveBackupRestoreCompatibility(opts LiveBackupOptions) error { + checks := []struct { + unsupported bool + flag string + }{ + {opts.DynamoDBBundleJSONL && opts.Adapters.DynamoDB, "--dynamodb-bundle-mode=jsonl"}, + {opts.IncludeIncompleteUploads && opts.Adapters.S3, "--include-incomplete-uploads"}, + {opts.IncludeOrphans && opts.Adapters.S3, "--include-orphans"}, + {opts.PreserveSQSVisibility && opts.Adapters.SQS, "--preserve-sqs-visibility"}, + {opts.IncludeSQSSideRecords && opts.Adapters.SQS, "--include-sqs-side-records"}, + } + for _, check := range checks { + if check.unsupported { + return errors.Wrap(ErrLiveBackupRestoreUnsupported, check.flag) + } } + return nil } func createLiveBackupOutputRoot(root string) error { @@ -321,8 +345,11 @@ func streamSelectedBackup( if _, ok := selected[scope]; !ok { return errors.Wrapf(ErrLiveBackupScope, "stream returned unselected %s", scope) } + if err := decoder.Add(pair.GetKey(), pair.GetValue()); err != nil { + return err + } actual[scope]++ - return decoder.Add(pair.GetKey(), pair.GetValue()) + return nil }) if err != nil { if cause := context.Cause(ctx); errors.Is(cause, ErrLiveBackupRenewal) { @@ -469,6 +496,9 @@ func minimumAcceptedLiveCount(expected uint64) uint64 { } tolerance := onePercent + integerSqrt(expected) if tolerance >= expected { + if expected > 0 { + return 1 + } return 0 } return expected - tolerance diff --git a/internal/backup/live_producer_test.go b/internal/backup/live_producer_test.go index 7b333c43c..d224cc8b8 100644 --- a/internal/backup/live_producer_test.go +++ b/internal/backup/live_producer_test.go @@ -423,6 +423,40 @@ func TestExpectedKeysBaselineToleratesTTLExpiry(t *testing.T) { } } +func TestExpectedKeysBaselineRequiresRecordForSmallNonEmptyScope(t *testing.T) { + t.Parallel() + scope := Scope{Adapter: adapterRedis, Name: "db_0"} + selected := map[Scope]struct{}{scope: {}} + for _, expected := range []uint64{1, 2} { + baseline := []*pb.BackupExpectedKeys{{Adapter: scope.Adapter, Scope: scope.Name, KeyCount: expected}} + err := validateExpectedLiveCounts(selected, map[Scope]uint64{scope: 0}, baseline) + if !errors.Is(err, ErrCompactionDuringDump) { + t.Fatalf("baseline=%d error=%v, want ErrCompactionDuringDump", expected, err) + } + } +} + +func TestValidateLiveBackupRestoreCompatibility(t *testing.T) { + t.Parallel() + cases := []LiveBackupOptions{ + {Adapters: AdapterSet{DynamoDB: true}, DynamoDBBundleJSONL: true}, + {Adapters: AdapterSet{S3: true}, IncludeIncompleteUploads: true}, + {Adapters: AdapterSet{S3: true}, IncludeOrphans: true}, + {Adapters: AdapterSet{SQS: true}, PreserveSQSVisibility: true}, + {Adapters: AdapterSet{SQS: true}, IncludeSQSSideRecords: true}, + } + for _, opts := range cases { + if err := ValidateLiveBackupRestoreCompatibility(opts); !errors.Is(err, ErrLiveBackupRestoreUnsupported) { + t.Fatalf("opts=%+v error=%v, want ErrLiveBackupRestoreUnsupported", opts, err) + } + } + if err := ValidateLiveBackupRestoreCompatibility(LiveBackupOptions{ + Adapters: AdapterSet{Redis: true}, DynamoDBBundleJSONL: true, IncludeOrphans: true, + }); err != nil { + t.Fatalf("adapter-scoped options rejected: %v", err) + } +} + func waitForLiveBackupRenewal(t *testing.T, rpc *fakeLiveBackupRPC) { t.Helper() deadline := time.Now().Add(time.Second) diff --git a/internal/backup/live_test.go b/internal/backup/live_test.go index ace0d9933..9800af608 100644 --- a/internal/backup/live_test.go +++ b/internal/backup/live_test.go @@ -21,7 +21,10 @@ func TestScopeForKey(t *testing.T) { {name: "dynamodb meta", key: []byte(DDBTableMetaPrefix + enc("orders")), want: Scope{Adapter: "dynamodb", Name: "orders"}, scoped: true}, {name: "s3 bucket", key: s3keys.BucketMetaKey("photos"), want: Scope{Adapter: "s3", Name: "photos"}, scoped: true}, {name: "sqs metadata", key: []byte(SQSQueueMetaPrefix + enc("jobs")), want: Scope{Adapter: "sqs", Name: "jobs"}, scoped: true}, - {name: "sqs sequence", key: []byte(SQSQueueSeqPrefix + enc("jobs")), want: Scope{Adapter: "sqs", Name: "jobs"}, scoped: true}, + {name: "sqs sequence", key: []byte(SQSQueueSeqPrefix + enc("jobs")), scoped: false}, + {name: "sqs visibility index", key: []byte(SQSMsgVisPrefix + enc("jobs") + "|message"), scoped: false}, + {name: "s3 generation", key: []byte(S3BucketGenPrefix + enc("photos")), scoped: false}, + {name: "s3 incomplete upload", key: []byte(S3UploadMetaPrefix + "ignored"), scoped: false}, {name: "redis", key: []byte(RedisStringPrefix + "key"), want: Scope{Adapter: "redis", Name: "db_0"}, scoped: true}, {name: "ddb derived gsi", key: []byte(DDBGSIPrefix + "ignored"), scoped: false}, {name: "dynamodb generation counter", key: []byte(DDBTableGenPrefix + enc("deleted")), scoped: false}, From e4af4dd52bf5a019b3d0d658ed9369c7e412d2f5 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 21:54:59 +0900 Subject: [PATCH 3/6] backup: fail closed on incomplete live dumps --- internal/backup/live.go | 24 +++++++++ internal/backup/live_producer.go | 75 +++++++++++++++++++++++---- internal/backup/live_producer_test.go | 65 ++++++++++++++++++++++- internal/backup/live_test.go | 1 + internal/backup/s3.go | 22 ++++++++ 5 files changed, 175 insertions(+), 12 deletions(-) diff --git a/internal/backup/live.go b/internal/backup/live.go index 40626376f..ed872f0ba 100644 --- a/internal/backup/live.go +++ b/internal/backup/live.go @@ -63,6 +63,8 @@ func scopeForDDBKey(key []byte) (Scope, bool, error) { switch { case bytes.HasPrefix(key, []byte(DDBTableMetaPrefix)): return ddbScopeFromDirectSegment(key, DDBTableMetaPrefix) + case bytes.HasPrefix(key, []byte(DDBTableGenPrefix)): + return Scope{}, false, nil case bytes.HasPrefix(key, []byte(DDBItemPrefix)): encoded, _, err := parseDDBItemKey(key) if err != nil { @@ -222,6 +224,28 @@ func (d *LiveDecoder) Finalize() (DecodeCounters, error) { return d.d.counters, nil } +// FinalizedScopeCounts replaces streamed counts for adapters whose encoders +// make relational keep/drop decisions during Finalize. S3 is authoritative +// here because orphan chunks and stale-generation objects are only known after +// the complete bucket state has been assembled. +func (d *LiveDecoder) FinalizedScopeCounts(streamed map[Scope]uint64) (map[Scope]uint64, error) { + if d == nil || d.d == nil || !d.finalized { + return nil, errors.Wrap(ErrDecodeOptionsInvalid, "live decoder is not finalized") + } + out := make(map[Scope]uint64, len(streamed)) + for scope, count := range streamed { + if scope.Adapter != adapterS3 || d.d.s3 == nil { + out[scope] = count + } + } + if d.d.s3 != nil { + for name, count := range d.d.s3.RetainedRecordCounts() { + out[Scope{Adapter: adapterS3, Name: name}] = count + } + } + return out, nil +} + func (s Scope) String() string { return fmt.Sprintf("%s/%s", s.Adapter, s.Name) } diff --git a/internal/backup/live_producer.go b/internal/backup/live_producer.go index 0c42af2bf..c59d0e25a 100644 --- a/internal/backup/live_producer.go +++ b/internal/backup/live_producer.go @@ -285,7 +285,7 @@ func dumpPinnedBackup( if err != nil { return LiveBackupResult{}, errors.Wrap(err, "list live backup scopes") } - selected, err := selectLiveBackupScopes(opts.Adapters, opts.Scopes, listed.GetScopes()) + selected, err := selectLiveBackupScopes(opts.Adapters, opts.Scopes, listed.GetScopes(), begin.GetExpectedKeys()) if err != nil { return LiveBackupResult{}, err } @@ -313,6 +313,10 @@ func dumpPinnedBackup( if finalizeErr != nil { return LiveBackupResult{ReadTS: begin.GetReadTs(), Scopes: sortedScopeSet(selected)}, finalizeErr } + actual, err = decoder.FinalizedScopeCounts(actual) + if err != nil { + return LiveBackupResult{ReadTS: begin.GetReadTs(), Scopes: sortedScopeSet(selected), Counters: counters}, err + } if err := validateExpectedLiveCounts(selected, actual, begin.GetExpectedKeys()); err != nil { return LiveBackupResult{ReadTS: begin.GetReadTs(), Scopes: sortedScopeSet(selected), Counters: counters}, err } @@ -360,17 +364,48 @@ func streamSelectedBackup( return nil } -func selectLiveBackupScopes(adapters AdapterSet, requested []Scope, listed []*pb.BackupScope) (map[Scope]struct{}, error) { +func selectLiveBackupScopes( + adapters AdapterSet, + requested []Scope, + listed []*pb.BackupScope, + baseline []*pb.BackupExpectedKeys, +) (map[Scope]struct{}, error) { available, err := availableLiveBackupScopes(adapters, listed) if err != nil { return nil, err } if len(requested) == 0 { + if err := addBaselineLiveBackupScopes(available, adapters, baseline); err != nil { + return nil, err + } return available, nil } return requestedLiveBackupScopes(adapters, requested, available) } +func addBaselineLiveBackupScopes( + selected map[Scope]struct{}, + adapters AdapterSet, + baseline []*pb.BackupExpectedKeys, +) error { + for _, item := range baseline { + if item == nil || item.GetKeyCount() == 0 { + continue + } + scope := Scope{Adapter: item.GetAdapter(), Name: item.GetScope()} + if scope.Adapter == "" || scope.Name == "" { + return errors.Wrap(ErrLiveBackupScope, "server returned an empty baseline scope") + } + if err := validateLiveScope(scope); err != nil { + return err + } + if adapterEnabled(adapters, scope.Adapter) { + selected[scope] = struct{}{} + } + } + return nil +} + func availableLiveBackupScopes(adapters AdapterSet, listed []*pb.BackupScope) (map[Scope]struct{}, error) { available := make(map[Scope]struct{}, len(listed)) for _, item := range listed { @@ -618,12 +653,13 @@ type liveBackupLease struct { ttl time.Duration cancel context.CancelCauseFunc - mu sync.RWMutex - token []byte - renewals uint64 - stopOnce sync.Once - stopCh chan struct{} - doneCh chan error + mu sync.RWMutex + token []byte + expiresAt time.Time + renewals uint64 + stopOnce sync.Once + stopCh chan struct{} + doneCh chan error } func newLiveBackupLease( @@ -634,7 +670,7 @@ func newLiveBackupLease( ) *liveBackupLease { return &liveBackupLease{ rpc: rpc, ttl: ttl, - token: append([]byte(nil), begin.GetPinToken()...), cancel: cancel, + token: append([]byte(nil), begin.GetPinToken()...), expiresAt: time.Now().Add(ttl), cancel: cancel, stopCh: make(chan struct{}), doneCh: make(chan error, 1), } } @@ -678,8 +714,15 @@ func (l *liveBackupLease) renewLoop(ctx context.Context) error { func (l *liveBackupLease) renewOnce(ctx context.Context) (time.Duration, error) { l.mu.RLock() ttl := l.ttl + expiresAt := l.expiresAt l.mu.RUnlock() - resp, err := l.rpc.RenewBackup(ctx, &pb.RenewBackupRequest{ + attemptTimeout, err := liveBackupRenewalAttemptTimeout(ttl, time.Until(expiresAt)) + if err != nil { + return 0, err + } + renewCtx, cancel := context.WithTimeout(ctx, attemptTimeout) + defer cancel() + resp, err := l.rpc.RenewBackup(renewCtx, &pb.RenewBackupRequest{ PinToken: l.tokenSnapshot(), TtlMs: durationMillis(ttl), }) if err != nil { @@ -699,11 +742,23 @@ func (l *liveBackupLease) renewOnce(ctx context.Context) (time.Duration, error) l.mu.Lock() l.token = append(l.token[:0], resp.GetPinToken()...) l.ttl = renewedTTL + l.expiresAt = time.Now().Add(renewedTTL) l.renewals++ l.mu.Unlock() return interval, nil } +func liveBackupRenewalAttemptTimeout(ttl, remaining time.Duration) (time.Duration, error) { + timeout := ttl / liveBackupRenewalDivisor + if remaining < timeout { + timeout = remaining + } + if timeout <= 0 { + return 0, errors.New("renewal deadline has expired") + } + return timeout, nil +} + func liveBackupRenewalInterval(ttl time.Duration) (time.Duration, error) { interval := ttl / liveBackupRenewalDivisor if interval <= 0 { diff --git a/internal/backup/live_producer_test.go b/internal/backup/live_producer_test.go index d224cc8b8..3b8b8db99 100644 --- a/internal/backup/live_producer_test.go +++ b/internal/backup/live_producer_test.go @@ -187,6 +187,21 @@ func TestRunLiveBackupCountShortfallLeavesNoManifest(t *testing.T) { } } +func TestRunLiveBackupValidatesBaselineScopeMissingFromList(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + rpc.listed = nil + rpc.records = nil + root := filepath.Join(t.TempDir(), "dump") + _, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, Adapters: AdapterSet{Redis: true}, TTL: time.Minute, + }) + if !errors.Is(err, ErrCompactionDuringDump) { + t.Fatalf("err=%v, want ErrCompactionDuringDump", err) + } + assertLiveBackupHasNoManifest(t, root) +} + func TestRunLiveBackupRenewalFailureCancelsStreamAndReleasesPin(t *testing.T) { t.Parallel() rpc := successfulLiveBackupRPC() @@ -217,10 +232,28 @@ func TestRunLiveBackupRenewalFailureCancelsStreamAndReleasesPin(t *testing.T) { } } +func TestRunLiveBackupRenewalDeadlineCancelsStream(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + rpc.begin.TtlMsEffective = 30 + rpc.renewWait = true + rpc.renewStarted = make(chan struct{}) + rpc.blockStream = true + root := filepath.Join(t.TempDir(), "dump") + _, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, Adapters: AdapterSet{Redis: true}, TTL: time.Minute, + }) + if !errors.Is(err, ErrLiveBackupRenewal) { + t.Fatalf("err=%v, want ErrLiveBackupRenewal", err) + } + assertLiveBackupHasNoManifest(t, root) + assertLiveBackupEnded(t, rpc, "pin-token") +} + func TestRunLiveBackupStopsRenewalBeforeManifestPublication(t *testing.T) { t.Parallel() rpc := successfulLiveBackupRPC() - rpc.begin.TtlMsEffective = 9 + rpc.begin.TtlMsEffective = 300 rpc.renewStarted = make(chan struct{}) rpc.renewWait = true rpc.streamRelease = rpc.renewStarted @@ -240,7 +273,7 @@ func TestRunLiveBackupStopsRenewalBeforeManifestPublication(t *testing.T) { t.Fatal("MANIFEST.json was published while a pin renewal was still active") } manifest := readLiveBackupManifest(t, root) - if manifest.Live == nil || manifest.Live.ReadTS != 42 || manifest.BackupTSTTLMS != 9 { + if manifest.Live == nil || manifest.Live.ReadTS != 42 || manifest.BackupTSTTLMS != 300 { t.Fatalf("manifest live metadata = %+v", manifest) } } @@ -344,6 +377,34 @@ func TestCrossAdapterConsistency(t *testing.T) { assertCrossAdapterLiveBackup(t, root, body, result) } +func TestRunLiveBackupRejectsS3ScopeWithOnlyOrphanRecords(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + const ( + bucket = "deleted" + object = "orphan" + uploadID = "stale-upload" + ) + rpc.listed = []*pb.BackupScope{{Adapter: adapterS3, Scope: bucket}} + rpc.begin.ExpectedKeys = []*pb.BackupExpectedKeys{{Adapter: adapterS3, Scope: bucket, KeyCount: 2}} + rpc.records = []*pb.BackupKV{ + {Key: s3keys.ObjectManifestKey(bucket, 1, object), Value: encodeS3ManifestValue(t, map[string]any{ + "upload_id": uploadID, "size_bytes": 1, "parts": []map[string]any{ + {"part_no": 1, "size_bytes": 1, "chunk_count": 1}, + }, + })}, + {Key: s3keys.BlobKey(bucket, 1, object, uploadID, 1, 0), Value: []byte("x")}, + } + root := filepath.Join(t.TempDir(), "dump") + _, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, Adapters: AdapterSet{S3: true}, TTL: time.Minute, + }) + if !errors.Is(err, ErrCompactionDuringDump) { + t.Fatalf("err=%v, want ErrCompactionDuringDump", err) + } + assertLiveBackupHasNoManifest(t, root) +} + func crossAdapterLiveBackupRPC(t *testing.T) (*fakeLiveBackupRPC, []byte) { t.Helper() rpc := successfulLiveBackupRPC() diff --git a/internal/backup/live_test.go b/internal/backup/live_test.go index 9800af608..cedf6de97 100644 --- a/internal/backup/live_test.go +++ b/internal/backup/live_test.go @@ -19,6 +19,7 @@ func TestScopeForKey(t *testing.T) { scoped bool }{ {name: "dynamodb meta", key: []byte(DDBTableMetaPrefix + enc("orders")), want: Scope{Adapter: "dynamodb", Name: "orders"}, scoped: true}, + {name: "dynamodb generation", key: []byte(DDBTableGenPrefix + enc("orders")), scoped: false}, {name: "s3 bucket", key: s3keys.BucketMetaKey("photos"), want: Scope{Adapter: "s3", Name: "photos"}, scoped: true}, {name: "sqs metadata", key: []byte(SQSQueueMetaPrefix + enc("jobs")), want: Scope{Adapter: "sqs", Name: "jobs"}, scoped: true}, {name: "sqs sequence", key: []byte(SQSQueueSeqPrefix + enc("jobs")), scoped: false}, diff --git a/internal/backup/s3.go b/internal/backup/s3.go index 504e2bec4..56ccca429 100644 --- a/internal/backup/s3.go +++ b/internal/backup/s3.go @@ -546,6 +546,28 @@ func (s *S3Encoder) Finalize() error { return firstErr } +// RetainedRecordCounts reports the source records represented by the native +// dump after Finalize filtering. It excludes orphan buckets/chunks, stale +// generations, stale uploads, and undeclared chunks. +func (s *S3Encoder) RetainedRecordCounts() map[string]uint64 { + out := make(map[string]uint64, len(s.buckets)) + for _, bucket := range s.buckets { + if bucket.meta == nil { + continue + } + count := uint64(1) // !s3|bucket|meta + for _, object := range bucket.objects { + if object.manifest == nil || (bucket.activeGen != 0 && object.generation != bucket.activeGen) { + continue + } + count++ // !s3|obj|head + count += uint64(len(filterChunksForManifest(object.chunkPaths, object.uploadID, object.declaredParts))) + } + out[bucket.name] = count + } + return out +} + func (s *S3Encoder) flushBucket(b *s3BucketState) error { // Reject bucket-name dot segments before the filesystem join. // `EncodeSegment(".") == "."` and `EncodeSegment("..") == ".."` From d4d09bebacd0e5d656d49f9a6f051cde2d657dbd Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 22:21:45 +0900 Subject: [PATCH 4/6] backup: validate retained adapter records --- internal/backup/dynamodb.go | 27 +++++++- internal/backup/live.go | 59 ++++++++++++---- internal/backup/live_producer_test.go | 65 ++++++++++++++++++ internal/backup/redis_stream.go | 2 + internal/backup/redis_string.go | 98 +++++++++++++++++++++++---- internal/backup/redis_zset.go | 2 + internal/backup/sqs.go | 23 ++++++- 7 files changed, 250 insertions(+), 26 deletions(-) diff --git a/internal/backup/dynamodb.go b/internal/backup/dynamodb.go index b9a69b2ff..ec15117af 100644 --- a/internal/backup/dynamodb.go +++ b/internal/backup/dynamodb.go @@ -76,6 +76,7 @@ type ddbTableState struct { name string schema *pb.DynamoTableSchema itemsByGen map[uint64][]*pb.DynamoItem + retained uint64 } func ensureItemsByGen(m map[uint64][]*pb.DynamoItem) map[uint64][]*pb.DynamoItem { @@ -210,6 +211,18 @@ func (d *DDBEncoder) Finalize() error { return nil } +// RetainedRecordCounts reports source records represented by each emitted +// table after schema and generation filtering. +func (d *DDBEncoder) RetainedRecordCounts() map[string]uint64 { + out := make(map[string]uint64, len(d.tables)) + for _, st := range d.tables { + if st.schema != nil { + out[st.name] = st.retained + } + } + return out +} + func totalItemsAcrossGens(m map[uint64][]*pb.DynamoItem) int { total := 0 for _, items := range m { @@ -247,7 +260,19 @@ func (d *DDBEncoder) flushTable(st *ddbTableState) error { if err != nil { return err } - return d.writeEffectiveDDBItems(itemsDir, st.name, hashKey, rangeKey, items) + if err := d.writeEffectiveDDBItems(itemsDir, st.name, hashKey, rangeKey, items); err != nil { + return err + } + st.retained = 1 + retainedDDBItemRecords(st.itemsByGen, emitOrder) + return nil +} + +func retainedDDBItemRecords(itemsByGen map[uint64][]*pb.DynamoItem, emitOrder []uint64) uint64 { + var count uint64 + for _, generation := range emitOrder { + count += uint64(len(itemsByGen[generation])) + } + return count } func ddbGenerationEmitOrder(activeGen, migrationSourceGen uint64) []uint64 { diff --git a/internal/backup/live.go b/internal/backup/live.go index ed872f0ba..fe2c107c4 100644 --- a/internal/backup/live.go +++ b/internal/backup/live.go @@ -158,12 +158,21 @@ func parsedS3Scope(name string, ok bool, key []byte) (Scope, bool, error) { } func isRedisBackupKey(key []byte) bool { + if hasAnyBackupPrefix(key, + RedisHashMetaDeltaPrefix, + ListMetaDeltaPrefix, + ListClaimPrefix, + RedisSetMetaDeltaPrefix, + RedisZSetMetaDeltaPrefix, + RedisZSetScorePrefix, + ) { + return false + } prefixes := [...]string{ - RedisHashMetaDeltaPrefix, RedisHashMetaPrefix, RedisHashFieldPrefix, - ListMetaDeltaPrefix, ListMetaPrefix, ListItemPrefix, ListClaimPrefix, - RedisSetMetaDeltaPrefix, RedisSetMetaPrefix, RedisSetMemberPrefix, - RedisZSetMetaDeltaPrefix, RedisZSetMetaPrefix, RedisZSetMemberPrefix, - RedisZSetScorePrefix, RedisZSetLegacyBlobPrefix, + RedisHashMetaPrefix, RedisHashFieldPrefix, + ListMetaPrefix, ListItemPrefix, + RedisSetMetaPrefix, RedisSetMemberPrefix, + RedisZSetMetaPrefix, RedisZSetMemberPrefix, RedisZSetLegacyBlobPrefix, RedisStreamMetaPrefix, RedisStreamEntryPrefix, RedisStringPrefix, RedisHLLPrefix, RedisTTLPrefix, } @@ -225,25 +234,51 @@ func (d *LiveDecoder) Finalize() (DecodeCounters, error) { } // FinalizedScopeCounts replaces streamed counts for adapters whose encoders -// make relational keep/drop decisions during Finalize. S3 is authoritative -// here because orphan chunks and stale-generation objects are only known after -// the complete bucket state has been assembled. +// make relational keep/drop decisions during Finalize. The encoder counts are +// authoritative because orphan and stale records are only known after the +// complete adapter state has been assembled. func (d *LiveDecoder) FinalizedScopeCounts(streamed map[Scope]uint64) (map[Scope]uint64, error) { if d == nil || d.d == nil || !d.finalized { return nil, errors.Wrap(ErrDecodeOptionsInvalid, "live decoder is not finalized") } out := make(map[Scope]uint64, len(streamed)) for scope, count := range streamed { - if scope.Adapter != adapterS3 || d.d.s3 == nil { + if !adapterFinalizesScopeCounts(scope.Adapter) { out[scope] = count } } - if d.d.s3 != nil { - for name, count := range d.d.s3.RetainedRecordCounts() { + d.d.addFinalizedScopeCounts(out) + return out, nil +} + +func adapterFinalizesScopeCounts(adapter string) bool { + switch adapter { + case adapterDynamoDB, adapterS3, adapterRedis, adapterSQS: + return true + default: + return false + } +} + +func (d *dispatcher) addFinalizedScopeCounts(out map[Scope]uint64) { + if d.ddb != nil { + for name, count := range d.ddb.RetainedRecordCounts() { + out[Scope{Adapter: adapterDynamoDB, Name: name}] = count + } + } + if d.s3 != nil { + for name, count := range d.s3.RetainedRecordCounts() { out[Scope{Adapter: adapterS3, Name: name}] = count } } - return out, nil + if d.redis != nil { + out[Scope{Adapter: adapterRedis, Name: "db_0"}] = d.redis.RetainedRecordCount() + } + if d.sqs != nil { + for name, count := range d.sqs.RetainedRecordCounts() { + out[Scope{Adapter: adapterSQS, Name: name}] = count + } + } } func (s Scope) String() string { diff --git a/internal/backup/live_producer_test.go b/internal/backup/live_producer_test.go index 3b8b8db99..0dcccd73a 100644 --- a/internal/backup/live_producer_test.go +++ b/internal/backup/live_producer_test.go @@ -2,6 +2,7 @@ package backup import ( "context" + "encoding/base64" "os" "path/filepath" "sync" @@ -405,6 +406,70 @@ func TestRunLiveBackupRejectsS3ScopeWithOnlyOrphanRecords(t *testing.T) { assertLiveBackupHasNoManifest(t, root) } +func TestRunLiveBackupRejectsFinalizeDroppedScopes(t *testing.T) { + t.Parallel() + tests := []struct { + name string + adapter string + scope string + adapters AdapterSet + record *pb.BackupKV + }{ + { + name: "deleted SQS queue generation", + adapter: adapterSQS, + scope: "deleted-queue", + adapters: AdapterSet{SQS: true}, + record: &pb.BackupKV{ + Key: []byte(SQSQueueGenPrefix + base64.RawURLEncoding.EncodeToString([]byte("deleted-queue"))), + Value: []byte("2"), + }, + }, + { + name: "DynamoDB items without schema", + adapter: adapterDynamoDB, + scope: "deleted-table", + adapters: AdapterSet{DynamoDB: true}, + record: &pb.BackupKV{ + Key: EncodeDDBItemKey("deleted-table", 1, "orphan", ""), + Value: encodeItemValue(t, &pb.DynamoItem{Attributes: map[string]*pb.DynamoAttributeValue{ + "id": sAttr("orphan"), + }}), + }, + }, + { + name: "Redis stream entries without meta", + adapter: adapterRedis, + scope: "db_0", + adapters: AdapterSet{Redis: true}, + record: &pb.BackupKV{ + Key: streamEntryKey("orphan", 1, 0), + Value: encodeStreamEntryValue(t, "1-0", []string{"field", "value"}), + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + rpc.listed = []*pb.BackupScope{{Adapter: tc.adapter, Scope: tc.scope}} + rpc.begin.ExpectedKeys = []*pb.BackupExpectedKeys{{Adapter: tc.adapter, Scope: tc.scope, KeyCount: 1}} + rpc.records = []*pb.BackupKV{tc.record} + root := filepath.Join(t.TempDir(), "dump") + + _, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, + Adapters: tc.adapters, + TTL: time.Minute, + }) + if !errors.Is(err, ErrCompactionDuringDump) { + t.Fatalf("err=%v, want ErrCompactionDuringDump", err) + } + assertLiveBackupHasNoManifest(t, root) + }) + } +} + func crossAdapterLiveBackupRPC(t *testing.T) (*fakeLiveBackupRPC, []byte) { t.Helper() rpc := successfulLiveBackupRPC() diff --git a/internal/backup/redis_stream.go b/internal/backup/redis_stream.go index eb82f532f..5d93788a4 100644 --- a/internal/backup/redis_stream.go +++ b/internal/backup/redis_stream.go @@ -107,6 +107,7 @@ type redisStreamState struct { expireAtMs uint64 hasTTL bool inlineTTLOwned bool + externalTTL bool } // HandleStreamMeta processes one !stream|meta| record. @@ -182,6 +183,7 @@ func (r *RedisDB) streamState(userKey []byte) *redisStreamState { if expireAtMs, ok := r.claimPendingTTL(userKey); ok { st.expireAtMs = expireAtMs st.hasTTL = true + st.externalTTL = true } r.streams[uk] = st r.kindByKey[uk] = redisKindStream diff --git a/internal/backup/redis_string.go b/internal/backup/redis_string.go index fbf41fe98..092ad6e21 100644 --- a/internal/backup/redis_string.go +++ b/internal/backup/redis_string.go @@ -289,6 +289,12 @@ type RedisDB struct { // can distinguish "snapshot exceeded the buffer budget" from // "TTL records remained unmatched after the entire scan". pendingTTLOverflow int + + // simpleRecordCount and ttlRecordCount track live-backup source records + // that remain represented after adapter finalization. Wide-column records + // are counted from their finalized state maps. + simpleRecordCount uint64 + ttlRecordCount uint64 } // defaultPendingTTLBytesCap caps pendingTTL at 1 GiB cumulative @@ -388,10 +394,13 @@ func (r *RedisDB) HandleString(userKey, value []byte) error { if newFormat { r.inlineTTLOwned[string(userKey)] = struct{}{} } - if expireAtMs == 0 { - return nil + if expireAtMs != 0 { + if err := r.appendTTL(&r.stringsTTL, redisStringsTTLFile, userKey, expireAtMs); err != nil { + return err + } } - return r.appendTTL(&r.stringsTTL, redisStringsTTLFile, userKey, expireAtMs) + r.simpleRecordCount++ + return nil } // HandleHLL processes one !redis|hll| record. Legacy values are raw @@ -411,10 +420,13 @@ func (r *RedisDB) HandleHLL(userKey, value []byte) error { if newFormat { r.inlineTTLOwned[string(userKey)] = struct{}{} } - if expireAtMs == 0 { - return nil + if expireAtMs != 0 { + if err := r.appendTTL(&r.hllTTL, redisHLLTTLFile, userKey, expireAtMs); err != nil { + return err + } } - return r.appendTTL(&r.hllTTL, redisHLLTTLFile, userKey, expireAtMs) + r.simpleRecordCount++ + return nil } // HandleTTL processes one !redis|ttl| record. Routing @@ -451,9 +463,9 @@ func (r *RedisDB) HandleTTL(userKey, value []byte) error { switch r.kindByKey[string(userKey)] { case redisKindHLL: if _, ok := r.inlineTTLOwned[string(userKey)]; ok { - return nil + return r.retainTTLRecord(nil) } - return r.appendTTL(&r.hllTTL, redisHLLTTLFile, userKey, expireAtMs) + return r.retainTTLRecord(r.appendTTL(&r.hllTTL, redisHLLTTLFile, userKey, expireAtMs)) case redisKindString: // New-format strings carry TTL inline in the magic-prefix // header; HandleString already wrote the entry to @@ -462,21 +474,28 @@ func (r *RedisDB) HandleTTL(userKey, value []byte) error { // legacy strings (no inline TTL) reach the appendTTL call. // Codex P1 round 5. if _, ok := r.inlineTTLOwned[string(userKey)]; ok { - return nil + return r.retainTTLRecord(nil) } - return r.appendTTL(&r.stringsTTL, redisStringsTTLFile, userKey, expireAtMs) + return r.retainTTLRecord(r.appendTTL(&r.stringsTTL, redisStringsTTLFile, userKey, expireAtMs)) case redisKindHash, redisKindList, redisKindSet, redisKindZSet, redisKindStream: // Wide-column types fold TTL into the per-key JSON record // (`expire_at_ms` field) so a restorer can replay the // content + EXPIRE in one shot from the per-key file. r.inlineWideColumnTTL(r.kindByKey[string(userKey)], userKey, expireAtMs) - return nil + return r.retainTTLRecord(nil) case redisKindUnknown: - return r.parkUnknownTTL(userKey, expireAtMs) + return r.retainTTLRecord(r.parkUnknownTTL(userKey, expireAtMs)) } return nil } +func (r *RedisDB) retainTTLRecord(err error) error { + if err == nil { + r.ttlRecordCount++ + } + return err +} + // inlineWideColumnTTL sets expireAtMs/hasTTL on the per-key state // for a wide-column type. Extracted from HandleTTL so the parent // stays under the cyclop budget; the wide-column types all share @@ -539,6 +558,7 @@ func (r *RedisDB) inlineZSetTTL(userKey []byte, expireAtMs uint64) { func (r *RedisDB) inlineStreamTTL(userKey []byte, expireAtMs uint64) { st := r.streamState(userKey) + st.externalTTL = true if st.inlineTTLOwned { return } @@ -688,6 +708,60 @@ func (r *RedisDB) Finalize() error { return firstErr } +// RetainedRecordCount reports Redis source records represented by the native +// dump after finalization. Derivable indexes are excluded by ScopeForKey; +// orphan TTLs and meta-less stream records are removed here. +func (r *RedisDB) RetainedRecordCount() uint64 { + count := r.simpleRecordCount + r.ttlRecordCount + count = subtractRecordCount(count, nonNegativeRecordCount(r.orphanTTLCount)) + for _, st := range r.hashes { + count += boolRecordCount(st.metaSeen) + uint64(len(st.fields)) + } + for _, st := range r.lists { + count += boolRecordCount(st.metaSeen) + uint64(len(st.items)) + } + for _, st := range r.sets { + count += boolRecordCount(st.metaSeen) + uint64(len(st.members)) + } + for _, st := range r.zsets { + count += boolRecordCount(st.metaSeen) + if st.sawWide { + count += uint64(len(st.members)) + } else { + count += boolRecordCount(st.legacySeen) + } + } + for _, st := range r.streams { + if st.metaSeen { + count += 1 + uint64(len(st.entries)) + } else if st.externalTTL { + count = subtractRecordCount(count, 1) + } + } + return count +} + +func nonNegativeRecordCount(count int) uint64 { + if count <= 0 { + return 0 + } + return uint64(count) //nolint:gosec // Positive int is representable by uint64 on supported platforms. +} + +func boolRecordCount(on bool) uint64 { + if on { + return 1 + } + return 0 +} + +func subtractRecordCount(count, dropped uint64) uint64 { + if dropped >= count { + return 0 + } + return count - dropped +} + // dbDir returns the per-encoder root, e.g. "/redis/db_0/". // Computed once per call rather than at construction so the encoder's // outRoot remains a plain field — easier to reason about in tests. diff --git a/internal/backup/redis_zset.go b/internal/backup/redis_zset.go index 7ef086843..a16e6ff53 100644 --- a/internal/backup/redis_zset.go +++ b/internal/backup/redis_zset.go @@ -112,6 +112,7 @@ type redisZSetState struct { declaredLen int64 members map[string]float64 sawWide bool + legacySeen bool expireAtMs uint64 hasTTL bool inlineTTLOwned bool @@ -287,6 +288,7 @@ func (r *RedisDB) HandleZSetLegacyBlob(key, value []byte) error { // would surface stale post-migration leftovers in the dump. return nil } + st.legacySeen = true for _, e := range entries { st.members[e.member] = e.score } diff --git a/internal/backup/sqs.go b/internal/backup/sqs.go index 8045eec68..87b7debf9 100644 --- a/internal/backup/sqs.go +++ b/internal/backup/sqs.go @@ -121,6 +121,7 @@ type sqsQueueState struct { // means we have an orphan-only queue and the orphan branch // already drops messages). Codex P1 round 10. activeGen uint64 + genSeen bool // internalBuf accumulates side records in their on-disk shape if // includeSideRecords is on. Each line is the encoded prefix + // hex(rest-of-key) + value (b64) — implementation-grade detail @@ -336,7 +337,9 @@ func (s *SQSEncoder) HandleQueueGen(key, value []byte) error { if err != nil { return errors.Wrap(ErrSQSMalformedKey, err.Error()) } - s.queueState(encoded).activeGen = gen + st := s.queueState(encoded) + st.activeGen = gen + st.genSeen = true return nil } @@ -439,6 +442,24 @@ func (s *SQSEncoder) Finalize() error { return firstErr } +// RetainedRecordCounts reports source records represented by each emitted +// queue after metadata and generation filtering. +func (s *SQSEncoder) RetainedRecordCounts() map[string]uint64 { + out := make(map[string]uint64, len(s.queues)) + for _, st := range s.queues { + if st.meta == nil { + continue + } + count := uint64(1) + if st.genSeen { + count++ + } + visible, _ := filterStaleGenMessages(st.messages, st.activeGen) + out[st.name] = count + uint64(len(visible)) + } + return out +} + // flushOrphanQueueSideRecords emits buffered side records for a queue // whose !sqs|queue|meta row never arrived. Without this branch, // --include-sqs-side-records silently drops the post-DeleteQueue From 6a11219df4fdc4e1941e69e9b26c57e6034a415d Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 18:46:56 +0900 Subject: [PATCH 5/6] Fix live backup retained baseline validation --- adapter/admin_backup.go | 42 ++++++-- adapter/admin_backup_test.go | 114 ++++++++++++++++++-- internal/backup/decode.go | 90 ++++++++++------ internal/backup/dynamodb.go | 5 +- internal/backup/live.go | 64 +++++++++-- internal/backup/live_producer.go | 27 +++-- internal/backup/live_producer_test.go | 150 ++++++++++++++++++++++++++ internal/backup/redis_string.go | 41 ++++++- internal/backup/s3.go | 15 ++- 9 files changed, 480 insertions(+), 68 deletions(-) diff --git a/adapter/admin_backup.go b/adapter/admin_backup.go index b78efdace..298f61e94 100644 --- a/adapter/admin_backup.go +++ b/adapter/admin_backup.go @@ -545,7 +545,7 @@ func (s *AdminServer) StreamBackup( if err != nil { return err } - if err := s.requireUnexpiredBackupToken(tok); err != nil { + if err := s.requireLiveBackupSession(tok); err != nil { return err } if _, err := s.backupGroupsForToken(tok); err != nil { @@ -1059,21 +1059,47 @@ func (s *AdminServer) scanBackupScopeCounts( groups []backupGroup, requireLive func() error, ) (counts map[logicalbackup.Scope]uint64, applied uint64, retErr error) { - scanner := s.backupStore.NewBackupKeyScannerAtSnapshot(routes, readTS, s.backupConfig.scanPageSize) + scanner := s.backupStore.NewBackupScannerAtSnapshot(routes, readTS, s.backupConfig.scanPageSize) if scanner == nil { - return nil, 0, errors.New("backup key scanner is nil") + return nil, 0, errors.New("backup scanner is nil") + } + counter, err := logicalbackup.NewLiveScopeCounter(logicalbackup.AllAdapters()) + if err != nil { + _ = scanner.Close() + return nil, 0, errors.Wrap(err, "create retained backup baseline counter") } defer func() { retErr = finishBackupScan(ctx, scanner, retErr) }() - counts, err := collectBackupScopeCounts(ctx, scanner, requireLive) - if err != nil { + for { + if err := requireBackupScopeScanLive(requireLive); err != nil { + return nil, 0, err + } + pair, ok, err := scanner.Next(ctx) + if err != nil { + return nil, 0, errors.Wrap(err, "scan backup baseline") + } + if !ok { + break + } + if pair == nil { + return nil, 0, errors.New("backup scanner returned a nil record") + } + if err := counter.Add(pair.Key, pair.Value); err != nil { + return nil, 0, errors.Wrap(err, "count retained backup baseline") + } + } + if err := requireBackupScopeScanLive(requireLive); err != nil { return nil, 0, err } applied, err = currentMinBackupAppliedIndex(groups) if err != nil { return nil, 0, err } + counts, err = counter.RetainedCounts() + if err != nil { + return nil, 0, errors.Wrap(err, "finalize retained backup baseline") + } return counts, applied, nil } @@ -1146,11 +1172,15 @@ func (s *AdminServer) requireLiveBackupSession(tok backupToken) error { now := s.nowSnapshot() s.backupStateMu.Lock() defer s.backupStateMu.Unlock() - s.reapBackupSessionsLocked(now) session, ok := s.backupSessions[tok.pinID] if !ok || session.readTS != tok.readTS { + return status.Errorf(codes.FailedPrecondition, "%s", "backup route snapshot is unavailable on this endpoint") + } + if session.deadline.IsZero() || !now.Before(session.deadline) { + delete(s.backupSessions, tok.pinID) return status.Errorf(codes.FailedPrecondition, "%s", "backup pin token has expired") } + s.reapBackupSessionsLocked(now) return nil } diff --git a/adapter/admin_backup_test.go b/adapter/admin_backup_test.go index 56a5c037a..f0ab5bcff 100644 --- a/adapter/admin_backup_test.go +++ b/adapter/admin_backup_test.go @@ -1,6 +1,7 @@ package adapter import ( + "bytes" "context" "encoding/base64" stderrors "errors" @@ -123,6 +124,7 @@ type backupTestStore struct { captureErr error validateErr error valueKeys [][]byte + values map[string][]byte } func (s *backupTestStore) ValidateBackupSnapshotAt(context.Context, kv.BackupRouteSnapshot, uint64, int) error { @@ -185,10 +187,47 @@ func (s *backupTestStore) newBackupScannerAtSnapshot(ts uint64, keyFilter kv.Bac } } s.valueKeys = append(s.valueKeys, append([]byte(nil), key...)) - pairs = append(pairs, &kvstore.KVPair{Key: append([]byte(nil), key...), Value: []byte("value")}) + value := s.values[string(key)] + if value == nil { + value = backupTestDefaultValueForKey(key) + } + pairs = append(pairs, &kvstore.KVPair{Key: append([]byte(nil), key...), Value: append([]byte(nil), value...)}) } + onExhaust := s.onExhaust + delay := s.scanDelay closeErr := s.pairCloseErr - return &backupPairScanner{pairs: pairs, err: filterErr, closeErr: closeErr} + return &backupPairScanner{pairs: pairs, err: filterErr, onExhaust: onExhaust, delay: delay, closeErr: closeErr} +} + +func backupTestDefaultValueForKey(key []byte) []byte { + scope, scoped, err := logicalbackup.ScopeForKey(key) + if err != nil || !scoped { + return []byte("value") + } + if scope.Adapter != "dynamodb" { + return []byte("value") + } + if bytes.HasPrefix(key, []byte(logicalbackup.DDBTableMetaPrefix)) { + value, err := encodeStoredDynamoTableSchema(&dynamoTableSchema{ + TableName: scope.Name, + AttributeDefinitions: map[string]string{"id": "S"}, + PrimaryKey: dynamoKeySchema{HashKey: "id"}, + Generation: 1, + }) + if err != nil { + panic(err) + } + return value + } + if bytes.HasPrefix(key, []byte(logicalbackup.DDBItemPrefix)) { + id := "value" + value, err := encodeStoredDynamoItem(map[string]attributeValue{"id": {S: &id}}) + if err != nil { + panic(err) + } + return value + } + return []byte("value") } type backupSliceScanner struct { @@ -229,10 +268,13 @@ func (s *backupSliceScanner) Next(ctx context.Context) ([]byte, bool, error) { func (s *backupSliceScanner) Close() error { return s.closeErr } type backupPairScanner struct { - pairs []*kvstore.KVPair - index int - err error - closeErr error + pairs []*kvstore.KVPair + index int + err error + onExhaust func() + once sync.Once + delay time.Duration + closeErr error } func (s *backupPairScanner) Next(ctx context.Context) (*kvstore.KVPair, bool, error) { @@ -244,7 +286,21 @@ func (s *backupPairScanner) Next(ctx context.Context) (*kvstore.KVPair, bool, er s.err = nil return nil, false, err } + if s.delay > 0 && s.index < len(s.pairs) { + timer := time.NewTimer(s.delay) + select { + case <-ctx.Done(): + timer.Stop() + return nil, false, ctx.Err() + case <-timer.C: + } + } if s.index >= len(s.pairs) { + s.once.Do(func() { + if s.onExhaust != nil { + s.onExhaust() + } + }) return nil, false, nil } pair := s.pairs[s.index] @@ -350,6 +406,45 @@ func TestBeginBackupLifecycleAndBaselineAtPinnedTimestamp(t *testing.T) { } } +func TestBeginBackupExpectedKeysUseRetainedCounts(t *testing.T) { + t.Parallel() + const table = "orders" + activeID := "active" + staleID := "stale" + schema, err := encodeStoredDynamoTableSchema(&dynamoTableSchema{ + TableName: table, + AttributeDefinitions: map[string]string{"id": "S"}, + PrimaryKey: dynamoKeySchema{HashKey: "id"}, + Generation: 2, + }) + require.NoError(t, err) + activeItem, err := encodeStoredDynamoItem(map[string]attributeValue{"id": {S: &activeID}}) + require.NoError(t, err) + staleItem, err := encodeStoredDynamoItem(map[string]attributeValue{"id": {S: &staleID}}) + require.NoError(t, err) + schemaKey := logicalbackup.EncodeDDBTableMetaKey(table) + activeKey := logicalbackup.EncodeDDBItemKey(table, 2, activeID, "") + staleKey := logicalbackup.EncodeDDBItemKey(table, 1, staleID, "") + store := &backupTestStore{ + keys: [][]byte{staleKey, activeKey, schemaKey}, + values: map[string][]byte{ + string(schemaKey): schema, + string(activeKey): activeItem, + string(staleKey): staleItem, + }, + } + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + require.Len(t, begin.GetExpectedKeys(), 1) + require.Equal(t, "dynamodb", begin.GetExpectedKeys()[0].GetAdapter()) + require.Equal(t, table, begin.GetExpectedKeys()[0].GetScope()) + require.EqualValues(t, 2, begin.GetExpectedKeys()[0].GetKeyCount()) +} + func TestSnapshotBackupGroupsExcludesReservedTSOGroup(t *testing.T) { t.Parallel() groups := map[uint64]*backupTestGroup{ @@ -486,6 +581,9 @@ func TestStreamBackupUsesPinTimestampAndScopeFilter(t *testing.T) { srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) require.NoError(t, err) + store.mu.Lock() + store.valueKeys = nil + store.mu.Unlock() stream := &backupTestStream{ctx: context.Background()} err = srv.StreamBackup(&pb.StreamBackupRequest{ @@ -644,7 +742,7 @@ func TestListBackupScopesReportsScannerCloseError(t *testing.T) { require.NoError(t, err) store.mu.Lock() - store.keyCloseErr = stderrors.New("close failed") + store.pairCloseErr = stderrors.New("close failed") store.mu.Unlock() _, err = srv.ListAdaptersAndScopes(context.Background(), &pb.ListAdaptersAndScopesRequest{PinToken: begin.GetPinToken()}) require.Equal(t, codes.FailedPrecondition, status.Code(err)) @@ -884,7 +982,7 @@ func TestBackupTokenDeadlineRotatesAndFailsClosed(t *testing.T) { _, err = srv.ListAdaptersAndScopes(context.Background(), &pb.ListAdaptersAndScopesRequest{PinToken: begin.GetPinToken()}) require.Equal(t, codes.FailedPrecondition, status.Code(err)) err = srv.StreamBackup(&pb.StreamBackupRequest{PinToken: begin.GetPinToken()}, &backupTestStream{ctx: context.Background()}) - require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.NoError(t, err, "streams use the renewed server-side session deadline") _, err = srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: renewed.GetPinToken()}) require.NoError(t, err) diff --git a/internal/backup/decode.go b/internal/backup/decode.go index 4e906ff8b..f709f6f8a 100644 --- a/internal/backup/decode.go +++ b/internal/backup/decode.go @@ -94,6 +94,11 @@ type DecodeOptions struct { // per-adapter encoders ("redis_orphan_ttl", "ddb_orphan_items", // ...). The dispatcher itself does not emit warnings. WarnSink func(event string, fields ...any) + + // countOnly builds adapter state for retained-record accounting without + // writing dump files. It is intentionally unexported: callers use + // LiveScopeCounter instead of depending on dispatcher internals. + countOnly bool } // DecodeCounters reports per-class entry counts after a successful @@ -187,11 +192,9 @@ type dispatcher struct { // validation it runs is the *parameter* validation; filesystem // existence / permission errors surface later, on first write. func newDispatcher(opts DecodeOptions) (*dispatcher, error) { - if opts.OutRoot == "" { - return nil, errors.Wrap(ErrDecodeOptionsInvalid, "OutRoot required") - } - if opts.DynamoDBBundleSizeBytes < 0 { - return nil, errors.Wrap(ErrDecodeOptionsInvalid, "DynamoDBBundleSizeBytes must not be negative") + opts, err := normalizeDecodeOptions(opts) + if err != nil { + return nil, err } d := &dispatcher{opts: opts} if opts.Adapters.DynamoDB { @@ -201,37 +204,15 @@ func newDispatcher(opts DecodeOptions) (*dispatcher, error) { WithWarnSink(opts.WarnSink) } if opts.Adapters.S3 { - scratch := opts.ScratchRoot - if scratch == "" { - // NEVER default scratch to OutRoot — S3Encoder.Finalize - // runs os.RemoveAll(scratchRoot/s3/) and OutRoot also - // holds the *final* assembled bodies at /s3/. - // Sharing the root would wipe the dump immediately after - // it lands (gemini r1 security-high on PR #806). The - // dedicated `.scratch` subtree keeps the two trees - // disjoint regardless of where OutRoot points. - scratch = filepath.Join(opts.OutRoot, ".scratch") - } - // Belt-and-braces on the operator-supplied path: the - // default-path fix above only protects empty ScratchRoot; - // a CLI invocation that passes --scratch-root=$OUT explicitly - // would still let Finalize wipe the dump (Codex r3 P1 on - // PR #806). Fail fast on cleaned-equal paths so the - // misconfiguration surfaces as ErrDecodeOptionsInvalid - // rather than as silent data loss after a long decode. - if filepath.Clean(scratch) == filepath.Clean(opts.OutRoot) { - return nil, errors.Wrap(ErrDecodeOptionsInvalid, - "ScratchRoot must not equal OutRoot: S3Encoder.Finalize "+ - "would os.RemoveAll the final /s3/ tree") + s3, err := newS3DecodeEncoder(opts) + if err != nil { + return nil, err } - d.s3 = NewS3Encoder(opts.OutRoot, scratch). - WithIncludeIncompleteUploads(opts.IncludeIncompleteUploads). - WithIncludeOrphans(opts.IncludeOrphans). - WithRenameCollisions(opts.RenameS3Collisions). - WithWarnSink(opts.WarnSink) + d.s3 = s3 } if opts.Adapters.Redis { d.redis = NewRedisDB(opts.OutRoot, opts.RedisDBIndex). + WithCountOnly(opts.countOnly). WithWarnSink(opts.WarnSink) } if opts.Adapters.SQS { @@ -243,6 +224,51 @@ func newDispatcher(opts DecodeOptions) (*dispatcher, error) { return d, nil } +func normalizeDecodeOptions(opts DecodeOptions) (DecodeOptions, error) { + if opts.OutRoot == "" && !opts.countOnly { + return DecodeOptions{}, errors.Wrap(ErrDecodeOptionsInvalid, "OutRoot required") + } + if opts.OutRoot == "" { + opts.OutRoot = "." + } + if opts.DynamoDBBundleSizeBytes < 0 { + return DecodeOptions{}, errors.Wrap(ErrDecodeOptionsInvalid, "DynamoDBBundleSizeBytes must not be negative") + } + return opts, nil +} + +func newS3DecodeEncoder(opts DecodeOptions) (*S3Encoder, error) { + scratch := opts.ScratchRoot + if scratch == "" { + // NEVER default scratch to OutRoot — S3Encoder.Finalize + // runs os.RemoveAll(scratchRoot/s3/) and OutRoot also + // holds the *final* assembled bodies at /s3/. + // Sharing the root would wipe the dump immediately after + // it lands (gemini r1 security-high on PR #806). The + // dedicated `.scratch` subtree keeps the two trees + // disjoint regardless of where OutRoot points. + scratch = filepath.Join(opts.OutRoot, ".scratch") + } + // Belt-and-braces on the operator-supplied path: the + // default-path fix above only protects empty ScratchRoot; + // a CLI invocation that passes --scratch-root=$OUT explicitly + // would still let Finalize wipe the dump (Codex r3 P1 on + // PR #806). Fail fast on cleaned-equal paths so the + // misconfiguration surfaces as ErrDecodeOptionsInvalid + // rather than as silent data loss after a long decode. + if filepath.Clean(scratch) == filepath.Clean(opts.OutRoot) { + return nil, errors.Wrap(ErrDecodeOptionsInvalid, + "ScratchRoot must not equal OutRoot: S3Encoder.Finalize "+ + "would os.RemoveAll the final /s3/ tree") + } + return NewS3Encoder(opts.OutRoot, scratch). + WithIncludeIncompleteUploads(opts.IncludeIncompleteUploads). + WithIncludeOrphans(opts.IncludeOrphans). + WithRenameCollisions(opts.RenameS3Collisions). + WithCountOnly(opts.countOnly). + WithWarnSink(opts.WarnSink), nil +} + // handleEntry is the per-entry hook ReadSnapshot calls. Tombstones // are counted and dropped before any prefix matching runs — Phase 0a // dumps reflect the *current* user-visible state, not the diff --git a/internal/backup/dynamodb.go b/internal/backup/dynamodb.go index ec15117af..93d0f96dc 100644 --- a/internal/backup/dynamodb.go +++ b/internal/backup/dynamodb.go @@ -76,7 +76,6 @@ type ddbTableState struct { name string schema *pb.DynamoTableSchema itemsByGen map[uint64][]*pb.DynamoItem - retained uint64 } func ensureItemsByGen(m map[uint64][]*pb.DynamoItem) map[uint64][]*pb.DynamoItem { @@ -217,7 +216,8 @@ func (d *DDBEncoder) RetainedRecordCounts() map[string]uint64 { out := make(map[string]uint64, len(d.tables)) for _, st := range d.tables { if st.schema != nil { - out[st.name] = st.retained + emitOrder := ddbGenerationEmitOrder(st.schema.GetGeneration(), st.schema.GetMigratingFromGeneration()) + out[st.name] = 1 + retainedDDBItemRecords(st.itemsByGen, emitOrder) } } return out @@ -263,7 +263,6 @@ func (d *DDBEncoder) flushTable(st *ddbTableState) error { if err := d.writeEffectiveDDBItems(itemsDir, st.name, hashKey, rangeKey, items); err != nil { return err } - st.retained = 1 + retainedDDBItemRecords(st.itemsByGen, emitOrder) return nil } diff --git a/internal/backup/live.go b/internal/backup/live.go index fe2c107c4..6f2a0c070 100644 --- a/internal/backup/live.go +++ b/internal/backup/live.go @@ -241,14 +241,56 @@ func (d *LiveDecoder) FinalizedScopeCounts(streamed map[Scope]uint64) (map[Scope if d == nil || d.d == nil || !d.finalized { return nil, errors.Wrap(ErrDecodeOptionsInvalid, "live decoder is not finalized") } + return finalizedScopeCounts(d.d, streamed), nil +} + +// LiveScopeCounter classifies live backup key/value pairs using the same +// adapter state as the decoder, but in count-only mode. Server-side baseline +// scans use it so BeginBackup.expected_keys uses the same retained-record +// denominator that the live producer validates after decoder finalization. +type LiveScopeCounter struct { + d *dispatcher + streamed map[Scope]uint64 +} + +func NewLiveScopeCounter(adapters AdapterSet) (*LiveScopeCounter, error) { + d, err := newDispatcher(DecodeOptions{Adapters: adapters, countOnly: true}) + if err != nil { + return nil, err + } + return &LiveScopeCounter{d: d, streamed: make(map[Scope]uint64)}, nil +} + +func (c *LiveScopeCounter) Add(key, value []byte) error { + if c == nil || c.d == nil { + return errors.Wrap(ErrDecodeOptionsInvalid, "live scope counter is unavailable") + } + scope, scoped, err := ScopeForKey(key) + if err != nil { + return err + } + if scoped { + c.streamed[scope]++ + } + return c.d.route(key, value) +} + +func (c *LiveScopeCounter) RetainedCounts() (map[Scope]uint64, error) { + if c == nil || c.d == nil { + return nil, errors.Wrap(ErrDecodeOptionsInvalid, "live scope counter is unavailable") + } + return finalizedScopeCounts(c.d, c.streamed), nil +} + +func finalizedScopeCounts(d *dispatcher, streamed map[Scope]uint64) map[Scope]uint64 { out := make(map[Scope]uint64, len(streamed)) for scope, count := range streamed { - if !adapterFinalizesScopeCounts(scope.Adapter) { + if count > 0 && !adapterFinalizesScopeCounts(scope.Adapter) { out[scope] = count } } - d.d.addFinalizedScopeCounts(out) - return out, nil + d.addFinalizedScopeCounts(out) + return out } func adapterFinalizesScopeCounts(adapter string) bool { @@ -263,24 +305,32 @@ func adapterFinalizesScopeCounts(adapter string) bool { func (d *dispatcher) addFinalizedScopeCounts(out map[Scope]uint64) { if d.ddb != nil { for name, count := range d.ddb.RetainedRecordCounts() { - out[Scope{Adapter: adapterDynamoDB, Name: name}] = count + addPositiveScopeCount(out, Scope{Adapter: adapterDynamoDB, Name: name}, count) } } if d.s3 != nil { for name, count := range d.s3.RetainedRecordCounts() { - out[Scope{Adapter: adapterS3, Name: name}] = count + addPositiveScopeCount(out, Scope{Adapter: adapterS3, Name: name}, count) } } if d.redis != nil { - out[Scope{Adapter: adapterRedis, Name: "db_0"}] = d.redis.RetainedRecordCount() + addPositiveScopeCount(out, Scope{Adapter: adapterRedis, Name: "db_0"}, d.redis.RetainedRecordCount()) } if d.sqs != nil { for name, count := range d.sqs.RetainedRecordCounts() { - out[Scope{Adapter: adapterSQS, Name: name}] = count + addPositiveScopeCount(out, Scope{Adapter: adapterSQS, Name: name}, count) } } } +func addPositiveScopeCount(out map[Scope]uint64, scope Scope, count uint64) { + if count == 0 { + delete(out, scope) + return + } + out[scope] = count +} + func (s Scope) String() string { return fmt.Sprintf("%s/%s", s.Adapter, s.Name) } diff --git a/internal/backup/live_producer.go b/internal/backup/live_producer.go index c59d0e25a..e26fe814c 100644 --- a/internal/backup/live_producer.go +++ b/internal/backup/live_producer.go @@ -374,10 +374,10 @@ func selectLiveBackupScopes( if err != nil { return nil, err } + if err := addBaselineLiveBackupScopes(available, adapters, baseline); err != nil { + return nil, err + } if len(requested) == 0 { - if err := addBaselineLiveBackupScopes(available, adapters, baseline); err != nil { - return nil, err - } return available, nil } return requestedLiveBackupScopes(adapters, requested, available) @@ -590,16 +590,17 @@ func liveBackupManifest(begin *pb.BeginBackupResponse, token []byte, selected ma func liveManifestAdapters(enabled AdapterSet, selected map[Scope]struct{}) *Adapters { out := &Adapters{} - if enabled.DynamoDB { + includeEnabledEmpty := len(selected) == 0 + if includeEnabledEmpty && enabled.DynamoDB { out.DynamoDB = &Adapter{} } - if enabled.S3 { + if includeEnabledEmpty && enabled.S3 { out.S3 = &Adapter{} } - if enabled.Redis { + if includeEnabledEmpty && enabled.Redis { out.Redis = &Adapter{} } - if enabled.SQS { + if includeEnabledEmpty && enabled.SQS { out.SQS = &Adapter{} } for _, scope := range sortedScopeSet(selected) { @@ -611,12 +612,24 @@ func liveManifestAdapters(enabled AdapterSet, selected map[Scope]struct{}) *Adap func addLiveManifestScope(out *Adapters, scope Scope) { switch scope.Adapter { case adapterDynamoDB: + if out.DynamoDB == nil { + out.DynamoDB = &Adapter{} + } out.DynamoDB.Tables = append(out.DynamoDB.Tables, scope.Name) case adapterS3: + if out.S3 == nil { + out.S3 = &Adapter{} + } out.S3.Buckets = append(out.S3.Buckets, scope.Name) case adapterRedis: + if out.Redis == nil { + out.Redis = &Adapter{} + } addLiveManifestRedisScope(out.Redis, scope.Name) case adapterSQS: + if out.SQS == nil { + out.SQS = &Adapter{} + } out.SQS.Queues = append(out.SQS.Queues, scope.Name) } } diff --git a/internal/backup/live_producer_test.go b/internal/backup/live_producer_test.go index 0dcccd73a..4a8b61078 100644 --- a/internal/backup/live_producer_test.go +++ b/internal/backup/live_producer_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "os" "path/filepath" + "reflect" "sync" "testing" "time" @@ -363,6 +364,48 @@ func TestRunLiveBackupUnavailableScopeLeavesNoManifest(t *testing.T) { assertLiveBackupHasNoManifest(t, root) } +func TestRunLiveBackupAcceptsRequestedBaselineOnlyScope(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + rpc.listed = nil + root := filepath.Join(t.TempDir(), "dump") + result, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, + Adapters: AdapterSet{Redis: true}, + Scopes: []Scope{{Adapter: adapterRedis, Name: "db_0"}}, + TTL: time.Minute, + }) + if err != nil { + t.Fatalf("RunLiveBackup: %v", err) + } + if len(result.Scopes) != 1 || result.Scopes[0] != (Scope{Adapter: adapterRedis, Name: "db_0"}) { + t.Fatalf("scopes=%+v", result.Scopes) + } + assertLiveBackupManifest(t, root) +} + +func TestRunLiveBackupScopedManifestOmitsUnselectedAdapters(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + root := filepath.Join(t.TempDir(), "dump") + _, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, + Adapters: AllAdapters(), + Scopes: []Scope{{Adapter: adapterRedis, Name: "db_0"}}, + TTL: time.Minute, + }) + if err != nil { + t.Fatalf("RunLiveBackup: %v", err) + } + manifest := readLiveBackupManifest(t, root) + if manifest.Adapters.Redis == nil || len(manifest.Adapters.Redis.Databases) != 1 { + t.Fatalf("manifest redis adapters=%+v", manifest.Adapters) + } + if manifest.Adapters.DynamoDB != nil || manifest.Adapters.S3 != nil || manifest.Adapters.SQS != nil { + t.Fatalf("manifest included unselected adapters=%+v", manifest.Adapters) + } +} + func TestCrossAdapterConsistency(t *testing.T) { t.Parallel() rpc, body := crossAdapterLiveBackupRPC(t) @@ -378,6 +421,113 @@ func TestCrossAdapterConsistency(t *testing.T) { assertCrossAdapterLiveBackup(t, root, body, result) } +func TestLiveScopeCounterUsesRetainedCounts(t *testing.T) { + t.Parallel() + schema := &pb.DynamoTableSchema{ + TableName: "orders", + PrimaryKey: &pb.DynamoKeySchema{HashKey: "id"}, + AttributeDefinitions: map[string]string{"id": "S"}, + Generation: 2, + } + item := func(id string) *pb.DynamoItem { + return &pb.DynamoItem{Attributes: map[string]*pb.DynamoAttributeValue{"id": sAttr(id)}} + } + const ( + bucket = "photos" + object = "image.jpg" + uploadID = "upload-1" + queue = "jobs" + ) + encQueue := base64.RawURLEncoding.EncodeToString([]byte(queue)) + tests := []struct { + name string + adapters AdapterSet + records []*pb.BackupKV + want map[Scope]uint64 + }{ + { + name: "S3 stale generation chunks", + adapters: AdapterSet{S3: true}, + records: []*pb.BackupKV{ + {Key: s3keys.BucketMetaKey(bucket), Value: encodeS3BucketMetaValue(t, map[string]any{ + "bucket_name": bucket, "generation": 2, + })}, + {Key: s3keys.ObjectManifestKey(bucket, 2, object), Value: encodeS3ManifestValue(t, map[string]any{ + "upload_id": uploadID, "size_bytes": 1, "parts": []map[string]any{ + {"part_no": 1, "size_bytes": 1, "chunk_count": 1}, + }, + })}, + {Key: s3keys.BlobKey(bucket, 2, object, uploadID, 1, 0), Value: []byte("x")}, + {Key: s3keys.ObjectManifestKey(bucket, 1, "old"), Value: encodeS3ManifestValue(t, map[string]any{ + "upload_id": uploadID, "size_bytes": 1, "parts": []map[string]any{ + {"part_no": 1, "size_bytes": 1, "chunk_count": 1}, + }, + })}, + {Key: s3keys.BlobKey(bucket, 1, "old", uploadID, 1, 0), Value: []byte("y")}, + }, + want: map[Scope]uint64{{Adapter: adapterS3, Name: bucket}: 3}, + }, + { + name: "DynamoDB stale generation rows", + adapters: AdapterSet{DynamoDB: true}, + records: []*pb.BackupKV{ + {Key: EncodeDDBTableMetaKey("orders"), Value: encodeSchemaValue(t, schema)}, + {Key: EncodeDDBItemKey("orders", 2, "active", ""), Value: encodeItemValue(t, item("active"))}, + {Key: EncodeDDBItemKey("orders", 1, "stale", ""), Value: encodeItemValue(t, item("stale"))}, + }, + want: map[Scope]uint64{{Adapter: adapterDynamoDB, Name: "orders"}: 2}, + }, + { + name: "SQS stale generation messages", + adapters: AdapterSet{SQS: true}, + records: []*pb.BackupKV{ + {Key: EncodeQueueMetaKey(queue), Value: encodeQueueMetaValue(t, map[string]any{ + "name": queue, "fifo": false, "partition_count": 1, "generation": 2, + })}, + {Key: []byte(SQSQueueGenPrefix + encQueue), Value: []byte("2")}, + {Key: EncodeMsgDataKey(queue, 2, "live"), Value: encodeMessageValue(t, map[string]any{ + "message_id": "live", "body": []byte("live"), "send_timestamp_millis": 1, "queue_generation": 2, + })}, + {Key: EncodeMsgDataKey(queue, 1, "stale"), Value: encodeMessageValue(t, map[string]any{ + "message_id": "stale", "body": []byte("stale"), "send_timestamp_millis": 1, "queue_generation": 1, + })}, + }, + want: map[Scope]uint64{{Adapter: adapterSQS, Name: queue}: 3}, + }, + { + name: "Redis zset legacy row superseded by wide columns", + adapters: AdapterSet{Redis: true}, + records: []*pb.BackupKV{ + {Key: zsetLegacyBlobKey("rank"), Value: encodeZSetLegacyBlobValue(t, []zsetLegacyEntry{{member: "old", score: 1}})}, + {Key: zsetMetaKey("rank"), Value: encodeZSetMetaValue(1)}, + {Key: zsetMemberKey("rank", []byte("new")), Value: encodeZSetScoreValue(2)}, + }, + want: map[Scope]uint64{{Adapter: adapterRedis, Name: "db_0"}: 2}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + counter, err := NewLiveScopeCounter(tc.adapters) + if err != nil { + t.Fatalf("NewLiveScopeCounter: %v", err) + } + for _, record := range tc.records { + if err := counter.Add(record.GetKey(), record.GetValue()); err != nil { + t.Fatalf("Add(%q): %v", record.GetKey(), err) + } + } + got, err := counter.RetainedCounts() + if err != nil { + t.Fatalf("RetainedCounts: %v", err) + } + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("retained counts=%v, want %v", got, tc.want) + } + }) + } +} + func TestRunLiveBackupRejectsS3ScopeWithOnlyOrphanRecords(t *testing.T) { t.Parallel() rpc := successfulLiveBackupRPC() diff --git a/internal/backup/redis_string.go b/internal/backup/redis_string.go index 092ad6e21..3096647a4 100644 --- a/internal/backup/redis_string.go +++ b/internal/backup/redis_string.go @@ -151,8 +151,10 @@ const ( // Handle* methods are NOT goroutine-safe; the decoder pipeline is // inherently sequential per scope, so a mutex would only add cost. type RedisDB struct { - outRoot string - dbIndex int + outRoot string + dbIndex int + countOnly bool + finalized bool // kindByKey records the Redis type each user key was first seen as. // Populated by HandleString and HandleHLL; consulted by HandleTTL. @@ -370,6 +372,11 @@ func (r *RedisDB) WithPendingTTLByteCap(capacity int) *RedisDB { return r } +func (r *RedisDB) WithCountOnly(on bool) *RedisDB { + r.countOnly = on + return r +} + // WithWarnSink wires a structured-warning sink. The sink is called with // stable event names ("redis_orphan_ttl", etc.) and key=value pairs. func (r *RedisDB) WithWarnSink(fn func(event string, fields ...any)) *RedisDB { @@ -684,7 +691,10 @@ func (r *RedisDB) Finalize() error { // record was dropped, or a `!redis|ttl|` entry written for a // key whose type prefix we don't recognise (a future Redis // type added on the live side without a backup-encoder update). - r.orphanTTLCount += len(r.pendingTTL) + if !r.finalized { + r.orphanTTLCount += len(r.pendingTTL) + r.finalized = true + } // Codex P2 (PR #790 r13): also warn on pendingTTLOverflow even // when orphanTTLCount is zero. Overflow entries fail-closed at // parkUnknownTTL with ErrPendingTTLBufferFull and never reach @@ -713,7 +723,7 @@ func (r *RedisDB) Finalize() error { // orphan TTLs and meta-less stream records are removed here. func (r *RedisDB) RetainedRecordCount() uint64 { count := r.simpleRecordCount + r.ttlRecordCount - count = subtractRecordCount(count, nonNegativeRecordCount(r.orphanTTLCount)) + count = subtractRecordCount(count, r.effectiveOrphanTTLCount()) for _, st := range r.hashes { count += boolRecordCount(st.metaSeen) + uint64(len(st.fields)) } @@ -741,6 +751,14 @@ func (r *RedisDB) RetainedRecordCount() uint64 { return count } +func (r *RedisDB) effectiveOrphanTTLCount() uint64 { + count := nonNegativeRecordCount(r.orphanTTLCount) + if !r.finalized { + count += uint64(len(r.pendingTTL)) + } + return count +} + func nonNegativeRecordCount(count int) uint64 { if count <= 0 { return 0 @@ -794,6 +812,9 @@ func flushWideColumnDir[T any](r *RedisDB, states map[string]T, subdir string, f if len(states) == 0 { return nil } + if r.countOnly { + return nil + } dir := filepath.Join(r.dbDir(), subdir) if err := r.ensureDir(dir); err != nil { return err @@ -927,6 +948,9 @@ func lstatRefuseSymlink(path string) error { } func (r *RedisDB) writeBlob(subdir string, userKey, value []byte) error { + if r.countOnly { + return nil + } encoded := EncodeSegment(userKey) if err := r.recordIfFallback(encoded, userKey); err != nil { return err @@ -943,6 +967,9 @@ func (r *RedisDB) writeBlob(subdir string, userKey, value []byte) error { } func (r *RedisDB) appendTTL(slot **jsonlFile, baseName string, userKey []byte, expireAtMs uint64) error { + if r.countOnly { + return nil + } if *slot == nil { // Route the parent directory through ensureDir so the // shared assertNoSymlinkAncestors guard fires before we @@ -988,6 +1015,9 @@ func (r *RedisDB) appendTTL(slot **jsonlFile, baseName string, userKey []byte, e // Idempotent: a duplicate (encoded, original) pair is harmless because // LoadKeymap's "last record wins" behaviour leaves the same mapping. func (r *RedisDB) recordIfFallback(encoded string, userKey []byte) error { + if r.countOnly { + return nil + } if !IsShaFallback(encoded) { return nil } @@ -1012,6 +1042,9 @@ func (r *RedisDB) recordIfFallback(encoded string, userKey []byte) error { // file is removed so dumps without any non-reversible keys carry no // spurious empty file (matches the s3 encoder's keymap policy). func (r *RedisDB) closeKeymap() error { + if r.countOnly { + return nil + } if r.keymap == nil { return nil } diff --git a/internal/backup/s3.go b/internal/backup/s3.go index 56ccca429..cb8c236d1 100644 --- a/internal/backup/s3.go +++ b/internal/backup/s3.go @@ -139,6 +139,7 @@ type S3Encoder struct { includeIncompleteUploads bool includeOrphans bool renameCollisions bool + countOnly bool buckets map[string]*s3BucketState warn func(event string, fields ...any) @@ -336,6 +337,11 @@ func (s *S3Encoder) WithRenameCollisions(on bool) *S3Encoder { return s } +func (s *S3Encoder) WithCountOnly(on bool) *S3Encoder { + s.countOnly = on + return s +} + // WithWarnSink wires a structured warning sink. func (s *S3Encoder) WithWarnSink(fn func(event string, fields ...any)) *S3Encoder { s.warn = fn @@ -436,6 +442,11 @@ func (s *S3Encoder) HandleBlob(key, value []byte) error { if err != nil { return err } + st.chunkPaths = ensureChunkPaths(st.chunkPaths) + if s.countOnly { + st.chunkPaths[s3ChunkKey{uploadID: uploadID, partNo: partNo, chunkNo: chunkNo, partVersion: partVersion}] = "" + return nil + } if !st.scratchDirCreated { if err := os.MkdirAll(dir, 0o755); err != nil { //nolint:mnd // 0755 == standard dir mode return errors.WithStack(err) @@ -446,7 +457,6 @@ func (s *S3Encoder) HandleBlob(key, value []byte) error { if err := writeFileAtomic(path, value); err != nil { return err } - st.chunkPaths = ensureChunkPaths(st.chunkPaths) st.chunkPaths[s3ChunkKey{uploadID: uploadID, partNo: partNo, chunkNo: chunkNo, partVersion: partVersion}] = path return nil } @@ -489,6 +499,9 @@ func (s *S3Encoder) HandleIncompleteUpload(prefix string, key, value []byte) err if !ok { return errors.Wrapf(ErrS3MalformedKey, "upload-family key: %q", key) } + if s.countOnly { + return nil + } // Reject dot-segment / empty bucket names BEFORE the // filesystem join — same fix as flushBucket (round 6) but for // the --include-incomplete-uploads code path, which runs From db7ec7d0ce9143ff9211e1f8e0e4a46066732324 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 23:36:48 +0900 Subject: [PATCH 6/6] backup: refine live baseline counting --- adapter/admin_backup.go | 228 ++++++++++++++++++++------ adapter/admin_backup_test.go | 67 ++++++++ internal/backup/decode.go | 2 + internal/backup/dynamodb.go | 25 ++- internal/backup/live.go | 87 +++++++++- internal/backup/live_producer.go | 39 +++-- internal/backup/live_producer_test.go | 21 +++ internal/backup/sqs.go | 81 ++++++--- proto/admin.pb.go | 106 +++++++----- proto/admin.proto | 7 + 10 files changed, 531 insertions(+), 132 deletions(-) diff --git a/adapter/admin_backup.go b/adapter/admin_backup.go index 298f61e94..6599e4c69 100644 --- a/adapter/admin_backup.go +++ b/adapter/admin_backup.go @@ -72,6 +72,11 @@ type filteredBackupScanStore interface { ) kv.BackupScanner } +type backupBaselineSelection struct { + adapters logicalbackup.AdapterSet + scopes map[logicalbackup.Scope]bool +} + type BackupPinLimiter interface { PinWithDeadline(pinID kv.BackupPinID, readTS uint64, deadline time.Time) error ReleaseBackupPin(pinID kv.BackupPinID) @@ -218,6 +223,10 @@ func (s *AdminServer) BeginBackup(ctx context.Context, req *pb.BeginBackupReques if err != nil { return nil, err } + selection, err := backupBaselineSelectionFromBeginRequest(req) + if err != nil { + return nil, err + } // Serialize BeginBackup on one admin endpoint so local preflight capacity // and proposal compensation cannot interleave. @@ -230,7 +239,7 @@ func (s *AdminServer) BeginBackup(ctx context.Context, req *pb.BeginBackupReques if err != nil { return nil, err } - counts, appliedAtCount, err := s.buildExpectedBackupBaseline(ctx, prepared) + counts, appliedAtCount, err := s.buildExpectedBackupBaseline(ctx, prepared, selection) if err != nil { s.compensateBackupRelease(prepared.controlGroup, prepared.groups, prepared.pinID) return nil, err @@ -359,6 +368,7 @@ func (s *AdminServer) pinBackupGroups( func (s *AdminServer) buildExpectedBackupBaseline( ctx context.Context, prepared preparedBackup, + selection backupBaselineSelection, ) (map[logicalbackup.Scope]uint64, uint64, error) { stopRenew := make(chan struct{}) renewDone := make(chan error, 1) @@ -376,7 +386,9 @@ func (s *AdminServer) buildExpectedBackupBaseline( if validateErr != nil { scanErr = errors.Wrap(validateErr, "validate backup transaction locks") } else { - counts, appliedAtCount, scanErr = s.scanBackupScopeCounts(ctx, prepared.routes, prepared.readTS, prepared.groups, nil) + counts, appliedAtCount, scanErr = s.scanBackupScopeCounts( + ctx, prepared.routes, prepared.readTS, prepared.groups, selection, nil, + ) } close(stopRenew) renewErr := <-renewDone @@ -520,9 +532,13 @@ func (s *AdminServer) ListAdaptersAndScopes( if err != nil { return nil, err } - counts, _, err := s.scanBackupScopeCounts(ctx, routes, tok.readTS, groups, func() error { - return s.requireLiveBackupSession(tok) - }) + counts, _, err := s.scanBackupScopeCounts( + ctx, routes, tok.readTS, groups, + backupBaselineSelection{adapters: logicalbackup.AllAdapters()}, + func() error { + return s.requireLiveBackupSession(tok) + }, + ) if err != nil { return nil, status.Errorf(codes.FailedPrecondition, "list backup scopes: %v", err) } @@ -545,9 +561,6 @@ func (s *AdminServer) StreamBackup( if err != nil { return err } - if err := s.requireLiveBackupSession(tok); err != nil { - return err - } if _, err := s.backupGroupsForToken(tok); err != nil { return err } @@ -555,6 +568,9 @@ func (s *AdminServer) StreamBackup( if err != nil { return err } + if err := s.requireLiveBackupSession(tok); err != nil { + return err + } selected, err := selectedBackupScopes(req.GetScopes()) if err != nil { return err @@ -679,6 +695,69 @@ func selectedBackupScopes(scopes []*pb.BackupScope) (map[logicalbackup.Scope]boo return selected, nil } +func backupBaselineSelectionFromBeginRequest(req *pb.BeginBackupRequest) (backupBaselineSelection, error) { + adapters, err := backupAdaptersFromNames(req.GetAdapters()) + if err != nil { + return backupBaselineSelection{}, err + } + scopes, err := selectedBackupScopes(req.GetScopes()) + if err != nil { + return backupBaselineSelection{}, err + } + for scope := range scopes { + if !logicalbackup.AdapterEnabled(adapters, scope.Adapter) { + return backupBaselineSelection{}, status.Errorf(codes.InvalidArgument, "%s is excluded by adapters", scope.Adapter) + } + } + return backupBaselineSelection{adapters: adapters, scopes: scopes}, nil +} + +func backupAdaptersFromNames(names []string) (logicalbackup.AdapterSet, error) { + if len(names) == 0 { + return logicalbackup.AllAdapters(), nil + } + var out logicalbackup.AdapterSet + for _, name := range names { + switch name { + case "dynamodb": + out.DynamoDB = true + case "s3": + out.S3 = true + case "redis": + out.Redis = true + case "sqs": + out.SQS = true + default: + return logicalbackup.AdapterSet{}, status.Errorf(codes.InvalidArgument, "unknown backup adapter %q", name) + } + } + if out == (logicalbackup.AdapterSet{}) { + return logicalbackup.AdapterSet{}, status.Errorf(codes.InvalidArgument, "%s", "backup requires at least one adapter") + } + return out, nil +} + +func backupBaselineKeySelected(key []byte, selection backupBaselineSelection) (bool, error) { + adapter, ok := logicalbackup.AdapterForKey(key) + if !ok { + return false, nil + } + if !logicalbackup.AdapterEnabled(selection.adapters, adapter) { + return false, nil + } + scope, scoped, err := logicalbackup.ScopeForKey(key) + if err != nil { + return false, errors.Wrap(err, "classify backup baseline key") + } + if !scoped { + return false, nil + } + if len(selection.scopes) > 0 && !selection.scopes[scope] { + return false, nil + } + return true, nil +} + func (s *AdminServer) requireBackupControl() error { if s == nil || s.backupStore == nil || s.backupReadFence == nil || s.backupPeerProbe == nil || s.backupLimiter == nil || s.backupTokenKey == ([32]byte{}) { return status.Errorf(codes.Unavailable, "%s", ErrBackupUnavailable) @@ -1057,37 +1136,22 @@ func (s *AdminServer) scanBackupScopeCounts( routes kv.BackupRouteSnapshot, readTS uint64, groups []backupGroup, + selection backupBaselineSelection, requireLive func() error, ) (counts map[logicalbackup.Scope]uint64, applied uint64, retErr error) { - scanner := s.backupStore.NewBackupScannerAtSnapshot(routes, readTS, s.backupConfig.scanPageSize) - if scanner == nil { - return nil, 0, errors.New("backup scanner is nil") + if selection.adapters == (logicalbackup.AdapterSet{}) { + selection.adapters = logicalbackup.AllAdapters() } - counter, err := logicalbackup.NewLiveScopeCounter(logicalbackup.AllAdapters()) + counter, err := logicalbackup.NewLiveScopeCounter(selection.adapters) if err != nil { - _ = scanner.Close() return nil, 0, errors.Wrap(err, "create retained backup baseline counter") } - defer func() { - retErr = finishBackupScan(ctx, scanner, retErr) - }() - for { - if err := requireBackupScopeScanLive(requireLive); err != nil { - return nil, 0, err - } - pair, ok, err := scanner.Next(ctx) - if err != nil { - return nil, 0, errors.Wrap(err, "scan backup baseline") - } - if !ok { - break - } - if pair == nil { - return nil, 0, errors.New("backup scanner returned a nil record") - } - if err := counter.Add(pair.Key, pair.Value); err != nil { - return nil, 0, errors.Wrap(err, "count retained backup baseline") - } + metadataKeys, err := s.scanBackupScopeKeys(ctx, routes, readTS, selection, counter, requireLive) + if err != nil { + return nil, 0, err + } + if err := s.scanBackupScopeMetadata(ctx, routes, readTS, metadataKeys, counter, requireLive); err != nil { + return nil, 0, err } if err := requireBackupScopeScanLive(requireLive); err != nil { return nil, 0, err @@ -1103,39 +1167,91 @@ func (s *AdminServer) scanBackupScopeCounts( return counts, applied, nil } -func collectBackupScopeCounts( +func (s *AdminServer) scanBackupScopeKeys( ctx context.Context, - scanner kv.BackupKeyScanner, + routes kv.BackupRouteSnapshot, + readTS uint64, + selection backupBaselineSelection, + counter *logicalbackup.LiveScopeCounter, requireLive func() error, -) (map[logicalbackup.Scope]uint64, error) { - counts := make(map[logicalbackup.Scope]uint64) +) (metadataKeys map[string]struct{}, retErr error) { + scanner := s.backupStore.NewBackupKeyScannerAtSnapshot(routes, readTS, s.backupConfig.scanPageSize) + if scanner == nil { + return nil, errors.New("backup key scanner is nil") + } + defer func() { + retErr = finishBackupScan(ctx, scanner, retErr) + }() + metadataKeys = make(map[string]struct{}) for { if err := requireBackupScopeScanLive(requireLive); err != nil { return nil, err } key, ok, err := scanner.Next(ctx) if err != nil { - return nil, errors.Wrap(err, "scan backup key baseline") + return nil, errors.Wrap(err, "scan backup baseline") } if !ok { break } - if err := countBackupScopeKey(counts, key); err != nil { - return nil, err + selected, err := backupBaselineKeySelected(key, selection) + if err != nil { + return nil, errors.Wrap(err, "select backup baseline key") + } + if !selected { + continue + } + needsValue, err := counter.AddKey(key) + if err != nil { + return nil, errors.Wrap(err, "count retained backup baseline key") + } + if needsValue { + metadataKeys[string(key)] = struct{}{} } } - return counts, requireBackupScopeScanLive(requireLive) + return metadataKeys, requireBackupScopeScanLive(requireLive) } -func countBackupScopeKey(counts map[logicalbackup.Scope]uint64, key []byte) error { - scope, scoped, err := logicalbackup.ScopeForKey(key) - if err != nil { - return errors.Wrap(err, "classify backup key baseline") +func (s *AdminServer) scanBackupScopeMetadata( + ctx context.Context, + routes kv.BackupRouteSnapshot, + readTS uint64, + metadataKeys map[string]struct{}, + counter *logicalbackup.LiveScopeCounter, + requireLive func() error, +) (retErr error) { + if len(metadataKeys) == 0 { + return requireBackupScopeScanLive(requireLive) } - if scoped { - counts[scope]++ + scanner := s.newBackupBaselineMetadataScanner(routes, readTS, metadataKeys) + if scanner == nil { + return errors.New("backup metadata scanner is nil") } - return nil + defer func() { + retErr = finishBackupScan(ctx, scanner, retErr) + }() + for { + if err := requireBackupScopeScanLive(requireLive); err != nil { + return err + } + pair, ok, err := scanner.Next(ctx) + if err != nil { + return errors.Wrap(err, "scan backup baseline metadata") + } + if !ok { + break + } + if pair == nil { + return errors.New("backup metadata scanner returned a nil record") + } + if _, ok := metadataKeys[string(pair.Key)]; !ok { + continue + } + if err := counter.AddValue(pair.Key, pair.Value); err != nil { + return errors.Wrap(err, "count retained backup baseline metadata") + } + } + return requireBackupScopeScanLive(requireLive) } func requireBackupScopeScanLive(requireLive func() error) error { @@ -1145,6 +1261,20 @@ func requireBackupScopeScanLive(requireLive func() error) error { return requireLive() } +func (s *AdminServer) newBackupBaselineMetadataScanner( + routes kv.BackupRouteSnapshot, + readTS uint64, + metadataKeys map[string]struct{}, +) kv.BackupScanner { + if filtered, ok := s.backupStore.(filteredBackupScanStore); ok { + return filtered.NewFilteredBackupScannerAtSnapshot(routes, readTS, s.backupConfig.scanPageSize, func(key []byte) (bool, error) { + _, ok := metadataKeys[string(key)] + return ok, nil + }) + } + return s.backupStore.NewBackupScannerAtSnapshot(routes, readTS, s.backupConfig.scanPageSize) +} + func (s *AdminServer) rememberBackupSession(tok backupToken, routes kv.BackupRouteSnapshot) { now := s.nowSnapshot() s.backupStateMu.Lock() @@ -1174,7 +1304,7 @@ func (s *AdminServer) requireLiveBackupSession(tok backupToken) error { defer s.backupStateMu.Unlock() session, ok := s.backupSessions[tok.pinID] if !ok || session.readTS != tok.readTS { - return status.Errorf(codes.FailedPrecondition, "%s", "backup route snapshot is unavailable on this endpoint") + return status.Errorf(codes.FailedPrecondition, "%s", "backup pin token has expired") } if session.deadline.IsZero() || !now.Before(session.deadline) { delete(s.backupSessions, tok.pinID) diff --git a/adapter/admin_backup_test.go b/adapter/admin_backup_test.go index f0ab5bcff..0a72094c4 100644 --- a/adapter/admin_backup_test.go +++ b/adapter/admin_backup_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/base64" + "encoding/json" stderrors "errors" "sync" "sync/atomic" @@ -12,6 +13,7 @@ import ( logicalbackup "github.com/bootjp/elastickv/internal/backup" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/s3keys" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" kvstore "github.com/bootjp/elastickv/store" @@ -230,6 +232,13 @@ func backupTestDefaultValueForKey(key []byte) []byte { return []byte("value") } +func backupTestJSON(t *testing.T, v any) []byte { + t.Helper() + out, err := json.Marshal(v) + require.NoError(t, err) + return out +} + type backupSliceScanner struct { keys [][]byte index int @@ -443,6 +452,64 @@ func TestBeginBackupExpectedKeysUseRetainedCounts(t *testing.T) { require.Equal(t, "dynamodb", begin.GetExpectedKeys()[0].GetAdapter()) require.Equal(t, table, begin.GetExpectedKeys()[0].GetScope()) require.EqualValues(t, 2, begin.GetExpectedKeys()[0].GetKeyCount()) + require.Equal(t, [][]byte{schemaKey}, store.valueKeys) +} + +func TestBeginBackupExpectedKeysAvoidMaterializingBlobValues(t *testing.T) { + t.Parallel() + const ( + bucket = "photos" + object = "large.bin" + uploadID = "upload-1" + ) + bucketKey := s3keys.BucketMetaKey(bucket) + manifestKey := s3keys.ObjectManifestKey(bucket, 1, object) + blobKey := s3keys.BlobKey(bucket, 1, object, uploadID, 1, 0) + store := &backupTestStore{ + keys: [][]byte{bucketKey, manifestKey, blobKey}, + values: map[string][]byte{ + string(bucketKey): backupTestJSON(t, map[string]any{ + "bucket_name": bucket, + "generation": 1, + }), + string(manifestKey): backupTestJSON(t, map[string]any{ + "upload_id": uploadID, + "parts": []map[string]any{{ + "part_no": 1, + "chunk_count": 1, + }}, + }), + string(blobKey): []byte("large-blob-body"), + }, + } + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{Adapters: []string{"s3"}}) + require.NoError(t, err) + require.Len(t, begin.GetExpectedKeys(), 1) + require.Equal(t, "s3", begin.GetExpectedKeys()[0].GetAdapter()) + require.Equal(t, bucket, begin.GetExpectedKeys()[0].GetScope()) + require.EqualValues(t, 3, begin.GetExpectedKeys()[0].GetKeyCount()) + require.Equal(t, [][]byte{bucketKey, manifestKey}, store.valueKeys) +} + +func TestBeginBackupBaselineSkipsUnselectedAdapterValues(t *testing.T) { + t.Parallel() + schemaKey := logicalbackup.EncodeDDBTableMetaKey("orders") + store := &backupTestStore{ + keys: [][]byte{schemaKey}, + values: map[string][]byte{string(schemaKey): []byte("future-format")}, + } + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{Adapters: []string{"redis"}}) + require.NoError(t, err) + require.Empty(t, begin.GetExpectedKeys()) + require.Empty(t, store.valueKeys) } func TestSnapshotBackupGroupsExcludesReservedTSOGroup(t *testing.T) { diff --git a/internal/backup/decode.go b/internal/backup/decode.go index f709f6f8a..badfe629e 100644 --- a/internal/backup/decode.go +++ b/internal/backup/decode.go @@ -201,6 +201,7 @@ func newDispatcher(opts DecodeOptions) (*dispatcher, error) { d.ddb = NewDDBEncoder(opts.OutRoot). WithBundleJSONL(opts.DynamoDBBundleJSONL). WithBundleSizeBytes(opts.DynamoDBBundleSizeBytes). + WithCountOnly(opts.countOnly). WithWarnSink(opts.WarnSink) } if opts.Adapters.S3 { @@ -219,6 +220,7 @@ func newDispatcher(opts DecodeOptions) (*dispatcher, error) { d.sqs = NewSQSEncoder(opts.OutRoot). WithIncludeSideRecords(opts.IncludeSQSSideRecords). WithPreserveVisibility(opts.PreserveSQSVisibility). + WithCountOnly(opts.countOnly). WithWarnSink(opts.WarnSink) } return d, nil diff --git a/internal/backup/dynamodb.go b/internal/backup/dynamodb.go index 93d0f96dc..0aeb8538e 100644 --- a/internal/backup/dynamodb.go +++ b/internal/backup/dynamodb.go @@ -65,6 +65,7 @@ type DDBEncoder struct { outRoot string bundleJSONL bool bundleSize int64 + countOnly bool tables map[string]*ddbTableState @@ -76,6 +77,7 @@ type ddbTableState struct { name string schema *pb.DynamoTableSchema itemsByGen map[uint64][]*pb.DynamoItem + itemCounts map[uint64]uint64 } func ensureItemsByGen(m map[uint64][]*pb.DynamoItem) map[uint64][]*pb.DynamoItem { @@ -112,6 +114,11 @@ func (d *DDBEncoder) WithBundleSizeBytes(size int64) *DDBEncoder { return d } +func (d *DDBEncoder) WithCountOnly(on bool) *DDBEncoder { + d.countOnly = on + return d +} + // WithWarnSink wires structured-warning emission (orphan items, // schema-less tables, etc.). func (d *DDBEncoder) WithWarnSink(fn func(event string, fields ...any)) *DDBEncoder { @@ -166,6 +173,14 @@ func (d *DDBEncoder) HandleItem(key, value []byte) error { if err != nil { return err } + if d.countOnly { + st := d.tableState(encoded) + if st.itemCounts == nil { + st.itemCounts = make(map[uint64]uint64) + } + st.itemCounts[generation]++ + return nil + } if !bytes.HasPrefix(value, storedDDBItemMagic) { return errors.Wrap(ErrDDBInvalidItem, "missing magic prefix") } @@ -217,7 +232,7 @@ func (d *DDBEncoder) RetainedRecordCounts() map[string]uint64 { for _, st := range d.tables { if st.schema != nil { emitOrder := ddbGenerationEmitOrder(st.schema.GetGeneration(), st.schema.GetMigratingFromGeneration()) - out[st.name] = 1 + retainedDDBItemRecords(st.itemsByGen, emitOrder) + out[st.name] = 1 + retainedDDBItemRecords(st, emitOrder) } } return out @@ -266,10 +281,14 @@ func (d *DDBEncoder) flushTable(st *ddbTableState) error { return nil } -func retainedDDBItemRecords(itemsByGen map[uint64][]*pb.DynamoItem, emitOrder []uint64) uint64 { +func retainedDDBItemRecords(st *ddbTableState, emitOrder []uint64) uint64 { var count uint64 for _, generation := range emitOrder { - count += uint64(len(itemsByGen[generation])) + if st.itemCounts != nil { + count += st.itemCounts[generation] + continue + } + count += uint64(len(st.itemsByGen[generation])) } return count } diff --git a/internal/backup/live.go b/internal/backup/live.go index 6f2a0c070..dfca3f943 100644 --- a/internal/backup/live.go +++ b/internal/backup/live.go @@ -59,6 +59,46 @@ func ScopeForKey(key []byte) (Scope, bool, error) { } } +// AdapterForKey reports the backup adapter family for keys that belong to a +// known adapter namespace, including malformed scoped keys. Callers can use it +// to skip disabled adapters before running stricter scope parsing. +func AdapterForKey(key []byte) (string, bool) { + switch { + case hasAnyBackupPrefix(key, DDBTableMetaPrefix, DDBTableGenPrefix, DDBItemPrefix, DDBGSIPrefix): + return adapterDynamoDB, true + case hasAnyBackupPrefix(key, + S3BucketMetaPrefix, S3BucketGenPrefix, S3ObjectManifestPrefix, + S3UploadMetaPrefix, S3UploadPartPrefix, S3BlobPrefix, S3GCUploadPrefix, S3RoutePrefix, + ): + return adapterS3, true + case hasAnyBackupPrefix(key, + SQSQueueMetaPrefix, SQSQueueGenPrefix, SQSMsgDataPrefix, SQSQueueSeqPrefix, + SQSQueueTombstonePrefix, SQSMsgVisPrefix, SQSMsgByAgePrefix, SQSMsgDedupPrefix, SQSMsgGroupPrefix, + ): + return adapterSQS, true + case isRedisBackupKey(key): + return adapterRedis, true + default: + return "", false + } +} + +// AdapterEnabled reports whether name is enabled in set. +func AdapterEnabled(set AdapterSet, name string) bool { + switch name { + case adapterDynamoDB: + return set.DynamoDB + case adapterS3: + return set.S3 + case adapterRedis: + return set.Redis + case adapterSQS: + return set.SQS + default: + return false + } +} + func scopeForDDBKey(key []byte) (Scope, bool, error) { switch { case bytes.HasPrefix(key, []byte(DDBTableMetaPrefix)): @@ -262,15 +302,40 @@ func NewLiveScopeCounter(adapters AdapterSet) (*LiveScopeCounter, error) { } func (c *LiveScopeCounter) Add(key, value []byte) error { + needsValue, err := c.AddKey(key) + if err != nil { + return err + } + if !needsValue { + return nil + } + return c.AddValue(key, value) +} + +// AddKey records a key observed by a count-only baseline scan. It returns true +// when the caller must also provide the value through AddValue for metadata +// needed by retained-count finalization. +func (c *LiveScopeCounter) AddKey(key []byte) (bool, error) { if c == nil || c.d == nil { - return errors.Wrap(ErrDecodeOptionsInvalid, "live scope counter is unavailable") + return false, errors.Wrap(ErrDecodeOptionsInvalid, "live scope counter is unavailable") } scope, scoped, err := ScopeForKey(key) if err != nil { - return err + return false, err + } + if !scoped { + return false, nil + } + c.streamed[scope]++ + if liveScopeCounterNeedsValue(key) { + return true, nil } - if scoped { - c.streamed[scope]++ + return false, c.d.route(key, nil) +} + +func (c *LiveScopeCounter) AddValue(key, value []byte) error { + if c == nil || c.d == nil { + return errors.Wrap(ErrDecodeOptionsInvalid, "live scope counter is unavailable") } return c.d.route(key, value) } @@ -282,6 +347,20 @@ func (c *LiveScopeCounter) RetainedCounts() (map[Scope]uint64, error) { return finalizedScopeCounts(c.d, c.streamed), nil } +func liveScopeCounterNeedsValue(key []byte) bool { + switch { + case hasAnyBackupPrefix(key, DDBTableMetaPrefix): + return true + case hasAnyBackupPrefix(key, S3BucketMetaPrefix, S3ObjectManifestPrefix): + return true + case hasAnyBackupPrefix(key, SQSQueueMetaPrefix, SQSQueueGenPrefix): + return true + default: + adapter, ok := AdapterForKey(key) + return ok && adapter == adapterRedis + } +} + func finalizedScopeCounts(d *dispatcher, streamed map[Scope]uint64) map[Scope]uint64 { out := make(map[Scope]uint64, len(streamed)) for scope, count := range streamed { diff --git a/internal/backup/live_producer.go b/internal/backup/live_producer.go index e26fe814c..ca388df3a 100644 --- a/internal/backup/live_producer.go +++ b/internal/backup/live_producer.go @@ -160,7 +160,11 @@ func publishCompletedLiveBackup( func beginLiveBackup(ctx context.Context, rpc LiveBackupRPC, opts LiveBackupOptions) (*pb.BeginBackupResponse, error) { timeout := effectiveTimeout(opts.BeginTimeout, defaultLiveBackupBeginTimeout) beginCtx, cancel := context.WithTimeout(ctx, timeout) - begin, err := rpc.BeginBackup(beginCtx, &pb.BeginBackupRequest{TtlMs: durationMillis(opts.TTL)}) + begin, err := rpc.BeginBackup(beginCtx, &pb.BeginBackupRequest{ + TtlMs: durationMillis(opts.TTL), + Adapters: liveBackupAdapterNames(opts.Adapters), + Scopes: protoScopes(scopeSet(opts.Scopes)), + }) cancel() if err != nil { return nil, errors.Wrap(err, "begin live backup") @@ -459,18 +463,24 @@ func validateLiveScope(scope Scope) error { } func adapterEnabled(set AdapterSet, name string) bool { - switch name { - case adapterDynamoDB: - return set.DynamoDB - case adapterS3: - return set.S3 - case adapterRedis: - return set.Redis - case adapterSQS: - return set.SQS - default: - return false + return AdapterEnabled(set, name) +} + +func liveBackupAdapterNames(set AdapterSet) []string { + names := make([]string, 0, 4) //nolint:mnd // four logical backup adapters. + if set.DynamoDB { + names = append(names, adapterDynamoDB) + } + if set.S3 { + names = append(names, adapterS3) + } + if set.Redis { + names = append(names, adapterRedis) + } + if set.SQS { + names = append(names, adapterSQS) } + return names } func protoScopes(scopes map[Scope]struct{}) []*pb.BackupScope { @@ -574,7 +584,7 @@ func liveBackupManifest(begin *pb.BeginBackupResponse, token []byte, selected ma } } sort.Slice(m.Shards, func(i, j int) bool { return m.Shards[i].RaftGroupID < m.Shards[j].RaftGroupID }) - m.Adapters = liveManifestAdapters(opts.Adapters, selected) + m.Adapters = liveManifestAdapters(opts.Adapters, selected, len(opts.Scopes) == 0) m.Exclusions = &Exclusions{ IncludeIncompleteUploads: opts.IncludeIncompleteUploads, IncludeOrphans: opts.IncludeOrphans, @@ -588,9 +598,8 @@ func liveBackupManifest(begin *pb.BeginBackupResponse, token []byte, selected ma return m } -func liveManifestAdapters(enabled AdapterSet, selected map[Scope]struct{}) *Adapters { +func liveManifestAdapters(enabled AdapterSet, selected map[Scope]struct{}, includeEnabledEmpty bool) *Adapters { out := &Adapters{} - includeEnabledEmpty := len(selected) == 0 if includeEnabledEmpty && enabled.DynamoDB { out.DynamoDB = &Adapter{} } diff --git a/internal/backup/live_producer_test.go b/internal/backup/live_producer_test.go index 4a8b61078..c2f496b8b 100644 --- a/internal/backup/live_producer_test.go +++ b/internal/backup/live_producer_test.go @@ -406,6 +406,27 @@ func TestRunLiveBackupScopedManifestOmitsUnselectedAdapters(t *testing.T) { } } +func TestRunLiveBackupDefaultManifestPreservesEnabledEmptyAdapters(t *testing.T) { + t.Parallel() + rpc := successfulLiveBackupRPC() + root := filepath.Join(t.TempDir(), "dump") + _, err := RunLiveBackup(context.Background(), rpc, LiveBackupOptions{ + OutputRoot: root, + Adapters: AllAdapters(), + TTL: time.Minute, + }) + if err != nil { + t.Fatalf("RunLiveBackup: %v", err) + } + manifest := readLiveBackupManifest(t, root) + if manifest.Adapters.Redis == nil || len(manifest.Adapters.Redis.Databases) != 1 { + t.Fatalf("manifest redis adapters=%+v", manifest.Adapters) + } + if manifest.Adapters.DynamoDB == nil || manifest.Adapters.S3 == nil || manifest.Adapters.SQS == nil { + t.Fatalf("manifest omitted enabled empty adapters=%+v", manifest.Adapters) + } +} + func TestCrossAdapterConsistency(t *testing.T) { t.Parallel() rpc, body := crossAdapterLiveBackupRPC(t) diff --git a/internal/backup/sqs.go b/internal/backup/sqs.go index 87b7debf9..e44769055 100644 --- a/internal/backup/sqs.go +++ b/internal/backup/sqs.go @@ -94,6 +94,7 @@ type SQSEncoder struct { outRoot string includeSideRecords bool preserveVisibility bool + countOnly bool // queues is keyed by the base64url-encoded queue name (the on-disk // segment in the !sqs|queue|meta| key). Pending messages are @@ -122,6 +123,9 @@ type sqsQueueState struct { // already drops messages). Codex P1 round 10. activeGen uint64 genSeen bool + // messageCounts is populated only in count-only mode, where baseline + // scans must avoid materializing message bodies. + messageCounts map[uint64]uint64 // internalBuf accumulates side records in their on-disk shape if // includeSideRecords is on. Each line is the encoded prefix + // hex(rest-of-key) + value (b64) — implementation-grade detail @@ -284,6 +288,11 @@ func (s *SQSEncoder) WithPreserveVisibility(on bool) *SQSEncoder { return s } +func (s *SQSEncoder) WithCountOnly(on bool) *SQSEncoder { + s.countOnly = on + return s +} + // WithWarnSink wires a structured warning hook (same shape as // RedisDB.WithWarnSink). Used for orphan messages and unresolvable side // records. @@ -348,10 +357,18 @@ func (s *SQSEncoder) HandleQueueGen(key, value []byte) error { // the per-queue routing key; the message is buffered until Finalize so it // can be sorted and emitted in send-order. func (s *SQSEncoder) HandleMessageData(key, value []byte) error { - encQueue, partition, isPartitioned, err := parseSQSMessageDataKey(key) + encQueue, partition, isPartitioned, generation, err := parseSQSMessageDataKeyWithGeneration(key) if err != nil { return err } + if s.countOnly { + st := s.queueState(encQueue) + if st.messageCounts == nil { + st.messageCounts = make(map[uint64]uint64) + } + st.messageCounts[generation]++ + return nil + } rec, err := decodeSQSMessageValue(value) if err != nil { return err @@ -454,12 +471,26 @@ func (s *SQSEncoder) RetainedRecordCounts() map[string]uint64 { if st.genSeen { count++ } - visible, _ := filterStaleGenMessages(st.messages, st.activeGen) - out[st.name] = count + uint64(len(visible)) + out[st.name] = count + retainedSQSMessageCount(st) } return out } +func retainedSQSMessageCount(st *sqsQueueState) uint64 { + if st.messageCounts != nil { + if st.activeGen == 0 { + var total uint64 + for _, count := range st.messageCounts { + total += count + } + return total + } + return st.messageCounts[st.activeGen] + } + visible, _ := filterStaleGenMessages(st.messages, st.activeGen) + return uint64(len(visible)) +} + // flushOrphanQueueSideRecords emits buffered side records for a queue // whose !sqs|queue|meta row never arrived. Without this branch, // --include-sqs-side-records silently drops the post-DeleteQueue @@ -586,16 +617,23 @@ func stripPrefixSegment(key, prefix []byte) (string, error) { // Boundary detection (partitioned): the queue segment is terminated by // a literal '|' before the fixed-width partition u32. Codex P1 round 9. func parseSQSMessageDataKey(key []byte) (encQueue string, partition uint32, isPartitioned bool, err error) { + encQueue, partition, isPartitioned, _, err = parseSQSMessageDataKeyWithGeneration(key) + return encQueue, partition, isPartitioned, err +} + +func parseSQSMessageDataKeyWithGeneration( + key []byte, +) (encQueue string, partition uint32, isPartitioned bool, generation uint64, err error) { rest, err := stripPrefixSegment(key, []byte(SQSMsgDataPrefix)) if err != nil { - return "", 0, false, err + return "", 0, false, 0, err } if isPartitionedRest(rest) { - enc, part, perr := parseSQSPartitionedQueueAndTrailer(rest, true /*hasMsgID*/, key) + enc, part, gen, perr := parseSQSPartitionedQueueAndTrailer(rest, true /*hasMsgID*/, key) if perr != nil { - return "", 0, false, perr + return "", 0, false, 0, perr } - return enc, part, true, nil + return enc, part, true, gen, nil } idx := scanBase64URLBoundary(rest) // idx == 0 -> no queue segment; idx+genBytes >= len(rest) -> no @@ -603,21 +641,22 @@ func parseSQSMessageDataKey(key []byte) (encQueue string, partition uint32, isPa // AWS SQS message IDs are non-empty by construction, so an empty // msg-id segment can never be a legitimate snapshot record. if idx == 0 || idx+genBytes >= len(rest) { - return "", 0, false, errors.Wrapf(ErrSQSMalformedKey, + return "", 0, false, 0, errors.Wrapf(ErrSQSMalformedKey, "queue segment or message-id segment not found in %q", key) } enc := rest[:idx] if _, err := base64.RawURLEncoding.DecodeString(enc); err != nil { - return "", 0, false, errors.Wrap(ErrSQSMalformedKey, err.Error()) + return "", 0, false, 0, errors.Wrap(ErrSQSMalformedKey, err.Error()) } // Validate the msg-id segment decodes too; if it doesn't, the // boundary detection got it wrong and we surface an error rather // than emit a record under a wrong queue. encMsgID := rest[idx+genBytes:] if _, err := base64.RawURLEncoding.DecodeString(encMsgID); err != nil { - return "", 0, false, errors.Wrap(ErrSQSMalformedKey, err.Error()) + return "", 0, false, 0, errors.Wrap(ErrSQSMalformedKey, err.Error()) } - return enc, 0, false, nil + gen := binary.BigEndian.Uint64([]byte(rest[idx : idx+genBytes])) + return enc, 0, false, gen, nil } // parseSQSGenericKey is a coarse parser for the side-record prefixes @@ -635,7 +674,7 @@ func parseSQSGenericKey(key []byte, prefix string) (string, error) { // Side-record dispatch routes by queue only — the partition // trailer is parsed for validation but the value is discarded // here. M5-3 design doc §"Decoder lift" pins this contract. - encQueue, _, perr := parseSQSPartitionedQueueAndTrailer(rest, false /*hasMsgID*/, key) + encQueue, _, _, perr := parseSQSPartitionedQueueAndTrailer(rest, false /*hasMsgID*/, key) if perr != nil { return "", perr } @@ -673,39 +712,41 @@ func isPartitionedRest(rest string) bool { // // Anything else surfaces ErrSQSMalformedKey rather than emitting // records under a wrong queue. -func parseSQSPartitionedQueueAndTrailer(rest string, hasMsgID bool, originalKey []byte) (string, uint32, error) { +func parseSQSPartitionedQueueAndTrailer(rest string, hasMsgID bool, originalKey []byte) (string, uint32, uint64, error) { body := rest[len(sqsPartitionedDiscriminator):] terminator := strings.IndexByte(body, '|') if terminator <= 0 { - return "", 0, errors.Wrapf(ErrSQSMalformedKey, + return "", 0, 0, errors.Wrapf(ErrSQSMalformedKey, "partitioned key missing queue terminator in %q", originalKey) } encQueue := body[:terminator] if _, err := base64.RawURLEncoding.DecodeString(encQueue); err != nil { - return "", 0, errors.Wrap(ErrSQSMalformedKey, err.Error()) + return "", 0, 0, errors.Wrap(ErrSQSMalformedKey, err.Error()) } trailer := body[terminator+1:] const fixedTrailerBytes = sqsPartitionBytes + genBytes if hasMsgID { // Need partition+gen plus at least 1 byte of msg-id. if len(trailer) <= fixedTrailerBytes { - return "", 0, errors.Wrapf(ErrSQSMalformedKey, + return "", 0, 0, errors.Wrapf(ErrSQSMalformedKey, "partitioned msg-data key missing message-id in %q", originalKey) } encMsgID := trailer[fixedTrailerBytes:] if _, err := base64.RawURLEncoding.DecodeString(encMsgID); err != nil { - return "", 0, errors.Wrap(ErrSQSMalformedKey, err.Error()) + return "", 0, 0, errors.Wrap(ErrSQSMalformedKey, err.Error()) } partition := binary.BigEndian.Uint32([]byte(trailer[:sqsPartitionBytes])) - return encQueue, partition, nil + gen := binary.BigEndian.Uint64([]byte(trailer[sqsPartitionBytes:fixedTrailerBytes])) + return encQueue, partition, gen, nil } // Side records: trailer must carry at least partition+gen. if len(trailer) < fixedTrailerBytes { - return "", 0, errors.Wrapf(ErrSQSMalformedKey, + return "", 0, 0, errors.Wrapf(ErrSQSMalformedKey, "partitioned side-record key trailer truncated in %q", originalKey) } partition := binary.BigEndian.Uint32([]byte(trailer[:sqsPartitionBytes])) - return encQueue, partition, nil + gen := binary.BigEndian.Uint64([]byte(trailer[sqsPartitionBytes:fixedTrailerBytes])) + return encQueue, partition, gen, nil } // scanBase64URLBoundary returns the index of the first byte in s that is diff --git a/proto/admin.pb.go b/proto/admin.pb.go index 73f04c0b1..d1dde7be3 100644 --- a/proto/admin.pb.go +++ b/proto/admin.pb.go @@ -1246,8 +1246,15 @@ func (x *BackupExpectedKeys) GetAppliedIndexAtCount() uint64 { } type BeginBackupRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - TtlMs uint64 `protobuf:"varint,1,opt,name=ttl_ms,json=ttlMs,proto3" json:"ttl_ms,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + TtlMs uint64 `protobuf:"varint,1,opt,name=ttl_ms,json=ttlMs,proto3" json:"ttl_ms,omitempty"` + // Empty means all adapters. Non-empty lets the server build the + // expected-key baseline only for adapters the producer will emit. + Adapters []string `protobuf:"bytes,2,rep,name=adapters,proto3" json:"adapters,omitempty"` + // Empty means every scope in the selected adapters. Non-empty narrows the + // expected-key baseline to the exact adapter/scope pairs requested by the + // producer. + Scopes []*BackupScope `protobuf:"bytes,3,rep,name=scopes,proto3" json:"scopes,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1289,6 +1296,20 @@ func (x *BeginBackupRequest) GetTtlMs() uint64 { return 0 } +func (x *BeginBackupRequest) GetAdapters() []string { + if x != nil { + return x.Adapters + } + return nil +} + +func (x *BeginBackupRequest) GetScopes() []*BackupScope { + if x != nil { + return x.Scopes + } + return nil +} + type BeginBackupResponse struct { state protoimpl.MessageState `protogen:"open.v1"` ReadTs uint64 `protobuf:"varint,1,opt,name=read_ts,json=readTs,proto3" json:"read_ts,omitempty"` @@ -2236,9 +2257,11 @@ const file_admin_proto_rawDesc = "" + "\aadapter\x18\x01 \x01(\tR\aadapter\x12\x14\n" + "\x05scope\x18\x02 \x01(\tR\x05scope\x12\x1b\n" + "\tkey_count\x18\x03 \x01(\x04R\bkeyCount\x123\n" + - "\x16applied_index_at_count\x18\x04 \x01(\x04R\x13appliedIndexAtCount\"+\n" + + "\x16applied_index_at_count\x18\x04 \x01(\x04R\x13appliedIndexAtCount\"m\n" + "\x12BeginBackupRequest\x12\x15\n" + - "\x06ttl_ms\x18\x01 \x01(\x04R\x05ttlMs\"\x91\x02\n" + + "\x06ttl_ms\x18\x01 \x01(\x04R\x05ttlMs\x12\x1a\n" + + "\badapters\x18\x02 \x03(\tR\badapters\x12$\n" + + "\x06scopes\x18\x03 \x03(\v2\f.BackupScopeR\x06scopes\"\x91\x02\n" + "\x13BeginBackupResponse\x12\x17\n" + "\aread_ts\x18\x01 \x01(\x04R\x06readTs\x12\x1b\n" + "\tpin_token\x18\x02 \x01(\fR\bpinToken\x12(\n" + @@ -2379,43 +2402,44 @@ var file_admin_proto_depIdxs = []int32{ 12, // 8: GetKeyVizMatrixResponse.rows:type_name -> KeyVizRow 12, // 9: GetRouteDetailResponse.row:type_name -> KeyVizRow 9, // 10: GetRouteDetailResponse.per_adapter:type_name -> AdapterSummary - 17, // 11: BeginBackupResponse.shards:type_name -> BackupShardApplied - 18, // 12: BeginBackupResponse.expected_keys:type_name -> BackupExpectedKeys - 26, // 13: ListAdaptersAndScopesResponse.scopes:type_name -> BackupScope - 26, // 14: StreamBackupRequest.scopes:type_name -> BackupScope - 34, // 15: StreamEventsEvent.route_transition:type_name -> RouteTransition - 35, // 16: StreamEventsEvent.keyviz_column:type_name -> KeyVizColumn - 0, // 17: KeyVizColumn.series:type_name -> KeyVizSeries - 12, // 18: KeyVizColumn.rows:type_name -> KeyVizRow - 4, // 19: Admin.GetClusterOverview:input_type -> GetClusterOverviewRequest - 7, // 20: Admin.GetRaftGroups:input_type -> GetRaftGroupsRequest - 10, // 21: Admin.GetAdapterSummary:input_type -> GetAdapterSummaryRequest - 13, // 22: Admin.GetKeyVizMatrix:input_type -> GetKeyVizMatrixRequest - 15, // 23: Admin.GetRouteDetail:input_type -> GetRouteDetailRequest - 19, // 24: Admin.BeginBackup:input_type -> BeginBackupRequest - 21, // 25: Admin.RenewBackup:input_type -> RenewBackupRequest - 23, // 26: Admin.EndBackup:input_type -> EndBackupRequest - 25, // 27: Admin.ListAdaptersAndScopes:input_type -> ListAdaptersAndScopesRequest - 28, // 28: Admin.StreamBackup:input_type -> StreamBackupRequest - 30, // 29: Admin.GetNodeVersion:input_type -> GetNodeVersionRequest - 32, // 30: Admin.StreamEvents:input_type -> StreamEventsRequest - 5, // 31: Admin.GetClusterOverview:output_type -> GetClusterOverviewResponse - 8, // 32: Admin.GetRaftGroups:output_type -> GetRaftGroupsResponse - 11, // 33: Admin.GetAdapterSummary:output_type -> GetAdapterSummaryResponse - 14, // 34: Admin.GetKeyVizMatrix:output_type -> GetKeyVizMatrixResponse - 16, // 35: Admin.GetRouteDetail:output_type -> GetRouteDetailResponse - 20, // 36: Admin.BeginBackup:output_type -> BeginBackupResponse - 22, // 37: Admin.RenewBackup:output_type -> RenewBackupResponse - 24, // 38: Admin.EndBackup:output_type -> EndBackupResponse - 27, // 39: Admin.ListAdaptersAndScopes:output_type -> ListAdaptersAndScopesResponse - 29, // 40: Admin.StreamBackup:output_type -> BackupKV - 31, // 41: Admin.GetNodeVersion:output_type -> GetNodeVersionResponse - 33, // 42: Admin.StreamEvents:output_type -> StreamEventsEvent - 31, // [31:43] is the sub-list for method output_type - 19, // [19:31] is the sub-list for method input_type - 19, // [19:19] is the sub-list for extension type_name - 19, // [19:19] is the sub-list for extension extendee - 0, // [0:19] is the sub-list for field type_name + 26, // 11: BeginBackupRequest.scopes:type_name -> BackupScope + 17, // 12: BeginBackupResponse.shards:type_name -> BackupShardApplied + 18, // 13: BeginBackupResponse.expected_keys:type_name -> BackupExpectedKeys + 26, // 14: ListAdaptersAndScopesResponse.scopes:type_name -> BackupScope + 26, // 15: StreamBackupRequest.scopes:type_name -> BackupScope + 34, // 16: StreamEventsEvent.route_transition:type_name -> RouteTransition + 35, // 17: StreamEventsEvent.keyviz_column:type_name -> KeyVizColumn + 0, // 18: KeyVizColumn.series:type_name -> KeyVizSeries + 12, // 19: KeyVizColumn.rows:type_name -> KeyVizRow + 4, // 20: Admin.GetClusterOverview:input_type -> GetClusterOverviewRequest + 7, // 21: Admin.GetRaftGroups:input_type -> GetRaftGroupsRequest + 10, // 22: Admin.GetAdapterSummary:input_type -> GetAdapterSummaryRequest + 13, // 23: Admin.GetKeyVizMatrix:input_type -> GetKeyVizMatrixRequest + 15, // 24: Admin.GetRouteDetail:input_type -> GetRouteDetailRequest + 19, // 25: Admin.BeginBackup:input_type -> BeginBackupRequest + 21, // 26: Admin.RenewBackup:input_type -> RenewBackupRequest + 23, // 27: Admin.EndBackup:input_type -> EndBackupRequest + 25, // 28: Admin.ListAdaptersAndScopes:input_type -> ListAdaptersAndScopesRequest + 28, // 29: Admin.StreamBackup:input_type -> StreamBackupRequest + 30, // 30: Admin.GetNodeVersion:input_type -> GetNodeVersionRequest + 32, // 31: Admin.StreamEvents:input_type -> StreamEventsRequest + 5, // 32: Admin.GetClusterOverview:output_type -> GetClusterOverviewResponse + 8, // 33: Admin.GetRaftGroups:output_type -> GetRaftGroupsResponse + 11, // 34: Admin.GetAdapterSummary:output_type -> GetAdapterSummaryResponse + 14, // 35: Admin.GetKeyVizMatrix:output_type -> GetKeyVizMatrixResponse + 16, // 36: Admin.GetRouteDetail:output_type -> GetRouteDetailResponse + 20, // 37: Admin.BeginBackup:output_type -> BeginBackupResponse + 22, // 38: Admin.RenewBackup:output_type -> RenewBackupResponse + 24, // 39: Admin.EndBackup:output_type -> EndBackupResponse + 27, // 40: Admin.ListAdaptersAndScopes:output_type -> ListAdaptersAndScopesResponse + 29, // 41: Admin.StreamBackup:output_type -> BackupKV + 31, // 42: Admin.GetNodeVersion:output_type -> GetNodeVersionResponse + 33, // 43: Admin.StreamEvents:output_type -> StreamEventsEvent + 32, // [32:44] is the sub-list for method output_type + 20, // [20:32] is the sub-list for method input_type + 20, // [20:20] is the sub-list for extension type_name + 20, // [20:20] is the sub-list for extension extendee + 0, // [0:20] is the sub-list for field type_name } func init() { file_admin_proto_init() } diff --git a/proto/admin.proto b/proto/admin.proto index d8fa1eee9..6ab674d10 100644 --- a/proto/admin.proto +++ b/proto/admin.proto @@ -178,6 +178,13 @@ message BackupExpectedKeys { message BeginBackupRequest { uint64 ttl_ms = 1; + // Empty means all adapters. Non-empty lets the server build the + // expected-key baseline only for adapters the producer will emit. + repeated string adapters = 2; + // Empty means every scope in the selected adapters. Non-empty narrows the + // expected-key baseline to the exact adapter/scope pairs requested by the + // producer. + repeated BackupScope scopes = 3; } message BeginBackupResponse {