test: Yiren/fkv mig int rebase v2

.github/workflows/integration-test.yml

@@ -187,6 +187,160 @@ jobs:
               "./integration_test/contracts/verify_flatkv_partial_loss_fails_loudly.sh",
             ],
           },
+          {
+            # FlatKV EVM migrate cluster coverage.
+            #
+            # The cluster boots with GIGA_MIGRATE_FROM_MEMIAVL=true so
+            # every validator starts in sc-write-mode = memiavl_only,
+            # i.e. v0 (FlatKV not yet allocated). This is the inverse of
+            # the FlatKV Integration row above (which boots in
+            # test_only_dual_write) and is what makes the migration
+            # script's pre-flip check meaningful: if the matrix env ever
+            # silently lands the cluster in any mode other than
+            # memiavl_only, the script will fail loudly at the pre-flip
+            # grep instead of "succeeding" with a no-op migration.
+            #
+            # Steps:
+            #   1  Deposit an EVM fixture while in v0 so the migration
+            #      has real account+code+storage to drain. The fixture
+            #      writes roughly 4000 storage keys by default so the
+            #      migration spans multiple batches instead of trivially
+            #      completing against an empty or smoke-sized tree.
+            #   2  Coordinated stop -> sed sc-write-mode -> restart on
+            #      all 4 validators, then poll seidb migrate-evm-status
+            #      until every validator reports completion. Cross-
+            #      validator FlatKV digest agreement is asserted at a
+            #      shared post-migration height; any non-determinism in
+            #      the batch copier would surface here as a digest
+            #      mismatch.
+            #   3  Re-run the fixture round-trip check against the
+            #      now-FlatKV-backed EVM state, confirming pre-migration
+            #      data survives the migration intact (read transparency).
+            name: "FlatKV EVM Migrate",
+            env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+            scripts: [
+              "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
+              "./integration_test/contracts/verify_flatkv_evm_migrate.sh",
+              "docker exec sei-node-0 integration_test/contracts/verify_flatkv_evm_store.sh",
+            ],
+          },
+          # ---------------------------------------------------------------
+          # FlatKV Migrate Resilience / Live Workload / Statesync are
+          # parked alongside the other five rows below: each of them ends
+          # up advancing the chain past the migration boundary while EVM
+          # traffic is being driven (or the post-migration workload script
+          # is invoked directly), so the very first EndBlock after the
+          # boundary trips
+          #
+          #   panic recovered in ProcessBlock
+          #   panic="RouterCommitKVStore.Iterator(store=\"evm\"): \
+          #          iteration not supported after migration completion \
+          #          for store \"evm\""
+          #
+          # in x/evm/keeper.RemoveFirstNTxHashes. Re-enable these rows
+          # together with the keeper/router fix tracked below.
+          # ---------------------------------------------------------------
+          # {
+          #   name: "FlatKV Migrate Resilience",
+          #   env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+          #   scripts: [
+          #     "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
+          #     "./integration_test/contracts/verify_flatkv_migrate_resume_after_crash.sh",
+          #     "./integration_test/contracts/verify_flatkv_post_migrate_evm_workload.sh",
+          #   ],
+          # },
+          # {
+          #   name: "FlatKV Migrate Live Workload",
+          #   env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+          #   scripts: [
+          #     "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
+          #     "./integration_test/contracts/verify_flatkv_migrate_with_live_workload.sh",
+          #   ],
+          # },
+          # {
+          #   name: "FlatKV Migrate Statesync",
+          #   env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+          #   scripts: [
+          #     "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
+          #     "./integration_test/contracts/verify_flatkv_migrate_statesync.sh",
+          #   ],
+          # },
+          {
+            name: "FlatKV Migrate Idle",
+            env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+            scripts: [
+              "./integration_test/contracts/verify_flatkv_migrate_idle.sh",
+            ],
+          },
+          {
+            name: "FlatKV Migrate Negative",
+            env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+            scripts: [
+              "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
+              "./integration_test/contracts/verify_flatkv_partial_quorum_flip_halts.sh",
+            ],
+          },
+          # ---------------------------------------------------------------
+          # The following five FlatKV v0->v1 EVM-migration rows are
+          # parked (not wired into CI) until the post-migration EVM
+          # iterator regression is fixed. They all reliably fail with
+          #
+          #   panic recovered in ProcessBlock
+          #   panic="RouterCommitKVStore.Iterator(store=\"evm\"): \
+          #          iteration not supported after migration completion \
+          #          for store \"evm\""
+          #
+          # at the first EndBlock after migration completes, because
+          # x/evm/keeper.RemoveFirstNTxHashes does a prefix iterator on
+          # the evm store and RouterCommitKVStore refuses iteration once
+          # the evm store has cut over to FlatKV. Re-enable these rows
+          # together with the keeper/router fix.
+          #
+          # Helpers (lib/flatkv_migration.sh), verify scripts and the
+          # MIGRATE_BATCH_EDGE knob are kept in-tree so the bug-fix PR
+          # only needs to wire the rows back in.
+          # ---------------------------------------------------------------
+          # {
+          #   name: "FlatKV Migrate AppHash Continuity",
+          #   env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+          #   scripts: [
+          #     "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
+          #     "./integration_test/contracts/verify_flatkv_migrate_apphash_continuity.sh",
+          #   ],
+          # },
+          # {
+          #   name: "FlatKV Post Migrate Iterator",
+          #   env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+          #   scripts: [
+          #     "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
+          #     "./integration_test/contracts/verify_flatkv_evm_migrate.sh",
+          #     "./integration_test/contracts/verify_flatkv_post_migrate_iterator.sh",
+          #   ],
+          # },
+          # {
+          #   name: "FlatKV Migrate Graceful Restart",
+          #   env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+          #   scripts: [
+          #     "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
+          #     "./integration_test/contracts/verify_flatkv_migrate_graceful_restart.sh",
+          #   ],
+          # },
+          # {
+          #   name: "FlatKV Migrate Batch Min",
+          #   env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+          #   scripts: [
+          #     "docker exec -e FLATKV_EVM_BULK_STORAGE_KEYS=50 sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
+          #     "MIGRATE_BATCH_EDGE=min ./integration_test/contracts/verify_flatkv_migrate_batch_edge.sh",
+          #   ],
+          # },
+          # {
+          #   name: "FlatKV Migrate Batch Oneshot",
+          #   env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+          #   scripts: [
+          #     "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
+          #     "MIGRATE_BATCH_EDGE=oneshot ./integration_test/contracts/verify_flatkv_migrate_batch_edge.sh",
+          #   ],
+          # },
           {
             name: "EVM Module",
             env: "GIGA_STORAGE=true",

Makefile

@@ -295,7 +295,8 @@ CLUSTER_ENV_VARS = DOCKER_PLATFORM=$(DOCKER_PLATFORM) USERID=$(shell id -u) GROU
 	GIGA_OCC=$(GIGA_OCC) \
 	RECEIPT_BACKEND=$(RECEIPT_BACKEND) \
 	AUTOBAHN=$(AUTOBAHN) \
-	GIGA_STORAGE=$(GIGA_STORAGE)
+	GIGA_STORAGE=$(GIGA_STORAGE) \
+	GIGA_MIGRATE_FROM_MEMIAVL=$(GIGA_MIGRATE_FROM_MEMIAVL)
 
 # Run a 4-node docker containers
 docker-cluster-start: docker-cluster-stop build-docker-node
@@ -321,7 +322,7 @@ docker-cluster-start-skipbuild: docker-cluster-stop build-docker-node
 		else \
 			DETACH_FLAG=""; \
 		fi; \
-		DOCKER_PLATFORM=$(DOCKER_PLATFORM) USERID=$(shell id -u) GROUPID=$(shell id -g) GOCACHE=$(shell go env GOCACHE) NUM_ACCOUNTS=10 SKIP_BUILD=true docker compose up $$DETACH_FLAG
+		$(CLUSTER_ENV_VARS) SKIP_BUILD=true docker compose up $$DETACH_FLAG
 .PHONY: localnet-start
 
 # Stop 4-node docker containers

app/seidb.go

@@ -28,6 +28,14 @@ const (
 	FlagSCHistoricalProofRateLimit   = "state-commit.sc-historical-proof-rate-limit"
 	FlagSCHistoricalProofBurst       = "state-commit.sc-historical-proof-burst"
 	FlagSCWriteMode                  = "state-commit.sc-write-mode"
+	// Per-block batch size used by the MigrationManager when sc-write-mode
+	// is one of the in-flight modes (migrate_evm, migrate_bank,
+	// migrate_all_but_bank). Optional: when unset in app.toml the field
+	// stays at DefaultStateCommitConfig().KeysToMigratePerBlock (= 1024),
+	// which is appropriate for production drains. Lowering it spreads the
+	// migration across more blocks, which is useful for tests that need to
+	// exercise the resume / hybrid-read path mid-flight.
+	FlagSCKeysToMigratePerBlock = "state-commit.sc-keys-to-migrate-per-block"
 
 	// SS Store configs
 	FlagSSEnable            = "state-store.ss-enable"
@@ -119,6 +127,16 @@ func parseSCConfigs(appOpts servertypes.AppOptions) config.StateCommitConfig {
 	if v := appOpts.Get(FlagSCHistoricalProofBurst); v != nil {
 		scConfig.HistoricalProofBurst = cast.ToInt(v)
 	}
+	// Guard with v != nil so that an absent app.toml entry preserves the
+	// default of 1024 instead of clobbering it to 0, which would fail
+	// StateCommitConfig.Validate ("keys-to-migrate-per-block must be > 0")
+	// and bring the node down at startup the first time write-mode is
+	// flipped to a migration mode.
+	if v := appOpts.Get(FlagSCKeysToMigratePerBlock); v != nil {
+		if n := cast.ToInt(v); n > 0 {
+			scConfig.KeysToMigratePerBlock = n
+		}
+	}
 
 	return scConfig
 }

docker/docker-compose.yml

@@ -21,6 +21,7 @@ services:
       - RECEIPT_BACKEND
       - AUTOBAHN
       - GIGA_STORAGE
+      - GIGA_MIGRATE_FROM_MEMIAVL
     volumes:
       - "${PROJECT_HOME}:/sei-protocol/sei-chain:Z"
       - "${PROJECT_HOME}/../sei-tendermint:/sei-protocol/sei-tendermint:Z"
@@ -54,6 +55,7 @@ services:
       - RECEIPT_BACKEND
       - AUTOBAHN
       - GIGA_STORAGE
+      - GIGA_MIGRATE_FROM_MEMIAVL
     volumes:
       - "${PROJECT_HOME}:/sei-protocol/sei-chain:Z"
       - "${PROJECT_HOME}/../sei-tendermint:/sei-protocol/sei-tendermint:Z"
@@ -83,6 +85,7 @@ services:
       - RECEIPT_BACKEND
       - AUTOBAHN
       - GIGA_STORAGE
+      - GIGA_MIGRATE_FROM_MEMIAVL
     ports:
       - "26662-26664:26656-26658"
       - "9094-9095:9090-9091"
@@ -116,6 +119,7 @@ services:
       - RECEIPT_BACKEND
       - AUTOBAHN
       - GIGA_STORAGE
+      - GIGA_MIGRATE_FROM_MEMIAVL
     ports:
       - "26665-26667:26656-26658"
       - "9096-9097:9090-9091"

docker/localnode/config/app.toml

@@ -236,6 +236,13 @@ sc-snapshot-writer-limit = 2
 # CacheSize defines the size of the LRU cache for each store on top of the tree, default to 100000.
 sc-cache-size = 1000
 
+# KeysToMigratePerBlock controls how many EVM keys the in-flight migration
+# (sc-write-mode = migrate_evm / migrate_bank / migrate_all_but_bank) drains
+# from memiavl into flatkv per block. Default 1024 is appropriate for
+# production drains; tests lower it to spread the migration across more
+# blocks and exercise the resume / hybrid-read path.
+sc-keys-to-migrate-per-block = 1024
+
 [state-store]
 
 # Enable defines if the state-store should be enabled for historical queries.

docker/localnode/scripts/step4_config_override.sh

@@ -5,6 +5,13 @@ GIGA_EXECUTOR=${GIGA_EXECUTOR:-false}
 GIGA_OCC=${GIGA_OCC:-false}
 AUTOBAHN=${AUTOBAHN:-false}
 GIGA_STORAGE=${GIGA_STORAGE:-false}
+# GIGA_MIGRATE_FROM_MEMIAVL=true boots the cluster in v0 (memiavl_only):
+# memiavl is the sole SC backend, FlatKV is not allocated. This is the
+# starting point for the FlatKV EVM migrate cluster test, which drives a
+# real workload in this mode and then performs a coordinated stop/flip/
+# restart into migrate_evm. Mutually exclusive with GIGA_STORAGE=true;
+# the script picks the more specific override if both are set.
+GIGA_MIGRATE_FROM_MEMIAVL=${GIGA_MIGRATE_FROM_MEMIAVL:-false}
 
 APP_CONFIG_FILE="build/generated/node_$NODE_ID/app.toml"
 TENDERMINT_CONFIG_FILE="build/generated/node_$NODE_ID/config.toml"
@@ -23,11 +30,31 @@ sed -i.bak -e "s|^snapshot-directory *=.*|snapshot-directory = \"./build/generat
 # Enable slow mode
 sed -i.bak -e 's/slow = .*/slow = true/' ~/.sei/config/app.toml
 
+# Boot the cluster in v0 (memiavl_only) for the FlatKV EVM migrate test.
+# Doing this here keeps the override surface narrow: the test runner
+# only has to set one env var to ship a v0-shaped config, and the
+# follow-up flip script just rewrites sc-write-mode in place during the
+# coordinated stop.
+if [ "$GIGA_MIGRATE_FROM_MEMIAVL" = "true" ]; then
+  echo "Booting node $NODE_ID in memiavl_only mode (FlatKV EVM migrate starting point)..."
+  if grep -q '^sc-write-mode[[:space:]]*=' ~/.sei/config/app.toml; then
+    sed -i 's/^sc-write-mode[[:space:]]*=.*/sc-write-mode = "memiavl_only"/' ~/.sei/config/app.toml
+  else
+    sed -i '/^\[state-store\]/i sc-write-mode = "memiavl_only"' ~/.sei/config/app.toml
+  fi
+  # The EVM SS split is irrelevant in this mode (flatkv is not allocated),
+  # but explicitly disabling it keeps app.toml self-describing in case an
+  # operator inspects it post-flip.
+  sed -i 's/^evm-ss-split[[:space:]]*=.*/evm-ss-split = false/' ~/.sei/config/app.toml
+fi
+
 # Enable Giga Storage: FlatKV SC dual-write + EVM SS split.
 # When GIGA_STORAGE=true we also default the receipt backend to parquet; callers
 # can still override this by setting RECEIPT_BACKEND explicitly.
 # Set GIGA_STORAGE=false to disable.
-if [ "$GIGA_STORAGE" = "true" ]; then
+# GIGA_MIGRATE_FROM_MEMIAVL takes precedence: if both are set, the memiavl-only
+# block above ran first and the test runner is responsible for the migration.
+if [ "$GIGA_STORAGE" = "true" ] && [ "$GIGA_MIGRATE_FROM_MEMIAVL" != "true" ]; then
   RECEIPT_BACKEND=${RECEIPT_BACKEND:-parquet}
   echo "Enabling Giga Storage for node $NODE_ID..."
 
@@ -36,14 +63,14 @@ if [ "$GIGA_STORAGE" = "true" ]; then
   # from the memiavl tree via GetChildStoreByName. dual-write keeps memiavl
   # up-to-date for reads while also populating FlatKV. This mode is for test
   # clusters only — never deploy to testnet/mainnet.
-  if grep -q "sc-write-mode" ~/.sei/config/app.toml; then
-    sed -i 's/sc-write-mode = .*/sc-write-mode = "test_only_dual_write"/' ~/.sei/config/app.toml
+  if grep -q '^sc-write-mode[[:space:]]*=' ~/.sei/config/app.toml; then
+    sed -i 's/^sc-write-mode[[:space:]]*=.*/sc-write-mode = "test_only_dual_write"/' ~/.sei/config/app.toml
   else
     sed -i '/^\[state-store\]/i sc-write-mode = "test_only_dual_write"' ~/.sei/config/app.toml
   fi
 
   # --- SS layer: enable EVM split ---
-  sed -i 's/evm-ss-split = .*/evm-ss-split = true/' ~/.sei/config/app.toml
+  sed -i 's/^evm-ss-split[[:space:]]*=.*/evm-ss-split = true/' ~/.sei/config/app.toml
 fi
 
 # Enable Giga Executor if requested

docker/localnode/scripts/step5_start_sei.sh

@@ -8,6 +8,6 @@ mkdir -p $LOG_DIR
 
 echo "Starting the seid process for node $NODE_ID with invariant check interval=$INVARIANT_CHECK_INTERVAL..."
 
-seid start --chain-id sei --inv-check-period ${INVARIANT_CHECK_INTERVAL} > "$LOG_DIR/seid-$NODE_ID.log" 2>&1 &
+seid start --chain-id sei --inv-check-period ${INVARIANT_CHECK_INTERVAL} >> "$LOG_DIR/seid-$NODE_ID.log" 2>&1 &
 echo "Node $NODE_ID seid is started now"
 echo "Done" >> build/generated/launch.complete