fix(compiler): sync non-lifted EVM stack state across block boundaries

src/action/evm_bytecode_visitor.h

@@ -124,6 +124,47 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
     }
   }
 
+  template <typename T, typename = void>
+  struct HasSetCompatibleDynamicJumpTargets : std::false_type {};
+  template <typename T>
+  struct HasSetCompatibleDynamicJumpTargets<
+      T,
+      std::void_t<decltype(std::declval<T &>().setCompatibleDynamicJumpTargets(
+          uint64_t{}, std::declval<const std::vector<uint64_t> &>()))>>
+      : std::true_type {};
+
+  void initializeCompatibleDynamicJumpTargets(const EVMAnalyzer &Analyzer) {
+    if constexpr (HasSetCompatibleDynamicJumpTargets<IRBuilder>::value) {
+      for (const auto &[EntryPC, BlockInfo] : Analyzer.getBlockInfos()) {
+        (void)BlockInfo;
+        const std::vector<uint64_t> TargetBlockPCs =
+            Analyzer.getPotentialDynamicJumpTargetBlocksForSourceBlock(EntryPC);
+        if (TargetBlockPCs.empty()) {
+          continue;
+        }
+        Builder.setCompatibleDynamicJumpTargets(EntryPC, TargetBlockPCs);
+      }
+    } else {
+      (void)Analyzer;
+    }
+  }
+
+  template <typename T, typename = void>
+  struct HasRegisterDirectLiftedPhiIncoming : std::false_type {};
+  template <typename T>
+  struct HasRegisterDirectLiftedPhiIncoming<
+      T,
+      std::void_t<decltype(std::declval<T &>().registerDirectLiftedPhiIncoming(
+          uint64_t{}, uint64_t{}))>> : std::true_type {};
+
+  void registerDirectLiftedPhiIncoming(uint64_t BlockPC) {
+    if constexpr (HasRegisterDirectLiftedPhiIncoming<IRBuilder>::value) {
+      Builder.registerDirectLiftedPhiIncoming(BlockPC, CurrentBlockEntryPC);
+    } else {
+      (void)BlockPC;
+    }
+  }
+
   void push(const Operand &Opnd) { Stack.push(Opnd); }
 
   void requireLogicalStackDepth(uint32_t Depth) {
@@ -193,7 +234,9 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
             PC++;
             continue;
           }
-          Builder.meterOpcode(Opcode, PC);
+          if (Opcode != OP_BLOCKHASH) {
+            Builder.meterOpcode(Opcode, PC);
+          }
         }
 
         switch (Opcode) {
@@ -693,7 +736,7 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
                   HasKnownSucc && isLiftedBlock(SuccPC);
               auto OutgoingStack = drainLogicalStack();
               if (HasKnownLiftedSucc) {
-                assignLiftedEntryState(SuccPC, OutgoingStack);
+                assignDirectLiftedEntryState(SuccPC, OutgoingStack);
               }
               if (!HasKnownSucc) {
                 assignCompatibleDynamicJumpRegionEntryStates(Analyzer,
@@ -749,17 +792,17 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
 
           if (CanTransferWithoutMaterialize) {
             auto OutgoingStack = drainLogicalStack();
-            assignLiftedEntryState(FallthroughPC, OutgoingStack);
-            assignLiftedEntryState(JumpSuccPC, OutgoingStack);
+            assignDirectLiftedEntryState(FallthroughPC, OutgoingStack);
+            assignDirectLiftedEntryState(JumpSuccPC, OutgoingStack);
             finalizeBlockExit(std::move(OutgoingStack), false);
           } else {
             if (CurrentBlockLifted) {
               auto OutgoingStack = drainLogicalStack();
               if (CanPreassignFallthrough) {
-                assignLiftedEntryState(FallthroughPC, OutgoingStack);
+                assignDirectLiftedEntryState(FallthroughPC, OutgoingStack);
               }
               if (CanPreassignJump) {
-                assignLiftedEntryState(JumpSuccPC, OutgoingStack);
+                assignDirectLiftedEntryState(JumpSuccPC, OutgoingStack);
               }
               if (!HasJumpSucc) {
                 assignCompatibleDynamicJumpRegionEntryStates(Analyzer,
@@ -807,12 +850,19 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
           if (PC > RunStartPC && HasLiveFallthrough) {
             Builder.meterOpcodeRange(RunStartPC, PC);
           }
+          if (HasLiveFallthrough && PC == CurrentBlockEntryPC) {
+            if (!CurrentBlockNeedsFinalize) {
+              Builder.meterOpcode(Opcode, PC);
+            }
+            registerCurrentBlockPC(PC);
+            break;
+          }
           if (HasLiveFallthrough && tryAssignFallthroughEntryState(PC)) {
             // Keep runtime stack materialization elided on lifted fallthrough.
           } else {
             if (HasLiveFallthrough && CurrentBlockLifted && isLiftedBlock(PC)) {
               auto OutgoingStack = drainLogicalStack();
-              assignLiftedEntryState(PC, OutgoingStack);
+              assignDirectLiftedEntryState(PC, OutgoingStack);
               finalizeBlockExit(std::move(OutgoingStack), false);
             } else {
               handleEndBlock();
@@ -822,8 +872,7 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
             }
           }
           Builder.handleJumpDest(PC);
-          handleBeginBlock(Analyzer);
-          Builder.meterOpcode(Opcode, PC);
+          handleBeginBlock(Analyzer, true);
           break;
         }
 
@@ -882,6 +931,7 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
   }
 
   void initializeLiftedBlocks(const EVMAnalyzer &Analyzer) {
+    initializeCompatibleDynamicJumpTargets(Analyzer);
     StackLifter.initialize(Analyzer);
   }
 
@@ -928,21 +978,55 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
   }
 
   void finalizeBlockExit(std::vector<Operand> Values, bool Materialize) {
+    if (!CurrentBlockNeedsFinalize) {
+      return;
+    }
+    if (CurrentBlockEntryPC == 2289 || CurrentBlockEntryPC == 2293 ||
+        CurrentBlockEntryPC == 2304 || CurrentBlockEntryPC == 2319 ||
+        CurrentBlockEntryPC == 2356) {
+      fprintf(stderr,
+              "[block-exit-debug] pc=%lu lifted=%d materialize=%d values=%zu "
+              "hidden_prefix=%u\n",
+              (unsigned long)CurrentBlockEntryPC, CurrentBlockLifted,
+              Materialize, Values.size(), CurrentBlockHiddenLiveInPrefixDepth);
+    }
     Builder.endMemoryCompileBlock();
     CurBlockLinearPrecheckPlan = BlockLinearPrecheckPlan();
     if (Materialize) {
       if (CurrentBlockLifted) {
-        spillTrackedStackPreservingPrefix(Values,
-                                          CurrentBlockHiddenLiveInPrefixDepth);
+        if (CurrentBlockHiddenLiveInPrefixDepth > 0) {
+          const size_t HiddenPrefixDepth =
+              static_cast<size_t>(CurrentBlockHiddenLiveInPrefixDepth);
+          ZEN_ASSERT(HiddenPrefixDepth <= Values.size() &&
+                     "Hidden live-in prefix must fit within the logical stack");
+          std::vector<Operand> VisibleSuffix(
+              Values.begin() + static_cast<ptrdiff_t>(HiddenPrefixDepth),
+              Values.end());
+          spillTrackedStackPreservingPrefix(
+              VisibleSuffix, CurrentBlockHiddenLiveInPrefixDepth);
+        } else {
+          spillTrackedStackPreservingPrefix(Values, 0);
+        }
       } else {
         for (const Operand &Opnd : Values) {
           Builder.stackPush(Opnd);
         }
+        Builder.syncTrackedStackMetadataToInstance();
       }
     }
     InDeadCode = true;
     CurrentBlockLifted = false;
     CurrentBlockHiddenLiveInPrefixDepth = 0;
+    CurrentBlockNeedsFinalize = false;
+  }
+
+  void abandonCurrentBlockAfterTrap() {
+    Builder.endMemoryCompileBlock();
+    CurBlockLinearPrecheckPlan = BlockLinearPrecheckPlan();
+    InDeadCode = true;
+    CurrentBlockLifted = false;
+    CurrentBlockHiddenLiveInPrefixDepth = 0;
+    CurrentBlockNeedsFinalize = false;
   }
 
   bool tryGetConstantJumpSuccessorPC(const EVMAnalyzer &Analyzer,
@@ -968,6 +1052,12 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
     StackLifter.assignEntryState(CurrentBlockEntryPC, BlockPC, Values);
   }
 
+  void assignDirectLiftedEntryState(uint64_t BlockPC,
+                                    const std::vector<Operand> &Values) {
+    registerDirectLiftedPhiIncoming(BlockPC);
+    assignLiftedEntryState(BlockPC, Values);
+  }
+
   void assignCompatibleDynamicJumpRegionEntryStates(
       const EVMAnalyzer &Analyzer, const std::vector<Operand> &Values) {
     for (uint64_t TargetBlockPC :
@@ -1001,6 +1091,7 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
                                               BlockPC)) {
       return;
     }
+    registerDirectLiftedPhiIncoming(BlockPC);
     StackLifter.assignEntryState(
         CurrentBlockEntryPC, BlockPC,
         loadLiftedEntryStateFromRuntime(Analyzer, BlockPC));
@@ -1015,7 +1106,7 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
       return false;
     }
     auto OutgoingStack = drainLogicalStack();
-    assignLiftedEntryState(SuccPC, OutgoingStack);
+    assignDirectLiftedEntryState(SuccPC, OutgoingStack);
     finalizeBlockExit(std::move(OutgoingStack), false);
     return true;
   }
@@ -1025,7 +1116,7 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
       return false;
     }
     auto OutgoingStack = drainLogicalStack();
-    assignLiftedEntryState(SuccPC, OutgoingStack);
+    assignDirectLiftedEntryState(SuccPC, OutgoingStack);
     finalizeBlockExit(std::move(OutgoingStack), false);
     return true;
   }
@@ -1062,27 +1153,31 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
         EntryDepth + static_cast<int64_t>(BlockInfo.MinStackHeight);
     if (MinDepth < 0) {
       Builder.handleTrap(common::ErrorCode::EVMStackUnderflow);
-      InDeadCode = true;
-      CurrentBlockLifted = false;
+      abandonCurrentBlockAfterTrap();
       return false;
     }
 
     int64_t MaxDepth =
         EntryDepth + static_cast<int64_t>(BlockInfo.MaxStackHeight);
     if (MaxDepth > static_cast<int64_t>(EVM_MAX_STACK_SIZE)) {
       Builder.handleTrap(common::ErrorCode::EVMStackOverflow);
-      InDeadCode = true;
-      CurrentBlockLifted = false;
+      abandonCurrentBlockAfterTrap();
       return false;
     }
 
     return true;
   }
 
-  void handleBeginBlock(EVMAnalyzer &Analyzer) {
+  void handleBeginBlock(EVMAnalyzer &Analyzer,
+                        bool EntryAlreadyRouted = false) {
     const auto &BlockInfos = Analyzer.getBlockInfos();
     ZEN_ASSERT(BlockInfos.count(PC) > 0 && "Block info not found");
+    const auto &BlockInfo = BlockInfos.at(PC);
+    if (!EntryAlreadyRouted && BlockInfo.IsJumpDest) {
+      Builder.handleJumpDest(PC);
+    }
     Builder.beginMemoryCompileBlock(PC);
+    CurrentBlockNeedsFinalize = true;
     CurBlockLinearPrecheckPlan = BlockLinearPrecheckPlan();
     const Byte *Bytecode = Ctx->getBytecode();
     size_t BytecodeSize = Ctx->getBytecodeSize();
@@ -1101,7 +1196,6 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
             CurBlockLinearPrecheckPlan.CoveredOpcode == OP_MSTORE);
       }
     }
-    const auto &BlockInfo = BlockInfos.at(PC);
     CurrentBlockEntryPC = PC;
     CurrentBlockHiddenLiveInPrefixDepth = 0;
     registerCurrentBlockPC(PC);
@@ -1112,18 +1206,17 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
 
     if (static_cast<size_t>(-BlockInfo.MinStackHeight) > EVM_MAX_STACK_SIZE) {
       Builder.handleTrap(common::ErrorCode::EVMStackUnderflow);
-      InDeadCode = true;
-      CurrentBlockLifted = false;
+      abandonCurrentBlockAfterTrap();
       return;
     }
     if (static_cast<size_t>(BlockInfo.MaxStackHeight) > EVM_MAX_STACK_SIZE) {
       Builder.handleTrap(common::ErrorCode::EVMStackOverflow);
-      InDeadCode = true;
-      CurrentBlockLifted = false;
+      abandonCurrentBlockAfterTrap();
       return;
     }
     InDeadCode = false;
     if (!LiftedBlock) {
+      Builder.reloadTrackedStackFromInstance();
       Builder.createStackCheckBlock(-BlockInfo.MinStackHeight,
                                     1024 - BlockInfo.MaxStackHeight);
     }
@@ -1139,6 +1232,10 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
 
     CurrentBlockLifted = false;
     int32_t TotalPopSize = -BlockInfo.MinPopHeight;
+    if (BlockInfo.HiddenLiveInPrefixDepth > 0 &&
+        BlockInfo.FullEntryStateDepth > TotalPopSize) {
+      TotalPopSize = BlockInfo.FullEntryStateDepth;
+    }
     EvalStack ReverseStack;
     // Refine each popped Operand's ValueRange from analyzer-computed entry
     // ranges so u64-narrow fast paths fire on values flowing through CFG
@@ -1151,9 +1248,17 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
     while (TotalPopSize > 0) {
       Operand Opnd = Builder.stackPop();
       const int32_t SlotIdx = EntryTopIdx - PopIter;
+      EVMValueRange SlotRange = EVMValueRange::U256;
       if (SlotIdx >= 0 && SlotIdx < static_cast<int32_t>(EntryRanges.size())) {
-        Opnd.setRange(EntryRanges[SlotIdx]);
+        SlotRange = EntryRanges[SlotIdx];
+        Opnd.setRange(SlotRange);
       }
+      // Anchor runtime-preloaded entry values in dedicated vars so later deep
+      // stack uses do not depend on reusing raw load trees across
+      // pops/branches.
+      Operand AnchoredOpnd = Builder.createStackEntryOperand(SlotRange);
+      Builder.assignStackEntryOperand(AnchoredOpnd, Opnd);
+      Opnd = AnchoredOpnd;
       ReverseStack.push(Opnd);
       ++PopIter;
       --TotalPopSize;
@@ -1192,7 +1297,12 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
     }
   }
 
-  void handleEndBlock() { finalizeBlockExit(drainLogicalStack(), true); }
+  void handleEndBlock() {
+    if (!CurrentBlockNeedsFinalize) {
+      return;
+    }
+    finalizeBlockExit(drainLogicalStack(), true);
+  }
 
   void handleStop() { Builder.handleStop(); }
 
@@ -1920,6 +2030,7 @@ template <typename IRBuilder> class EVMByteCodeVisitor {
   uint64_t CurrentBlockEntryPC = 0;
   bool CurrentBlockLifted = false;
   uint32_t CurrentBlockHiddenLiveInPrefixDepth = 0;
+  bool CurrentBlockNeedsFinalize = false;
 };
 
 } // namespace COMPILER

src/compiler/evm_compiler.cpp

@@ -99,7 +99,6 @@ void EagerEVMJITCompiler::compile() {
   ZEN_ASSERT(Ctx.ExternRelocs.empty());
 
   uint8_t *JITFuncPtr = Ctx.CodePtr + Ctx.FuncOffsetMap[0];
-  EVMMod->setJITCodeAndSize(JITFuncPtr, Ctx.CodeSize);
 #ifdef ZEN_ENABLE_LINUX_PERF
   // Write block symbols instead of EVM_Main
   // JIT_DUMP_WRITE_FUNC("EVM_Main", JITFuncPtr, Ctx.FuncSizeMap[0]);
@@ -114,7 +113,11 @@ void EagerEVMJITCompiler::compile() {
   size_t CodeSize = CodeMPool.getMemEnd() - JITCode;
   platform::mprotect(JITCode, TO_MPROTECT_CODE_SIZE(CodeSize),
                      PROT_READ | PROT_EXEC);
-  EVMMod->setJITCodeAndSize(JITCode, CodeSize);
+  // Runtime must enter at the compiled EVM main function, not at the start of
+  // the backing code buffer. In release builds the first function may have a
+  // non-zero offset due to emitted helper blocks, and using the raw buffer
+  // base as the entrypoint jumps into unrelated code.
+  EVMMod->setJITCodeAndSize(JITFuncPtr, CodeSize - Ctx.FuncOffsetMap[0]);
 
   Stats.stopRecord(Timer);
 }