LLVM and SPIRV-LLVM-Translator pulldown (WW20 2026)

.gitignore

@@ -58,6 +58,7 @@ autoconf/autom4te.cache
 /.idea
 /cmake-build*
 # Coding assistants' stuff
+.agents/
 /CLAUDE.md
 /instructions.md
 .claude/

bolt/docs/profiles.md

@@ -74,18 +74,51 @@ Where:
 - `R` — Aggregated trace originating at a return.
 - `B` — Aggregated branch from `<start>` to `<end>`.
 - `F` — Aggregated fall-through from `<start>` to `<end>`.
-- `f` — Aggregated fall-through with external origin (disambiguates returns
-  hitting a basic block head from regular internal jumps).
-- `r` — Aggregated fall-through originating at an external return (no checks
-  performed for fall-through start).
+- `f` — Aggregated fall-through with external origin.
+- `r` — Aggregated fall-through with external return as origin.
+
+### Trace mapping
+Internally, all branch/fall-through formats are represented as traces, with the
+following field mapping:
+- `B <start> <end>` -> `T <start> <end> BR_ONLY`
+- `F <start> <end>` -> `T FT_ONLY <start> <end>`
+- `f <start> <end>` -> `T FT_EXTERNAL_ORIGIN <start> <end>`
+- `r <start> <end>` -> `R FT_EXTERNAL_RETURN <start> <end>`
+
+The constants have the following values and can be specified directly:
+- `BR_ONLY`/`FT_ONLY`: UINT64_MAX: ffffffffffffffff or -1
+- `FT_EXTERNAL_ORIGIN`: UINT64_MAX-1: fffffffffffffffe or -2
+- `FT_EXTERNAL_RETURN`: UINT64_MAX-2: fffffffffffffffd or -3
+
+### Call continuation fall-throughs
+Where applicable, call continuation fall-throughs are extended back to cover
+the call site, improving profile continuity:
+- `f`: check that the fall-through start is a block start, not an entry point
+  or a landing pad.
+- `R`/`r`: unconditional.
+- `T`: local origin: extend only if it's a return. External origin: use `f`.
+
+Return hint is only needed for external returns (e.g. from a PLT call), where
+the address can't be disassembled to distinguish a return from a jump.
+
+### Best practices
+Use T/R format. To capture branch type using Linux perf events, set
+the `PERF_SAMPLE_BRANCH_TYPE_SAVE` flag in `perf_event_attr::branch_sample_type`.
+
+To represent top of brstack entry with no fall-through, use `BR_ONLY` as `ft_end`.
+Such traces can be extended with an average fall-through length by passing
+`--impute-trace-fall-through`.
 
 ### Location format
 
-Locations have the format `[<buildid>:]<offset>`:
-- `<offset>` — Hex offset from the object base load address.
+Locations have the format `[<buildid>:]<addr>`:
+- `<addr>` — Hex vaddr (non-PIE) or offset from the object base load address (PIE).
 - `<buildid>:<offset>` — Offset within the object identified by `<buildid>`.
 - `X:<addr>` — External address (outside the profiled binary).
 
+Base load address is the address of the first `PT_LOAD` segment in the binary, which
+may not be the same as the segment containing code address.
+
 ### Examples
 
 Basic samples profile:
@@ -101,6 +134,11 @@ Trace profile combining branches and fall-throughs:
 T 4b196f 4b19e0 4b19ef 2
 ```
 
+Trace with unknown fall-through:
+```
+T 4b196f 4b19e0 -1 2
+```
+
 Legacy branch profile with separate branches and fall-throughs:
 ```
 F 41be50 41be50 3

bolt/include/bolt/Core/BinaryContext.h

@@ -679,8 +679,6 @@ class BinaryContext {
 
   std::unique_ptr<MCObjectFileInfo> MOFI;
 
-  MCTargetOptions MCOptions;
-
   std::unique_ptr<const MCAsmInfo> AsmInfo;
 
   std::unique_ptr<const MCInstrInfo> MII;
@@ -726,6 +724,11 @@ class BinaryContext {
   /// FunctionFragment::getFragmentNum() == FragmentNum::warm()
   bool HasWarmSection{false};
 
+  /// Indicates if the binary should assume large code model
+  /// Can be triggered by the presence of .ltext sections if
+  /// unspecified.
+  bool UseLargeCodeModel{false};
+
   /// Is the binary always loaded at a fixed address. Shared objects and
   /// position-independent executables (PIEs) are examples of binaries that
   /// will have HasFixedLoadAddress set to false.
@@ -851,6 +854,10 @@ class BinaryContext {
   /// enum Constants, e.g. DW_EH_PE_omit.
   unsigned LSDAEncoding = dwarf::DW_EH_PE_omit;
 
+  /// Update LSDAEncoding for the binary taking into account
+  /// large code model and position-independent executables.
+  void updateLSDAEncoding();
+
   BinaryContext(std::unique_ptr<MCContext> Ctx,
                 std::unique_ptr<DWARFContext> DwCtx,
                 std::unique_ptr<Triple> TheTriple,
@@ -1539,7 +1546,7 @@ class BinaryContext {
   IndependentCodeEmitter createIndependentMCCodeEmitter() const {
     IndependentCodeEmitter MCEInstance;
     MCEInstance.LocalCtx.reset(
-        new MCContext(*TheTriple, AsmInfo.get(), MRI.get(), STI.get()));
+        new MCContext(*TheTriple, *AsmInfo, MRI.get(), STI.get()));
     MCEInstance.LocalMOFI.reset(
         TheTarget->createMCObjectFileInfo(*MCEInstance.LocalCtx,
                                           /*PIC=*/!HasFixedLoadAddress));

bolt/include/bolt/Core/MCPlus.h

@@ -113,13 +113,20 @@ template <typename ValueType> class MCSimpleAnnotation : public MCAnnotation {
   ValueType Value;
 };
 
+/// Return true if \p Op is the annotation sentinel: a null MCInst operand
+/// that marks the start of BOLT annotations. Non-null MCInst operands
+/// (e.g. Hexagon duplex sub-instructions) are legitimate prime operands.
+inline bool isAnnotationSentinel(const MCOperand &Op) {
+  return Op.isInst() && Op.getInst() == nullptr;
+}
+
 /// Return a number of operands in \Inst excluding operands representing
 /// annotations.
 inline unsigned getNumPrimeOperands(const MCInst &Inst) {
   for (signed I = Inst.getNumOperands() - 1; I >= 0; --I) {
-    if (Inst.getOperand(I).isInst())
+    if (isAnnotationSentinel(Inst.getOperand(I)))
       return I;
-    if (!Inst.getOperand(I).isImm())
+    if (!Inst.getOperand(I).isInst() && !Inst.getOperand(I).isImm())
       return Inst.getNumOperands();
   }
   return Inst.getNumOperands();

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -138,10 +138,8 @@ class MCPlusBuilder {
 
   MCInst::iterator getAnnotationInstOp(MCInst &Inst) const {
     for (MCInst::iterator Iter = Inst.begin(); Iter != Inst.end(); ++Iter) {
-      if (Iter->isInst()) {
-        assert(Iter->getInst() == nullptr && "Empty instruction expected.");
+      if (MCPlus::isAnnotationSentinel(*Iter))
         return Iter;
-      }
     }
     return Inst.end();
   }

bolt/include/bolt/Profile/DataAggregator.h

@@ -87,6 +87,7 @@ class DataAggregator : public DataReader {
   friend raw_ostream &operator<<(raw_ostream &OS, const LBREntry &);
 
   friend struct PerfSpeEventsTestHelper;
+  friend struct PreAggregatedTestHelper;
 
   struct PerfBranchSample {
     SmallVector<LBREntry, 32> LBR;

bolt/lib/Core/BinaryContext.cpp

@@ -47,6 +47,8 @@ using namespace llvm;
 
 namespace opts {
 
+extern cl::opt<bool> LargeCodeModel;
+
 static cl::opt<bool>
     NoHugePages("no-huge-pages",
                 cl::desc("use regular size pages for code alignment"),
@@ -155,9 +157,6 @@ BinaryContext::BinaryContext(std::unique_ptr<MCContext> Ctx,
       STI(std::move(STI)), InstPrinter(std::move(InstPrinter)),
       MIA(std::move(MIA)), MIB(std::move(MIB)), MRI(std::move(MRI)),
       DisAsm(std::move(DisAsm)), Logger(Logger), InitialDynoStats(isAArch64()) {
-  // createMCAsmInfo stored a pointer to a local MCTargetOptions in MCAsmInfo.
-  // Update it to point to our member that will outlive MCAsmInfo.
-  const_cast<MCAsmInfo *>(this->AsmInfo.get())->setTargetOptions(MCOptions);
   RegularPageSize = isAArch64() ? RegularPageSizeAArch64 : RegularPageSizeX86;
   PageAlign = opts::NoHugePages ? RegularPageSize : HugePageSize;
 }
@@ -227,8 +226,9 @@ Expected<std::unique_ptr<BinaryContext>> BinaryContext::createBinaryContext(
         make_error_code(std::errc::not_supported),
         Twine("BOLT-ERROR: no register info for target ", TripleName));
 
-  // Set up disassembler.
-  MCTargetOptions MCOptions;
+  // Set up disassembler. The MCAsmInfo holds a reference to MCTargetOptions, so
+  // make it static to outlive the AsmInfo.
+  static const MCTargetOptions MCOptions;
   std::unique_ptr<MCAsmInfo> AsmInfo(
       TheTarget->createMCAsmInfo(*MRI, TheTriple, MCOptions));
   if (!AsmInfo)
@@ -255,20 +255,10 @@ Expected<std::unique_ptr<BinaryContext>> BinaryContext::createBinaryContext(
         Twine("BOLT-ERROR: no instruction info for target ", TripleName));
 
   std::unique_ptr<MCContext> Ctx(
-      new MCContext(TheTriple, AsmInfo.get(), MRI.get(), STI.get()));
+      new MCContext(TheTriple, *AsmInfo, MRI.get(), STI.get()));
   std::unique_ptr<MCObjectFileInfo> MOFI(
       TheTarget->createMCObjectFileInfo(*Ctx, IsPIC));
   Ctx->setObjectFileInfo(MOFI.get());
-  // We do not support X86 Large code model. Change this in the future.
-  bool Large = false;
-  if (TheTriple.getArch() == llvm::Triple::aarch64)
-    Large = true;
-  unsigned LSDAEncoding =
-      Large ? dwarf::DW_EH_PE_absptr : dwarf::DW_EH_PE_udata4;
-  if (IsPIC) {
-    LSDAEncoding = dwarf::DW_EH_PE_pcrel |
-                   (Large ? dwarf::DW_EH_PE_sdata8 : dwarf::DW_EH_PE_sdata4);
-  }
 
   std::unique_ptr<MCDisassembler> DisAsm(
       TheTarget->createMCDisassembler(*STI, *Ctx));
@@ -306,7 +296,13 @@ Expected<std::unique_ptr<BinaryContext>> BinaryContext::createBinaryContext(
       std::move(InstructionPrinter), std::move(MIA), nullptr, std::move(MRI),
       std::move(DisAsm), Logger);
 
-  BC->LSDAEncoding = LSDAEncoding;
+  // Use large code model encoding for AArch64 (always). For X86, this is
+  // updated after detecting .ltext if unset.
+  // Otherwise allow the user to force it via `--large-code-model` flag.
+  if (TheTriple.getArch() == llvm::Triple::aarch64)
+    BC->UseLargeCodeModel = true;
+  else if (opts::LargeCodeModel.getNumOccurrences())
+    BC->UseLargeCodeModel = opts::LargeCodeModel;
 
   BC->MAB = std::unique_ptr<MCAsmBackend>(
       BC->TheTarget->createMCAsmBackend(*BC->STI, *BC->MRI, MCTargetOptions()));
@@ -318,6 +314,8 @@ Expected<std::unique_ptr<BinaryContext>> BinaryContext::createBinaryContext(
   BC->SymbolicDisAsm = std::unique_ptr<MCDisassembler>(
       BC->TheTarget->createMCDisassembler(*BC->STI, *BC->Ctx));
 
+  BC->updateLSDAEncoding();
+
   if (!BC->SymbolicDisAsm)
     return createStringError(
         make_error_code(std::errc::not_supported),
@@ -393,6 +391,14 @@ bool BinaryContext::validateHoles() const {
   return Valid;
 }
 
+void BinaryContext::updateLSDAEncoding() {
+  LSDAEncoding = HasFixedLoadAddress
+                     ? dwarf::DW_EH_PE_absptr
+                     : (dwarf::DW_EH_PE_pcrel |
+                        (this->UseLargeCodeModel ? dwarf::DW_EH_PE_sdata8
+                                                 : dwarf::DW_EH_PE_sdata4));
+}
+
 void BinaryContext::updateObjectNesting(BinaryDataMapType::iterator GAI) {
   const uint64_t Address = GAI->second->getAddress();
   const uint64_t Size = GAI->second->getSize();

bolt/lib/Core/BinaryEmitter.cpp

@@ -355,7 +355,7 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
   }
 
   MCContext &Context = Streamer.getContext();
-  const MCAsmInfo *MAI = Context.getAsmInfo();
+  const MCAsmInfo &MAI = Context.getAsmInfo();
 
   MCSymbol *const StartSymbol = Function.getSymbol(FF.getFragmentNum());
 
@@ -387,7 +387,7 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
     for (const MCCFIInstruction &CFIInstr : Function.cie()) {
       // Only write CIE CFI insns that LLVM will not already emit
       const std::vector<MCCFIInstruction> &FrameInstrs =
-          MAI->getInitialFrameState();
+          MAI.getInitialFrameState();
       if (!llvm::is_contained(FrameInstrs, CFIInstr))
         emitCFIInstruction(CFIInstr);
     }
@@ -413,7 +413,7 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
   if (size_t Padding = opts::padFunctionAfter(Function)) {
     LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with "
                       << Padding << " bytes\n");
-    Streamer.emitFill(Padding, MAI->getTextAlignFillValue());
+    Streamer.emitFill(Padding, MAI.getTextAlignFillValue());
   }
 
   if (opts::MarkFuncs)
@@ -426,7 +426,7 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
   MCSymbol *EndSymbol = Function.getFunctionEndLabel(FF.getFragmentNum());
   Streamer.emitLabel(EndSymbol);
 
-  if (MAI->hasDotTypeDotSizeDirective()) {
+  if (MAI.hasDotTypeDotSizeDirective()) {
     const MCExpr *SizeExpr = MCBinaryExpr::createSub(
         MCSymbolRefExpr::create(EndSymbol, Context),
         MCSymbolRefExpr::create(StartSymbol, Context), Context);

bolt/lib/Core/DebugData.cpp

@@ -966,7 +966,7 @@ static inline void emitBinaryDwarfLineTable(
   unsigned Discriminator = 0;
   uint64_t LastAddress = InvalidAddress;
   uint64_t PrevEndOfSequence = InvalidAddress;
-  const MCAsmInfo *AsmInfo = MCOS->getContext().getAsmInfo();
+  const MCAsmInfo &AsmInfo = MCOS->getContext().getAsmInfo();
 
   auto emitEndOfSequence = [&](uint64_t Address) {
     MCDwarfLineAddr::Emit(MCOS, Params, INT64_MAX, Address - LastAddress);
@@ -1037,7 +1037,7 @@ static inline void emitBinaryDwarfLineTable(
       } else {
         if (LastAddress == InvalidAddress)
           emitDwarfSetLineAddrAbs(*MCOS, Params, LineDelta, Address,
-                                  AsmInfo->getCodePointerSize());
+                                  AsmInfo.getCodePointerSize());
         else
           MCDwarfLineAddr::Emit(MCOS, Params, LineDelta, Address - LastAddress);
 
@@ -1068,13 +1068,13 @@ static inline void emitDwarfLineTable(
   unsigned Isa = 0;
   unsigned Discriminator = 0;
   MCSymbol *LastLabel = nullptr;
-  const MCAsmInfo *AsmInfo = MCOS->getContext().getAsmInfo();
+  const MCAsmInfo &AsmInfo = MCOS->getContext().getAsmInfo();
 
   // Loop through each MCDwarfLineEntry and encode the dwarf line number table.
   for (const MCDwarfLineEntry &LineEntry : LineEntries) {
     if (LineEntry.getFlags() & DWARF2_FLAG_END_SEQUENCE) {
       MCOS->emitDwarfAdvanceLineAddr(INT64_MAX, LastLabel, LineEntry.getLabel(),
-                                     AsmInfo->getCodePointerSize());
+                                     AsmInfo.getCodePointerSize());
       FileNum = 1;
       LastLine = 1;
       Column = 0;
@@ -1128,7 +1128,7 @@ static inline void emitDwarfLineTable(
     // in line numbers and the increment of the address from the previous
     // Label and the current Label.
     MCOS->emitDwarfAdvanceLineAddr(LineDelta, LastLabel, Label,
-                                   AsmInfo->getCodePointerSize());
+                                   AsmInfo.getCodePointerSize());
     Discriminator = 0;
     LastLine = LineEntry.getLine();
     LastLabel = Label;

bolt/lib/Passes/CMOVConversion.cpp

@@ -271,6 +271,11 @@ void CMOVConversion::runOnFunction(BinaryFunction &Function) {
 }
 
 Error CMOVConversion::runOnFunctions(BinaryContext &BC) {
+  if (!BC.isX86()) {
+    BC.errs() << "BOLT-ERROR: " << getName() << " is supported only on X86\n";
+    exit(1);
+  }
+
   for (auto &It : BC.getBinaryFunctions()) {
     BinaryFunction &Function = It.second;
     if (!shouldOptimize(Function))

bolt/lib/Passes/IndirectCallPromotion.cpp

@@ -1142,6 +1142,11 @@ Error IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
   if (opts::ICP == ICP_NONE)
     return Error::success();
 
+  if (!BC.isX86()) {
+    BC.errs() << "BOLT-ERROR: " << getName() << " is supported only on X86\n";
+    exit(1);
+  }
+
   auto &BFs = BC.getBinaryFunctions();
 
   const bool OptimizeCalls = (opts::ICP == ICP_CALLS || opts::ICP == ICP_ALL);