diff --git a/cmd/internal/flags/flags.go b/cmd/internal/flags/flags.go index 1001aaca85..4015475bde 100644 --- a/cmd/internal/flags/flags.go +++ b/cmd/internal/flags/flags.go @@ -40,6 +40,7 @@ var ForceEspClang bool var SizeReport bool var SizeFormat string var SizeLevel string +var SizeOptimize bool var ForceRebuild bool var PrintCommands bool @@ -65,6 +66,7 @@ func AddBuildFlags(fs *flag.FlagSet) { fs.BoolVar(&SizeReport, "size", false, "Print size report after build (default format=text, level=module)") fs.StringVar(&SizeFormat, "size-format", "", "Size report format (text,json). Default text.") fs.StringVar(&SizeLevel, "size-level", "", "Size report aggregation level (full,module,package). Default module.") + fs.BoolVar(&SizeOptimize, "sizeopt", false, "Enable size-oriented global IR optimization at final link stage") } func AddBuildModeFlags(fs *flag.FlagSet) { @@ -181,6 +183,7 @@ func UpdateConfig(conf *build.Config) error { conf.Port = Port conf.BaudRate = BaudRate conf.ForceRebuild = ForceRebuild + conf.SizeOpt = SizeOptimize if SizeReport || SizeFormat != "" || SizeLevel != "" { conf.SizeReport = true if SizeFormat != "" { diff --git a/internal/build/build.go b/internal/build/build.go index 841a6dbf03..e9e55bb6fa 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -141,6 +141,7 @@ type Config struct { SizeReport bool // print size report after successful build SizeFormat string // size report format: text,json (default text) SizeLevel string // size aggregation level: full,module,package (default module) + SizeOpt bool // enable size-oriented global IR optimization at final link stage CompilerHash string // metadata hash for the running compiler (development builds only) // GlobalRewrites specifies compile-time overrides for global string variables. // Keys are fully qualified package paths (e.g. "main" or "github.com/user/pkg"). @@ -580,28 +581,241 @@ func (c *context) hasAltPkg(pkgPath string) bool { return hasAltPkgForTarget(c.buildConf, pkgPath) } -// normalizeToArchive creates an archive from object files and sets ArchiveFile. -// This ensures the link step always consumes .a archives regardless of cache state. -func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { +func splitBitcodeAndNativeInputs(inputs []string) (bitcodeFiles []string, nativeInputs []string) { + for _, input := range inputs { + if strings.HasSuffix(input, ".bc") { + bitcodeFiles = append(bitcodeFiles, input) + continue + } + nativeInputs = append(nativeInputs, input) + } + return +} + +func tempNamePrefix(moduleName string) string { + name := path.Base(moduleName) + if name == "" || name == "." || name == "/" { + return "llgo" + } + return name +} + +const linkedBitcodeSizePassPipeline = "module(globalopt,ipsccp,globaldce)" + +func optimizeLinkedBitcode(ctx *context, moduleName, inputBitcode string) (string, error) { + llvmCtx := gllvm.NewContext() + defer llvmCtx.Dispose() + + mod, err := llvmCtx.ParseBitcodeFile(inputBitcode) + if err != nil { + return "", fmt.Errorf("parse linked bitcode %s: %w", inputBitcode, err) + } + defer mod.Dispose() + + mod.SetDataLayout(ctx.prog.DataLayout()) + mod.SetTarget(ctx.prog.Target().Spec().Triple) + + pbo := gllvm.NewPassBuilderOptions() + defer pbo.Dispose() + + if err := mod.RunPasses(linkedBitcodeSizePassPipeline, ctx.prog.TargetMachine(), pbo); err != nil { + return "", fmt.Errorf("run linked bitcode LLVM passes for %s failed: %w", moduleName, err) + } + + out, err := os.CreateTemp("", tempNamePrefix(moduleName)+"-postpass-*.bc") + if err != nil { + return "", fmt.Errorf("create post-pass bitcode file: %w", err) + } + defer out.Close() + + if err := gllvm.WriteBitcodeToFile(mod, out); err != nil { + return "", fmt.Errorf("write post-pass bitcode file: %w", err) + } + return out.Name(), nil +} + +func mergeBitcodeFiles(moduleName string, bitcodeFiles []string) (string, error) { + if len(bitcodeFiles) == 0 { + return "", nil + } + if len(bitcodeFiles) == 1 { + return bitcodeFiles[0], nil + } + + ctx := gllvm.NewContext() + defer ctx.Dispose() + + mod, err := ctx.ParseBitcodeFile(bitcodeFiles[0]) + if err != nil { + return "", fmt.Errorf("parse bitcode %s: %w", bitcodeFiles[0], err) + } + defer mod.Dispose() + + for _, bitcodeFile := range bitcodeFiles[1:] { + srcMod, err := ctx.ParseBitcodeFile(bitcodeFile) + if err != nil { + return "", fmt.Errorf("parse bitcode %s: %w", bitcodeFile, err) + } + if err := gllvm.LinkModules(mod, srcMod); err != nil { + return "", fmt.Errorf("link bitcode module %s: %w", bitcodeFile, err) + } + } + + out, err := os.CreateTemp("", tempNamePrefix(moduleName)+"-*.bc") + if err != nil { + return "", fmt.Errorf("create merged bitcode file: %w", err) + } + defer out.Close() + + if err := gllvm.WriteBitcodeToFile(mod, out); err != nil { + return "", fmt.Errorf("write merged bitcode file: %w", err) + } + return out.Name(), nil +} + +func mergeBitcodeFilesWithIR(moduleName string, bitcodeFiles []string) (string, string, error) { + if len(bitcodeFiles) == 0 { + return "", "", nil + } + + ctx := gllvm.NewContext() + defer ctx.Dispose() + + mod, err := ctx.ParseBitcodeFile(bitcodeFiles[0]) + if err != nil { + return "", "", fmt.Errorf("parse bitcode %s: %w", bitcodeFiles[0], err) + } + defer mod.Dispose() + + for _, bitcodeFile := range bitcodeFiles[1:] { + srcMod, err := ctx.ParseBitcodeFile(bitcodeFile) + if err != nil { + return "", "", fmt.Errorf("parse bitcode %s: %w", bitcodeFile, err) + } + if err := gllvm.LinkModules(mod, srcMod); err != nil { + return "", "", fmt.Errorf("link bitcode module %s: %w", bitcodeFile, err) + } + } + + ir := mod.String() + if len(bitcodeFiles) == 1 { + return bitcodeFiles[0], ir, nil + } + + out, err := os.CreateTemp("", tempNamePrefix(moduleName)+"-*.bc") + if err != nil { + return "", "", fmt.Errorf("create merged bitcode file: %w", err) + } + defer out.Close() + + if err := gllvm.WriteBitcodeToFile(mod, out); err != nil { + return "", "", fmt.Errorf("write merged bitcode file: %w", err) + } + return out.Name(), ir, nil +} + +func writeLLVMIRFromBitcode(bitcodeFile, llFile string) error { + ctx := gllvm.NewContext() + defer ctx.Dispose() + + mod, err := ctx.ParseBitcodeFile(bitcodeFile) + if err != nil { + return fmt.Errorf("parse bitcode %s: %w", bitcodeFile, err) + } + defer mod.Dispose() + + if err := os.WriteFile(llFile, []byte(mod.String()), 0o644); err != nil { + return fmt.Errorf("write ll file %s: %w", llFile, err) + } + return nil +} + +func normalizePackageOutputs(ctx *context, aPkg *aPackage, verbose bool) error { if len(aPkg.ObjFiles) == 0 { return nil } - archiveFile, err := os.CreateTemp("", "pkg-*.a") + bitcodeFiles, nativeInputs := splitBitcodeAndNativeInputs(aPkg.ObjFiles) + var normalized []string + + if len(bitcodeFiles) > 0 { + mergedBitcode, err := mergeBitcodeFiles(aPkg.PkgPath, bitcodeFiles) + if err != nil { + return fmt.Errorf("merge bitcode for %s: %w", aPkg.PkgPath, err) + } + normalized = append(normalized, mergedBitcode) + } + + if len(nativeInputs) > 0 { + archiveFile, err := os.CreateTemp("", "pkg-native-*.a") + if err != nil { + return fmt.Errorf("create temp native archive: %w", err) + } + archiveFile.Close() + archivePath := archiveFile.Name() + + if err := ctx.createArchiveFile(archivePath, nativeInputs, verbose); err != nil { + os.Remove(archivePath) + return fmt.Errorf("create native archive for %s: %w", aPkg.PkgPath, err) + } + normalized = append(normalized, archivePath) + } + aPkg.ObjFiles = normalized + return nil +} + +func compileLinkedBitcodeToObject(ctx *context, moduleName string, bitcodeFiles []string, verbose bool) (string, error) { + if len(bitcodeFiles) == 0 { + return "", nil + } + + mergedBitcode, linkedModuleIR, err := mergeBitcodeFilesWithIR(moduleName, bitcodeFiles) if err != nil { - return fmt.Errorf("create temp archive: %w", err) + return "", err } - archiveFile.Close() - archivePath := archiveFile.Name() + printCmds := ctx.shouldPrintCommands(verbose) - if err := ctx.createArchiveFile(archivePath, aPkg.ObjFiles, verbose); err != nil { - os.Remove(archivePath) - return fmt.Errorf("create archive for %s: %w", aPkg.PkgPath, err) + // In verbose mode, emit the final linked module as .ll beside the .bc so + // users can inspect exactly what will be compiled to native object code. + if printCmds { + linkedLL := strings.TrimSuffix(mergedBitcode, filepath.Ext(mergedBitcode)) + ".ll" + fmt.Fprintf(os.Stderr, "# emitting linked bitcode ll for %s: %s\n", moduleName, linkedLL) + if err := os.WriteFile(linkedLL, []byte(linkedModuleIR), 0o644); err != nil { + fmt.Fprintf(os.Stderr, "warning: failed to emit linked bitcode ll for %s: %v\n", moduleName, err) + } } - aPkg.ObjFiles = nil - aPkg.ArchiveFile = archivePath - return nil + compileInputBitcode := mergedBitcode + if ctx.buildConf.SizeOpt && ctx.passOpt { + if printCmds { + fmt.Fprintf(os.Stderr, "# running linked bitcode size-opt pass pipeline for %s: %s\n", moduleName, linkedBitcodeSizePassPipeline) + } + optimizedBitcode, err := optimizeLinkedBitcode(ctx, moduleName, mergedBitcode) + if err != nil { + return "", err + } + compileInputBitcode = optimizedBitcode + } + + objFile, err := os.CreateTemp("", tempNamePrefix(moduleName)+"-*.o") + if err != nil { + return "", fmt.Errorf("create linked object temp file: %w", err) + } + objFile.Close() + + args := []string{"-o", objFile.Name(), "-c", compileInputBitcode, "-Wno-override-module"} + if IsDbgSymsEnabled() { + args = append(args, "-gdwarf-4") + } + if printCmds { + fmt.Fprintf(os.Stderr, "# compiling linked bitcode for %s\n", moduleName) + fmt.Fprintln(os.Stderr, "clang", args) + } + if err := ctx.compiler().Compile(args...); err != nil { + return "", fmt.Errorf("compile linked bitcode for %s: %w", moduleName, err) + } + + return objFile.Name(), nil } func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, error) { @@ -648,7 +862,7 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er return err } if !aPkg.CacheHit { - if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { + if err := normalizePackageOutputs(ctx, aPkg, verbose); err != nil { return err } if kind == cl.PkgLinkExtern { @@ -683,7 +897,7 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er needRuntime = needRuntime || aPkg.NeedRt needPyInit = needPyInit || aPkg.NeedPyInit if !aPkg.CacheHit { - if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { + if err := normalizePackageOutputs(ctx, aPkg, verbose); err != nil { return err } if err := ctx.saveToCache(aPkg); err != nil && verbose { @@ -815,14 +1029,15 @@ func validateRewriteInput(pkg, varName, value string) { } } -// compileExtraFiles compiles extra files (.s/.c) from target configuration and returns object files +// compileExtraFiles compiles extra files (.s/.c) from target configuration and returns link inputs. +// C-like files are emitted as .bc for LTO while assembly remains .o. func compileExtraFiles(ctx *context, verbose bool) ([]string, error) { if len(ctx.crossCompile.ExtraFiles) == 0 { return nil, nil } printCmds := ctx.shouldPrintCommands(verbose) - var objFiles []string + var linkInputs []string llgoRoot := env.LLGoROOT() for _, extraFile := range ctx.crossCompile.ExtraFiles { @@ -855,35 +1070,44 @@ func compileExtraFiles(ctx *context, verbose bool) ([]string, error) { baseArgs = append(baseArgs, "-x", "assembler-with-cpp") } - // If GenLL is enabled, first emit .ll for debugging - if ctx.buildConf.GenLL { - llFile := baseName + ".ll" - llArgs := append(slices.Clone(baseArgs), "-emit-llvm", "-S", "-o", llFile, "-c", srcFile) + emitBitcode := ext != ".S" && ext != ".s" + + if emitBitcode { + bcFile := baseName + ".bc" + bcArgs := append(baseArgs, "-emit-llvm", "-o", bcFile, "-c", srcFile) if printCmds { - fmt.Fprintf(os.Stderr, "Compiling extra file (ll): clang %s\n", strings.Join(llArgs, " ")) + fmt.Fprintf(os.Stderr, "Compiling extra file (bc): clang %s\n", strings.Join(bcArgs, " ")) } cmd := ctx.compiler() - if err := cmd.Compile(llArgs...); err != nil { - return nil, fmt.Errorf("failed to compile extra file %s to .ll: %w", srcFile, err) + if err := cmd.Compile(bcArgs...); err != nil { + return nil, fmt.Errorf("failed to compile extra file %s to .bc: %w", srcFile, err) } + if ctx.buildConf.GenLL { + llFile := baseName + ".ll" + if printCmds { + fmt.Fprintf(os.Stderr, "Emitting extra file (ll): %s (from %s)\n", llFile, bcFile) + } + if err := writeLLVMIRFromBitcode(bcFile, llFile); err != nil { + return nil, fmt.Errorf("failed to emit extra file %s to .ll: %w", srcFile, err) + } + } + linkInputs = append(linkInputs, bcFile) + } else { + objFile := baseName + ".o" + objArgs := append(baseArgs, "-o", objFile, "-c", srcFile) + if printCmds { + fmt.Fprintf(os.Stderr, "Compiling extra file: clang %s\n", strings.Join(objArgs, " ")) + } + cmd := ctx.compiler() + if err := cmd.Compile(objArgs...); err != nil { + return nil, fmt.Errorf("failed to compile extra file %s: %w", srcFile, err) + } + linkInputs = append(linkInputs, objFile) } - - // Always compile to .o for linking - objFile := baseName + ".o" - objArgs := append(baseArgs, "-o", objFile, "-c", srcFile) - if printCmds { - fmt.Fprintf(os.Stderr, "Compiling extra file: clang %s\n", strings.Join(objArgs, " ")) - } - cmd := ctx.compiler() - if err := cmd.Compile(objArgs...); err != nil { - return nil, fmt.Errorf("failed to compile extra file %s: %w", srcFile, err) - } - - objFiles = append(objFiles, objFile) os.Remove(baseName) // Remove the temp file we created for naming } - return objFiles, nil + return linkInputs, nil } func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPath string, verbose bool) error { @@ -894,10 +1118,11 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa for _, v := range pkgs { allPkgs = append(allPkgs, v.Package) } - // linkInputs contains .a archives from all packages and .o files from main module - var linkInputs []string + // Package link inputs are collected first, then split into bitcode/native + // before the program-level bitcode link and final native link. + var pkgLinkInputs []string var linkArgs []string - var rtLinkInputs []string + var rtPkgLinkInputs []string var rtLinkArgs []string linkedPkgs := make(map[string]bool) // Track linked packages by ID to avoid duplicates packages.Visit(allPkgs, nil, func(p *packages.Package) { @@ -916,9 +1141,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa // Defer linking runtime packages unless we actually need the runtime. if isRuntimePkg(p.PkgPath) { rtLinkArgs = append(rtLinkArgs, aPkg.LinkArgs...) - if aPkg.ArchiveFile != "" { - rtLinkInputs = append(rtLinkInputs, aPkg.ArchiveFile) - } + rtPkgLinkInputs = append(rtPkgLinkInputs, aPkg.ObjFiles...) return } else { // Only let non-runtime packages influence whether runtime is needed. @@ -931,34 +1154,43 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa } linkArgs = append(linkArgs, aPkg.LinkArgs...) - if aPkg.ArchiveFile != "" { - linkInputs = append(linkInputs, aPkg.ArchiveFile) - } + pkgLinkInputs = append(pkgLinkInputs, aPkg.ObjFiles...) } }) // Only link runtime objects when needed (or for host builds where runtime is always required). if needRuntime || needPyInit || ctx.buildConf.Target == "" { linkArgs = append(linkArgs, rtLinkArgs...) - linkInputs = append(linkInputs, rtLinkInputs...) + pkgLinkInputs = append(pkgLinkInputs, rtPkgLinkInputs...) } + linkBitcodeInputs, nativeLinkInputs := splitBitcodeAndNativeInputs(pkgLinkInputs) // Generate main module file (needed for global variables even in library modes) - // This is compiled directly to .o and added to linkInputs (not cached) + // This is compiled to .bc and included in the program-level bitcode link (not cached). // Use a stable synthetic name to avoid confusing it with the real main package in traces/logs. entryPkg := genMainModule(ctx, llssa.PkgRuntime, pkg, needRuntime, needPyInit, needAbiInit) - entryObjFile, err := exportObject(ctx, "entry_main", entryPkg.ExportFile, []byte(entryPkg.LPkg.String())) + entryBitcodeFile, err := exportObject(ctx, "entry_main", entryPkg.ExportFile, []byte(entryPkg.LPkg.String())) if err != nil { return err } - linkInputs = append(linkInputs, entryObjFile) + linkBitcodeInputs = append(linkBitcodeInputs, entryBitcodeFile) // Compile extra files from target configuration - extraObjFiles, err := compileExtraFiles(ctx, verbose) + extraLinkInputs, err := compileExtraFiles(ctx, verbose) if err != nil { return err } - linkInputs = append(linkInputs, extraObjFiles...) + extraBitcodeInputs, extraNativeInputs := splitBitcodeAndNativeInputs(extraLinkInputs) + linkBitcodeInputs = append(linkBitcodeInputs, extraBitcodeInputs...) + nativeLinkInputs = append(nativeLinkInputs, extraNativeInputs...) + + programObjFile, err := compileLinkedBitcodeToObject(ctx, pkg.PkgPath, linkBitcodeInputs, verbose) + if err != nil { + return err + } + if programObjFile != "" { + nativeLinkInputs = append(nativeLinkInputs, programObjFile) + } if IsFullRpathEnabled() { // Treat every link-time library search path, specified by the -L parameter, as a runtime search path as well. @@ -977,7 +1209,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa } } - err = linkObjFiles(ctx, outputPath, linkInputs, linkArgs, verbose) + err = linkObjFiles(ctx, outputPath, nativeLinkInputs, linkArgs, verbose) if err != nil { return err } @@ -1014,31 +1246,37 @@ func linkObjFiles(ctx *context, app string, objFiles, linkArgs []string, verbose buildArgs = append(buildArgs, "-gdwarf-4") } - if ctx.buildConf.GenLL { - var compiledObjFiles []string - for _, objFile := range objFiles { - if strings.HasSuffix(objFile, ".ll") { - oFile := strings.TrimSuffix(objFile, ".ll") + ".o" - args := []string{"-o", oFile, "-c", objFile, "-Wno-override-module"} - if printCmds { - fmt.Fprintln(os.Stderr, "clang", args) - } - if err := ctx.compiler().Compile(args...); err != nil { - return fmt.Errorf("failed to compile %s: %v", objFile, err) - } - compiledObjFiles = append(compiledObjFiles, oFile) - } else { - compiledObjFiles = append(compiledObjFiles, objFile) - } - } - objFiles = compiledObjFiles - } - buildArgs = append(buildArgs, objFiles...) cmd := ctx.linker() cmd.Verbose = printCmds - return cmd.Link(buildArgs...) + if err := cmd.Link(buildArgs...); err != nil { + return err + } + if err := emitDarwinDSYMIfNeeded(ctx, app, printCmds); err != nil { + return err + } + return nil +} + +func emitDarwinDSYMIfNeeded(ctx *context, app string, verbose bool) error { + if !IsDbgSymsEnabled() || ctx.buildConf.Goos != "darwin" || runtime.GOOS != "darwin" { + return nil + } + if ctx.buildConf.BuildMode != BuildModeExe { + return nil + } + dsymutil, err := exec.LookPath("dsymutil") + if err != nil { + return fmt.Errorf("dsymutil not found for debug-symbol build: %w", err) + } + cmd := exec.Command(dsymutil, app) + if verbose { + fmt.Fprintln(os.Stderr, cmd.String()) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() } // archiver returns the archiving tool to use for the current context. @@ -1260,24 +1498,24 @@ func exportObject(ctx *context, pkgPath string, exportFile string, data []byte) if err := os.Chmod(f.Name(), 0644); err != nil { return "", err } - // Copy instead of rename so we can still compile to .o + // Copy instead of rename so we can still compile to .bc if err := copyFileAtomic(f.Name(), llFile); err != nil { return "", err } } - // Always compile .ll to .o for linking - objFile, err := os.CreateTemp("", base+"-*.o") + // Compile .ll to .bc for program-level bitcode linking. + bitcodeFile, err := os.CreateTemp("", base+"-*.bc") if err != nil { return "", err } - objFile.Close() - args := []string{"-o", objFile.Name(), "-c", f.Name(), "-Wno-override-module"} + bitcodeFile.Close() + args := []string{"-o", bitcodeFile.Name(), "-emit-llvm", "-c", f.Name(), "-Wno-override-module"} if ctx.shouldPrintCommands(false) { fmt.Fprintf(os.Stderr, "# compiling %s for pkg: %s\n", f.Name(), pkgPath) fmt.Fprintln(os.Stderr, "clang", args) } cmd := ctx.compiler() - return objFile.Name(), cmd.Compile(args...) + return bitcodeFile.Name(), cmd.Compile(args...) } func llcCheck(env *llvm.Env, exportFile string) (msg string, err error) { @@ -1341,8 +1579,7 @@ type aPackage struct { NeedPyInit bool LinkArgs []string - ObjFiles []string // object files: .o or .ll (output of compiler, input to archiver) - ArchiveFile string // archive file: .a (output of archiver, used for linking) + ObjFiles []string // normalized package link inputs (.bc and optional native .a) rewriteVars map[string]string // Cache related fields @@ -1650,29 +1887,40 @@ func clFiles(ctx *context, files string, pkg *packages.Package, procFile func(li func clFile(ctx *context, args []string, cFile, expFile, pkgPath string, procFile func(linkFile string), verbose bool) { baseName := expFile + filepath.Base(cFile) ext := filepath.Ext(cFile) + compileArgs := slices.Clone(args) // default clang++ will use c++ to compile c file,will cause symbol be mangled if ext == ".c" { - args = append(args, "-x", "c") + compileArgs = append(compileArgs, "-x", "c") } - // If GenLL is enabled, first emit .ll for debugging, then compile to .o + emitBitcode := ext != ".S" && ext != ".s" + printCmds := ctx.shouldPrintCommands(verbose) - if ctx.buildConf.GenLL { - llFile := baseName + ".ll" - llArgs := append(slices.Clone(args), "-emit-llvm", "-S", "-o", llFile, "-c", cFile) + if emitBitcode { + bcFile := baseName + ".bc" + bcArgs := append(compileArgs, "-emit-llvm", "-o", bcFile, "-c", cFile) if printCmds { - fmt.Fprintf(os.Stderr, "# compiling %s for pkg: %s\n", llFile, pkgPath) - fmt.Fprintln(os.Stderr, "clang", llArgs) + fmt.Fprintf(os.Stderr, "# compiling %s for pkg: %s\n", bcFile, pkgPath) + fmt.Fprintln(os.Stderr, "clang", bcArgs) } cmd := ctx.compiler() - err := cmd.Compile(llArgs...) + err := cmd.Compile(bcArgs...) check(err) + if ctx.buildConf.GenLL { + llFile := baseName + ".ll" + if printCmds { + fmt.Fprintf(os.Stderr, "# emitting %s for pkg: %s (from %s)\n", llFile, pkgPath, bcFile) + } + err := writeLLVMIRFromBitcode(bcFile, llFile) + check(err) + } + procFile(bcFile) + return } - // Always compile to .o for linking objFile := baseName + ".o" - objArgs := append(args, "-o", objFile, "-c", cFile) + objArgs := append(compileArgs, "-o", objFile, "-c", cFile) if printCmds { fmt.Fprintf(os.Stderr, "# compiling %s for pkg: %s\n", objFile, pkgPath) fmt.Fprintln(os.Stderr, "clang", objArgs) diff --git a/internal/build/cache.go b/internal/build/cache.go index f8870f6d92..30d25bc9c7 100644 --- a/internal/build/cache.go +++ b/internal/build/cache.go @@ -29,6 +29,7 @@ import ( const ( cacheBuildDirName = "build" + cacheBitcodeExt = ".bc" cacheArchiveExt = ".a" cacheManifestExt = ".manifest" ) @@ -54,7 +55,8 @@ func newCacheManager() *cacheManager { // cachePaths holds the paths for a cached package type cachePaths struct { Dir string // Directory containing cache files - Archive string // Path to .a file + Bitcode string // Path to .bc file + Archive string // Path to optional native .a file Manifest string // Path to .manifest file } @@ -64,6 +66,7 @@ func (cm *cacheManager) PackagePaths(targetTriple, pkgPath, fingerprint string) fingerprint = sanitizeComponent(fingerprint) return cachePaths{ Dir: dir, + Bitcode: filepath.Join(dir, fingerprint+cacheBitcodeExt), Archive: filepath.Join(dir, fingerprint+cacheArchiveExt), Manifest: filepath.Join(dir, fingerprint+cacheManifestExt), } @@ -176,8 +179,8 @@ func readManifest(path string) (string, error) { // cacheExists checks if a valid cache entry exists func (cm *cacheManager) cacheExists(paths cachePaths) bool { - // Both archive and manifest must exist - if _, err := os.Stat(paths.Archive); err != nil { + // Bitcode and manifest must exist. Native archive is optional. + if _, err := os.Stat(paths.Bitcode); err != nil { return false } if _, err := os.Stat(paths.Manifest); err != nil { @@ -211,8 +214,8 @@ func (cm *cacheManager) listCachedPackages(targetTriple, pkgPath string) ([]stri var fingerprints []string for _, entry := range entries { name := entry.Name() - if strings.HasSuffix(name, cacheArchiveExt) { - fp := strings.TrimSuffix(name, cacheArchiveExt) + if strings.HasSuffix(name, cacheBitcodeExt) { + fp := strings.TrimSuffix(name, cacheBitcodeExt) fingerprints = append(fingerprints, fp) } } @@ -239,7 +242,7 @@ func (cm *cacheManager) stats() (cacheStats, error) { } if !info.IsDir() { stats.TotalSize += info.Size() - if strings.HasSuffix(path, cacheArchiveExt) { + if strings.HasSuffix(path, cacheBitcodeExt) { stats.TotalPackages++ } } diff --git a/internal/build/cache_test.go b/internal/build/cache_test.go index 6c6d4a0eb2..b006089dd5 100644 --- a/internal/build/cache_test.go +++ b/internal/build/cache_test.go @@ -64,6 +64,10 @@ func TestCacheManager_PackagePaths(t *testing.T) { if paths.Archive != expectedArchive { t.Errorf("Archive = %q, want %q", paths.Archive, expectedArchive) } + expectedBitcode := filepath.Join(expectedDir, "abc123.bc") + if paths.Bitcode != expectedBitcode { + t.Errorf("Bitcode = %q, want %q", paths.Bitcode, expectedBitcode) + } expectedManifest := filepath.Join(expectedDir, "abc123.manifest") if paths.Manifest != expectedManifest { @@ -167,7 +171,7 @@ func TestCacheManager_CacheExists(t *testing.T) { if err := cm.EnsureDir(paths); err != nil { t.Fatal(err) } - os.WriteFile(paths.Archive, []byte("archive"), 0644) + os.WriteFile(paths.Bitcode, []byte("bitcode"), 0644) // Still should not exist (manifest missing) if cm.cacheExists(paths) { @@ -179,7 +183,7 @@ func TestCacheManager_CacheExists(t *testing.T) { // Now should exist if !cm.cacheExists(paths) { - t.Error("cache should exist with both files") + t.Error("cache should exist with bitcode and manifest files") } } @@ -215,7 +219,7 @@ func TestCacheManager_CleanPackageCache(t *testing.T) { // Create cache cm.EnsureDir(paths) - os.WriteFile(paths.Archive, []byte("archive"), 0644) + os.WriteFile(paths.Bitcode, []byte("bitcode"), 0644) os.WriteFile(paths.Manifest, []byte("manifest"), 0644) // Clean @@ -243,8 +247,8 @@ func TestCacheManager_CleanAllCache(t *testing.T) { cm.EnsureDir(paths1) cm.EnsureDir(paths2) - os.WriteFile(paths1.Archive, []byte("1"), 0644) - os.WriteFile(paths2.Archive, []byte("2"), 0644) + os.WriteFile(paths1.Bitcode, []byte("1"), 0644) + os.WriteFile(paths2.Bitcode, []byte("2"), 0644) // Clean all if err := cm.cleanAllCache(); err != nil { @@ -278,8 +282,8 @@ func TestCacheManager_ListCachedPackages(t *testing.T) { paths1 := cm.PackagePaths("arm64-darwin", "test/pkg", "fp1") paths2 := cm.PackagePaths("arm64-darwin", "test/pkg", "fp2") cm.EnsureDir(paths1) - os.WriteFile(paths1.Archive, []byte("1"), 0644) - os.WriteFile(paths2.Archive, []byte("2"), 0644) + os.WriteFile(paths1.Bitcode, []byte("1"), 0644) + os.WriteFile(paths2.Bitcode, []byte("2"), 0644) fps, err = cm.listCachedPackages("arm64-darwin", "test/pkg") if err != nil { @@ -304,10 +308,10 @@ func TestCacheManager_Stats(t *testing.T) { cm.EnsureDir(paths1) cm.EnsureDir(paths2) - content1 := []byte("archive content 1") - content2 := []byte("archive content 2 longer") - os.WriteFile(paths1.Archive, content1, 0644) - os.WriteFile(paths2.Archive, content2, 0644) + content1 := []byte("bitcode content 1") + content2 := []byte("bitcode content 2 longer") + os.WriteFile(paths1.Bitcode, content1, 0644) + os.WriteFile(paths2.Bitcode, content2, 0644) os.WriteFile(paths1.Manifest, []byte("m1"), 0644) os.WriteFile(paths2.Manifest, []byte("m2"), 0644) diff --git a/internal/build/collect.go b/internal/build/collect.go index bdd1977cd0..41093cf851 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -329,8 +329,8 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { cm := c.ensureCacheManager() paths := cm.PackagePaths(c.targetTriple(), pkg.PkgPath, pkg.Fingerprint) - // Check if archive file exists - if _, err := os.Stat(paths.Archive); err != nil { + // Check if bitcode cache exists. + if _, err := os.Stat(paths.Bitcode); err != nil { return false } @@ -346,8 +346,11 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { return false } - // Use the .a archive directly for linking (no extraction needed) - pkg.ArchiveFile = paths.Archive + // Use cached package link inputs for final linking. + pkg.ObjFiles = []string{paths.Bitcode} + if _, err := os.Stat(paths.Archive); err == nil { + pkg.ObjFiles = append(pkg.ObjFiles, paths.Archive) + } pkg.LinkArgs = meta.LinkArgs pkg.NeedRt = meta.NeedRt pkg.NeedPyInit = meta.NeedPyInit @@ -444,18 +447,30 @@ func (c *context) saveToCache(pkg *aPackage) error { return err } - // If ArchiveFile is already set (from normalizeToArchive), copy it to cache - if pkg.ArchiveFile != "" { - if err := copyFileAtomic(pkg.ArchiveFile, paths.Archive); err != nil { - return err + bitcodeFiles, nativeInputs := splitBitcodeAndNativeInputs(pkg.ObjFiles) + // Package bitcode is mandatory for bitcode LTO cache entries. + if len(bitcodeFiles) == 0 { + return nil + } + pkgBitcode := bitcodeFiles[0] + if len(bitcodeFiles) > 1 { + merged, err := mergeBitcodeFiles(pkg.PkgPath, bitcodeFiles) + if err != nil { + return fmt.Errorf("merge bitcode for cache %s: %w", pkg.PkgPath, err) } - } else if len(pkg.ObjFiles) > 0 { - // Otherwise, create archive from object files - if err := c.createArchiveFile(paths.Archive, pkg.ObjFiles); err != nil { + pkgBitcode = merged + } + if err := copyFileAtomic(pkgBitcode, paths.Bitcode); err != nil { + return err + } + if len(nativeInputs) > 0 { + if len(nativeInputs) == 1 && strings.HasSuffix(nativeInputs[0], ".a") { + if err := copyFileAtomic(nativeInputs[0], paths.Archive); err != nil { + return err + } + } else if err := c.createArchiveFile(paths.Archive, nativeInputs); err != nil { return err } - } else { - return nil } // Append metadata to existing manifest (pkg.Manifest was built in collectFingerprint). diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index 1c859c919b..ae0b7e5ce6 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -411,15 +411,14 @@ func TestTryLoadFromCache_ForceRebuild(t *testing.T) { }(), } - // Create a temporary .o file - objFile, err := os.CreateTemp(td, "test-*.o") + // Create a temporary .bc file + bcFile, err := os.CreateTemp(td, "test-*.bc") if err != nil { t.Fatalf("CreateTemp: %v", err) } - objFile.WriteString("fake object file") - objFile.Close() - - pkg.ObjFiles = []string{objFile.Name()} + bcFile.WriteString("fake bitcode file") + bcFile.Close() + pkg.ObjFiles = []string{bcFile.Name()} // First save to cache ctx.buildConf.ForceRebuild = false @@ -430,16 +429,15 @@ func TestTryLoadFromCache_ForceRebuild(t *testing.T) { // Verify cache exists cm := ctx.ensureCacheManager() paths := cm.PackagePaths("arm64-apple-darwin", "example.com/cached", "test123") - if _, err := os.Stat(paths.Archive); err != nil { + if _, err := os.Stat(paths.Bitcode); err != nil { t.Fatalf("cache should exist: %v", err) } // Now enable ForceRebuild and try to load ctx.buildConf.ForceRebuild = true - // Clear ObjFiles to verify it's not loaded from cache + // Clear fields to verify they are not loaded from cache. pkg.ObjFiles = nil - pkg.ArchiveFile = "" pkg.CacheHit = false if ctx.tryLoadFromCache(pkg) { @@ -450,8 +448,8 @@ func TestTryLoadFromCache_ForceRebuild(t *testing.T) { t.Error("CacheHit should remain false when ForceRebuild is enabled") } - if pkg.ArchiveFile != "" { - t.Error("ArchiveFile should not be populated when ForceRebuild is enabled") + if len(pkg.ObjFiles) != 0 { + t.Error("ObjFiles should not be populated when ForceRebuild is enabled") } } @@ -511,19 +509,19 @@ func TestSaveToCache_Success(t *testing.T) { }, } - // Create a temporary .o file - objFile, err := os.CreateTemp(td, "test-*.o") + // Create a temporary .bc file + bcFile, err := os.CreateTemp(td, "test-*.bc") if err != nil { t.Fatalf("CreateTemp: %v", err) } - objFile.WriteString("fake object file") - objFile.Close() + bcFile.WriteString("fake bitcode file") + bcFile.Close() pkg := &aPackage{ Package: &packages.Package{ PkgPath: "example.com/lib", Name: "lib", - GoFiles: []string{objFile.Name()}, // Add GoFiles for manifest generation + GoFiles: []string{bcFile.Name()}, // Add GoFiles for manifest generation }, Fingerprint: "def456", Manifest: func() string { @@ -532,7 +530,7 @@ func TestSaveToCache_Success(t *testing.T) { m.pkg.PkgPath = "example.com/lib" return m.Build() }(), - ObjFiles: []string{objFile.Name()}, + ObjFiles: []string{bcFile.Name()}, } if err := ctx.saveToCache(pkg); err != nil { @@ -559,9 +557,9 @@ func TestSaveToCache_Success(t *testing.T) { t.Errorf("metadata should be empty when no link args/runtime flags") } - // Check archive exists - if _, err := os.Stat(paths.Archive); err != nil { - t.Errorf("archive should exist: %v", err) + // Check bitcode exists + if _, err := os.Stat(paths.Bitcode); err != nil { + t.Errorf("bitcode should exist: %v", err) } } diff --git a/ssa/eh.go b/ssa/eh.go index 841018d353..e03f412848 100644 --- a/ssa/eh.go +++ b/ssa/eh.go @@ -149,6 +149,16 @@ func (b Builder) Longjmp(jb, retval Expr) { func (p Function) deferInitBuilder() (b Builder, next BasicBlock) { b = p.NewBuilder() + // getDefer may switch to a fresh builder when wiring defer prologue. + // Seed a valid function-scope debug location so synthesized calls carry + // !dbg in debug builds (LLVM verifies this for inlinable calls). + if sp := p.impl.Subprogram(); sp.C != nil { + line := sp.SubprogramLine() + if line == 0 { + line = 1 + } + b.impl.SetCurrentDebugLocation(line, 0, sp, llvm.Metadata{}) + } next = b.setBlockMoveLast(p.blks[0]) p.blks[0].last = next.last return diff --git a/ssa/stmt_builder.go b/ssa/stmt_builder.go index b5671a9ae6..e2d9aa8589 100644 --- a/ssa/stmt_builder.go +++ b/ssa/stmt_builder.go @@ -130,6 +130,17 @@ func (b Builder) SetBlockEx(blk BasicBlock, pos InsertPoint, setBlk bool) { default: panic("SetBlockEx: invalid pos") } + // Some synthesized instructions (for example defer lowering) may be emitted + // without a source instruction position. Seed a function-scope debug + // location after changing insert point so those instructions still carry a + // valid !dbg in debug builds. + if sp := b.Func.impl.Subprogram(); sp.C != nil { + line := sp.SubprogramLine() + if line == 0 { + line = 1 + } + b.impl.SetCurrentDebugLocation(line, 0, sp, llvm.Metadata{}) + } if setBlk { b.blk = blk }