Microsoft/dev.boringcrypto.go1.17

VERSION

@@ -0,0 +1 @@
+go1.17.13

eng/_core/archive/archive.go

@@ -16,12 +16,49 @@ import (
 	"io"
 	"io/fs"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
 )
 
+// CreateFromSource runs a Git command to generate an archive file from the Go source code at
+// "source". If output is "", the archive is produced in the build directory inside the
+// "eng/artifacts/bin" directory. A checksum file is also produced.
+func CreateFromSource(source string, output string) error {
+	fmt.Printf("---- Creating Go source archive (tarball) from '%v'...\n", source)
+
+	if output == "" {
+		output = filepath.Join(getBinDir(source), fmt.Sprintf("go.%v.src.tar.gz", getBuildID()))
+	}
+
+	// Ensure the target directory exists.
+	archiveDir := filepath.Dir(output)
+	if err := os.MkdirAll(archiveDir, os.ModeDir|os.ModePerm); err != nil {
+		return err
+	}
+
+	// Use "^{tree}" to avoid Git including a global extended pax header. The commit it would list
+	// is a temporary commit, and would only be confusing. See https://git-scm.com/docs/git-archive.
+	cmd := exec.Command("git", "archive", "-o", output, "--prefix=go/", "HEAD^{tree}")
+	cmd.Dir = source
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	fmt.Printf("---- Running command: %v\n", cmd.Args)
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	fmt.Printf("---- Creating checksum file...\n")
+	if err := writeSHA256ChecksumFile(output); err != nil {
+		return err
+	}
+
+	fmt.Printf("---- Pack complete.\n")
+	return nil
+}
+
 // CreateFromBuild walks the Go build directory at "source" and produces an archive with the path
 // "output". If output is "", CreateFromBuild produces a file in the build directory inside the
 // "eng/artifacts/bin" directory. The output directory is created if it doesn't exist. This function
@@ -30,25 +67,14 @@ import (
 // The inclusion of some files depends on the OS/ARCH. If output is specified, its filename must
 // follow the pattern "*{GOOS}-{GOARCH}{extension}" so OS and ARCH can be detected. If output is not
 // specified, the current Go runtime's OS and ARCH are used.
-func CreateFromBuild(source string, output string) error {
+//
+// If runtime.GOOS and runtime.GOARCH don't match the OS/ARCH of the output file name, the build
+// directory is treated as a cross-compiled build of Go.
+func CreateFromBuild(source, output string) error {
 	fmt.Printf("---- Creating Go archive (zip/tarball) from '%v'...\n", source)
 
 	if output == "" {
-		// If BUILD_BUILDNUMBER is defined (e.g. a CI build), use it. For local builds, use "dev".
-		archiveVersion := os.Getenv("BUILD_BUILDNUMBER")
-		if archiveVersion == "" {
-			archiveVersion = "dev"
-		}
-
-		archiveExtension := ".tar.gz"
-		if runtime.GOOS == "windows" {
-			archiveExtension = ".zip"
-		}
-
-		archiveName := fmt.Sprintf("go.%v.%v-%v%v", archiveVersion, runtime.GOOS, runtime.GOARCH, archiveExtension)
-		binDir := filepath.Join(source, "..", "eng", "artifacts", "bin")
-
-		output = filepath.Join(binDir, archiveName)
+		output = DefaultBuildOutputPath(source, "", "")
 	}
 
 	// Ensure the target directory exists.
@@ -81,6 +107,22 @@ func CreateFromBuild(source string, output string) error {
 		filepath.Join("pkg", "tool", os+"_"+arch, "api.exe"),
 	}
 
+	hostOS := runtime.GOOS
+	hostArch := runtime.GOARCH
+	if hostOS != os || hostArch != arch {
+		fmt.Printf("Handling cross-compile: %v-%v host to %v-%v target\n", hostOS, hostArch, os, arch)
+		skipPaths = append(skipPaths, []string{
+			// Don't include binaries that were built for the host toolchain.
+			filepath.Join("pkg", hostOS+"_"+hostArch),
+			filepath.Join("pkg", "tool", hostOS+"_"+hostArch),
+			// Don't include the host binaries: the target binaries are in a subdir.
+			filepath.Join("bin", "go"),
+			filepath.Join("bin", "go.exe"),
+			filepath.Join("bin", "gofmt"),
+			filepath.Join("bin", "gofmt.exe"),
+		}...)
+	}
+
 	// Figure out what the race detection syso (precompiled binary) is named for the current
 	// os/arch. We want to exclude all race syso files other than this one.
 	targetRuntimeRaceSyso := fmt.Sprintf("race_%v_%v.syso", os, arch)
@@ -89,17 +131,26 @@ func CreateFromBuild(source string, output string) error {
 	// data the script has processed to avoid appearing unresponsive.
 	lastProgressUpdate := time.Now()
 
-	filepath.WalkDir(source, func(path string, info fs.DirEntry, err error) error {
+	// Keep track of target paths added to the archive (key) and the source path that it comes from
+	// (value). This map ensures no target files are double-added.
+	addedPaths := make(map[string]string)
+	// Keep track of dir entries that have been added.
+	addedDirs := make(map[string]struct{})
+
+	err := filepath.WalkDir(source, func(path string, info fs.DirEntry, err error) error {
 		if err != nil {
 			fmt.Printf("Failure accessing a path %q: %v\n", path, err)
 			return err
 		}
 
 		relPath, err := filepath.Rel(source, path)
 		if err != nil {
-			panic(err)
+			return err
 		}
 
+		// targetPath is where relPath should be placed in the output archive, if it belongs inside.
+		targetPath := relPath
+
 		// Walk every dir/file in the root of the repository.
 		if relPath == "." {
 			// Ignore the rest of the logic in this func by returning nil early.
@@ -138,6 +189,11 @@ func CreateFromBuild(source string, output string) error {
 				}
 			}
 
+			// Include "bin/{OS}_{ARCH}/go" as "bin/go" if this is a cross-compilation build.
+			if filepath.Dir(relPath) == filepath.Join("bin", os+"_"+arch) {
+				targetPath = filepath.Join("bin", filepath.Base(relPath))
+			}
+
 			// Skip race detection syso file if it doesn't match the target runtime.
 			//
 			// Ignore error: the only possible error is one that says the pattern is invalid (see
@@ -151,16 +207,52 @@ func CreateFromBuild(source string, output string) error {
 
 		if info.IsDir() {
 			// We want to continue the recursive search in this directory for more files, but we
-			// don't need to add it to the archive. Return nil to continue.
+			// don't necessarily want to add this dir to the archive. Return nil to continue.
 			return nil
 		}
 
-		// At this point, we know "path" is a file that should be included. Add it.
-		archiver.AddFile(
-			path,
-			// Store everything in a root "go" directory to match upstream Go archives.
-			filepath.Join("go", relPath),
-		)
+		if relPath != targetPath {
+			fmt.Printf("Archiving %#q as %#q\n", relPath, targetPath)
+		}
+
+		if otherSource, ok := addedPaths[targetPath]; ok {
+			return fmt.Errorf(
+				"adding archive file %#q from %#q, but target already added from %#q",
+				targetPath, relPath, otherSource)
+		}
+		addedPaths[targetPath] = relPath
+
+		// At this point, we know "path" is a file that should be included. Add it. Store everything
+		// in a root "go" directory to match upstream Go archives.
+		goTargetPath := filepath.Join("go", targetPath)
+		if err := archiver.AddFile(path, goTargetPath); err != nil {
+			return err
+		}
+
+		// Add all dirs that are ancestors of this target file. Even though explicitly added dirs
+		// aren't necessary to create a valid archive file, upstream does this, so we do too. This
+		// reduces the diff, e.g. when comparing results with tools like "tar -tf".
+		dir := goTargetPath
+		for {
+			dir = filepath.Dir(dir)
+			if dir == "." {
+				break
+			}
+			if dir == "/" {
+				return fmt.Errorf("unexpected rooted target path: %#q", goTargetPath)
+			}
+
+			if _, ok := addedDirs[dir]; ok {
+				break
+			}
+			// Use root repository dir as a stand-in for any filesystem information. This is simpler
+			// than reproducing the actual dir's path based on the target path, especially
+			// considering the target path may not match up with a directory that actually exists.
+			if err := archiver.AddFile(source, dir); err != nil {
+				return err
+			}
+			addedDirs[dir] = struct{}{}
+		}
 
 		// If it's been long enough, log an update on our progress.
 		now := time.Now()
@@ -174,6 +266,9 @@ func CreateFromBuild(source string, output string) error {
 
 		return nil
 	})
+	if err != nil {
+		return err
+	}
 
 	fmt.Printf(
 		"Complete! %v (%v kB uncompressed data archived)\n",
@@ -186,20 +281,61 @@ func CreateFromBuild(source string, output string) error {
 	}
 
 	fmt.Printf("---- Creating checksum file...\n")
-	writeSHA256ChecksumFile(output)
+	if err := writeSHA256ChecksumFile(output); err != nil {
+		return err
+	}
 
 	fmt.Printf("---- Pack complete.\n")
 	return nil
 }
 
+// DefaultBuildOutputPath returns the default path to place the output archive given a built Go
+// directory. Optionally takes os and arch, or detects their values from the environment and runtime
+// if empty string.
+func DefaultBuildOutputPath(source, os, arch string) string {
+	if os == "" {
+		os = runtime.GOOS
+	}
+	if arch == "" {
+		arch = runtime.GOARCH
+	}
+	// Add "v6l" suffix to "arm" arch. More robust handling would be necessary if there were
+	// multiple "arm" builds with GOARM=6 and GOARM=7, but there are not, and "arm64" obsoletes it.
+	if arch == "arm" {
+		arch += "v6l"
+	}
+
+	ext := ".tar.gz"
+	if runtime.GOOS == "windows" {
+		ext = ".zip"
+	}
+
+	archiveName := fmt.Sprintf("go.%v.%v-%v%v", getBuildID(), os, arch, ext)
+	return filepath.Join(getBinDir(source), archiveName)
+}
+
+// getBuildID returns BUILD_BUILDNUMBER if defined (e.g. a CI build). Otherwise, "dev".
+func getBuildID() string {
+	archiveVersion := os.Getenv("BUILD_BUILDNUMBER")
+	if archiveVersion == "" {
+		return "dev"
+	}
+	return archiveVersion
+}
+
+func getBinDir(source string) string {
+	return filepath.Join(source, "..", "eng", "artifacts", "bin")
+}
+
 // getArchivePathRuntime takes a path like "go1.7.linux-amd64.tar.gz" and extension like ".tar.gz",
 // and returns the os (linux) and arch (amd64). The "path" extension may have multiple '.'
 // characters in it, so "ext" must be passed in explicitly or else the match would be ambiguous.
 func getArchivePathRuntime(path string, ext string) (os string, arch string) {
 	pathNoExt := path[0 : len(path)-len(ext)]
 	firstRuntimeIndex := strings.LastIndex(pathNoExt, ".") + 1
 	osArch := strings.Split(pathNoExt[firstRuntimeIndex:], "-")
-	return osArch[0], osArch[1]
+	// "v6l" is added to the end of the "arm" arch filename, but is not part of the arch. Remove it.
+	return osArch[0], strings.TrimSuffix(osArch[1], "v6l")
 }
 
 // writeSHA256ChecksumFile reads the content of the file at the given path into a SHA256 hasher, and

eng/_core/archive/writer.go

@@ -88,14 +88,20 @@ func (a *tarGzArchiver) AddFile(filePath string, archivePath string) error {
 
 	// tar.FileInfoHeader only takes base name, so set full path here. See FileInfoHeader doc.
 	header.Name = archivePath
+	if stat.IsDir() {
+		header.Name += "/"
+	}
 
 	if err := a.tarWriter.WriteHeader(header); err != nil {
 		return err
 	}
 
-	n, err := io.Copy(a.tarWriter, fileReader)
-	a.processedBytes += n
-	return err
+	if !stat.IsDir() {
+		n, err := io.Copy(a.tarWriter, fileReader)
+		a.processedBytes += n
+		return err
+	}
+	return nil
 }
 
 func (a *tarGzArchiver) Close() error {
@@ -129,20 +135,36 @@ func newZipArchiver(path string) *zipArchiver {
 }
 
 func (a *zipArchiver) AddFile(filePath string, archivePath string) error {
-	archiveFileWriter, err := a.writer.Create(archivePath)
+	fileReader, err := os.Open(filePath)
 	if err != nil {
 		return err
 	}
+	defer fileReader.Close()
 
-	fileReader, err := os.Open(filePath)
+	stat, err := fileReader.Stat()
 	if err != nil {
 		return err
 	}
-	defer fileReader.Close()
 
-	n, err := io.Copy(archiveFileWriter, fileReader)
-	a.processedBytes += n
-	return err
+	// Upstream Go uses "/" for archive dir separators, even on Windows.
+	archivePath = filepath.ToSlash(archivePath)
+
+	// Give dirs a trailing forward slash to indicate to the zip writer that it's a dir.
+	if stat.IsDir() {
+		archivePath += "/"
+	}
+
+	archiveFileWriter, err := a.writer.Create(archivePath)
+	if err != nil {
+		return err
+	}
+
+	if !stat.IsDir() {
+		n, err := io.Copy(archiveFileWriter, fileReader)
+		a.processedBytes += n
+		return err
+	}
+	return nil
 }
 
 func (a *zipArchiver) Close() error {