diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index b16031e3a10..00000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# This is a CodeQL analysis job that runs on each PR to point out whether it -# adds new potentially insecure code patterns. It also runs on a periodic basis -# to analyze the checked-in code. - -# For more overall info about CodeQL: https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning -# More about the CodeQL actions: https://github.com/github/codeql-action -# OneNote page with more internal info: https://microsoft.sharepoint.com/teams/managedlanguages/_layouts/OneNote.aspx?id=%2Fteams%2Fmanagedlanguages%2Ffiles%2FTeam%20Notebook%2FGoLang%20Team&wd=target%28Main.one%7C62B655D4-14E7-41D6-A063-0869C28D63FC%2FSDL%20Tools%7C3908F727-3751-4ACC-8C71-6CEB2DF277B4%2F%29 - -name: "CodeQL" - -on: - push: - branches: [ microsoft/* ] - pull_request: - branches: [ microsoft/* ] - schedule: - # Run at 08:39 UTC each Thursday. - - cron: '39 8 * * 4' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'cpp', 'go' ] - - env: - # Instead of running "go build" during the analysis step, instrument our custom build. - CODEQL_EXTRACTOR_GO_BUILD_TRACING: "on" - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - - # Custom build command. The Go repo itself doesn't have a module at the - # root, so typical Go module build commands don't work. - - run: | - pwsh eng/run.ps1 build -refresh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000000..30a6944a828 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This job tests that the patches apply cleanly, and nothing else. The goal is to test this as +# quickly as possible. This job is a good signal for devs: GitHub Actions is quick to get an agent, +# so when this job fails, it's easy to see, and clear to the dev that the rest of the PR's jobs +# aren't going to succeed and can be ignored. +# +# Ideally, failure of this job would block the tests from running, because it would be a waste of +# time to hit the patch failure N times. However, the actual tests run in AzDO, so we can't +# reasonably cancel them from here (GitHub Actions). + +name: "Test" + +on: + pull_request: + branches: [ microsoft/* ] + +jobs: + check_patches: + name: Patches Apply Cleanly + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - run: pwsh eng/run.ps1 submodule-refresh -shallow diff --git a/VERSION b/VERSION new file mode 100644 index 00000000000..a246dd81516 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +go1.18.10 \ No newline at end of file diff --git a/eng/_core/README.md b/eng/_core/README.md index bffc0a923d3..02b5cd0e4ff 100644 --- a/eng/_core/README.md +++ b/eng/_core/README.md @@ -16,7 +16,7 @@ output and stderr redirection to stdout. The high-level execution flow looks roughly like this when running in CI: -* `eng/pipeline/jobs/run-job.yml` +* `eng/pipeline/jobs/run-stage.yml` runs: * `eng/run.ps1 run-builder -builder linux-amd64-test -junitfile [...]` which runs the Go function: diff --git a/eng/_core/archive/archive.go b/eng/_core/archive/archive.go index 213c874e5ed..e514db511a2 100644 --- a/eng/_core/archive/archive.go +++ b/eng/_core/archive/archive.go @@ -67,18 +67,14 @@ func CreateFromSource(source string, output string) error { // The inclusion of some files depends on the OS/ARCH. If output is specified, its filename must // follow the pattern "*{GOOS}-{GOARCH}{extension}" so OS and ARCH can be detected. If output is not // specified, the current Go runtime's OS and ARCH are used. -func CreateFromBuild(source string, output string) error { +// +// If runtime.GOOS and runtime.GOARCH don't match the OS/ARCH of the output file name, the build +// directory is treated as a cross-compiled build of Go. +func CreateFromBuild(source, output string) error { fmt.Printf("---- Creating Go archive (zip/tarball) from '%v'...\n", source) if output == "" { - archiveVersion := getBuildID() - archiveExtension := ".tar.gz" - if runtime.GOOS == "windows" { - archiveExtension = ".zip" - } - - archiveName := fmt.Sprintf("go.%v.%v-%v%v", archiveVersion, runtime.GOOS, runtime.GOARCH, archiveExtension) - output = filepath.Join(getBinDir(source), archiveName) + output = DefaultBuildOutputPath(source, "", "") } // Ensure the target directory exists. @@ -111,6 +107,22 @@ func CreateFromBuild(source string, output string) error { filepath.Join("pkg", "tool", os+"_"+arch, "api.exe"), } + hostOS := runtime.GOOS + hostArch := runtime.GOARCH + if hostOS != os || hostArch != arch { + fmt.Printf("Handling cross-compile: %v-%v host to %v-%v target\n", hostOS, hostArch, os, arch) + skipPaths = append(skipPaths, []string{ + // Don't include binaries that were built for the host toolchain. + filepath.Join("pkg", hostOS+"_"+hostArch), + filepath.Join("pkg", "tool", hostOS+"_"+hostArch), + // Don't include the host binaries: the target binaries are in a subdir. + filepath.Join("bin", "go"), + filepath.Join("bin", "go.exe"), + filepath.Join("bin", "gofmt"), + filepath.Join("bin", "gofmt.exe"), + }...) + } + // Figure out what the race detection syso (precompiled binary) is named for the current // os/arch. We want to exclude all race syso files other than this one. targetRuntimeRaceSyso := fmt.Sprintf("race_%v_%v.syso", os, arch) @@ -119,7 +131,13 @@ func CreateFromBuild(source string, output string) error { // data the script has processed to avoid appearing unresponsive. lastProgressUpdate := time.Now() - filepath.WalkDir(source, func(path string, info fs.DirEntry, err error) error { + // Keep track of target paths added to the archive (key) and the source path that it comes from + // (value). This map ensures no target files are double-added. + addedPaths := make(map[string]string) + // Keep track of dir entries that have been added. + addedDirs := make(map[string]struct{}) + + err := filepath.WalkDir(source, func(path string, info fs.DirEntry, err error) error { if err != nil { fmt.Printf("Failure accessing a path %q: %v\n", path, err) return err @@ -127,9 +145,12 @@ func CreateFromBuild(source string, output string) error { relPath, err := filepath.Rel(source, path) if err != nil { - panic(err) + return err } + // targetPath is where relPath should be placed in the output archive, if it belongs inside. + targetPath := relPath + // Walk every dir/file in the root of the repository. if relPath == "." { // Ignore the rest of the logic in this func by returning nil early. @@ -168,6 +189,11 @@ func CreateFromBuild(source string, output string) error { } } + // Include "bin/{OS}_{ARCH}/go" as "bin/go" if this is a cross-compilation build. + if filepath.Dir(relPath) == filepath.Join("bin", os+"_"+arch) { + targetPath = filepath.Join("bin", filepath.Base(relPath)) + } + // Skip race detection syso file if it doesn't match the target runtime. // // Ignore error: the only possible error is one that says the pattern is invalid (see @@ -181,16 +207,52 @@ func CreateFromBuild(source string, output string) error { if info.IsDir() { // We want to continue the recursive search in this directory for more files, but we - // don't need to add it to the archive. Return nil to continue. + // don't necessarily want to add this dir to the archive. Return nil to continue. return nil } - // At this point, we know "path" is a file that should be included. Add it. - archiver.AddFile( - path, - // Store everything in a root "go" directory to match upstream Go archives. - filepath.Join("go", relPath), - ) + if relPath != targetPath { + fmt.Printf("Archiving %#q as %#q\n", relPath, targetPath) + } + + if otherSource, ok := addedPaths[targetPath]; ok { + return fmt.Errorf( + "adding archive file %#q from %#q, but target already added from %#q", + targetPath, relPath, otherSource) + } + addedPaths[targetPath] = relPath + + // At this point, we know "path" is a file that should be included. Add it. Store everything + // in a root "go" directory to match upstream Go archives. + goTargetPath := filepath.Join("go", targetPath) + if err := archiver.AddFile(path, goTargetPath); err != nil { + return err + } + + // Add all dirs that are ancestors of this target file. Even though explicitly added dirs + // aren't necessary to create a valid archive file, upstream does this, so we do too. This + // reduces the diff, e.g. when comparing results with tools like "tar -tf". + dir := goTargetPath + for { + dir = filepath.Dir(dir) + if dir == "." { + break + } + if dir == "/" { + return fmt.Errorf("unexpected rooted target path: %#q", goTargetPath) + } + + if _, ok := addedDirs[dir]; ok { + break + } + // Use root repository dir as a stand-in for any filesystem information. This is simpler + // than reproducing the actual dir's path based on the target path, especially + // considering the target path may not match up with a directory that actually exists. + if err := archiver.AddFile(source, dir); err != nil { + return err + } + addedDirs[dir] = struct{}{} + } // If it's been long enough, log an update on our progress. now := time.Now() @@ -204,6 +266,9 @@ func CreateFromBuild(source string, output string) error { return nil }) + if err != nil { + return err + } fmt.Printf( "Complete! %v (%v kB uncompressed data archived)\n", @@ -224,6 +289,31 @@ func CreateFromBuild(source string, output string) error { return nil } +// DefaultBuildOutputPath returns the default path to place the output archive given a built Go +// directory. Optionally takes os and arch, or detects their values from the environment and runtime +// if empty string. +func DefaultBuildOutputPath(source, os, arch string) string { + if os == "" { + os = runtime.GOOS + } + if arch == "" { + arch = runtime.GOARCH + } + // Add "v6l" suffix to "arm" arch. More robust handling would be necessary if there were + // multiple "arm" builds with GOARM=6 and GOARM=7, but there are not, and "arm64" obsoletes it. + if arch == "arm" { + arch += "v6l" + } + + ext := ".tar.gz" + if runtime.GOOS == "windows" { + ext = ".zip" + } + + archiveName := fmt.Sprintf("go.%v.%v-%v%v", getBuildID(), os, arch, ext) + return filepath.Join(getBinDir(source), archiveName) +} + // getBuildID returns BUILD_BUILDNUMBER if defined (e.g. a CI build). Otherwise, "dev". func getBuildID() string { archiveVersion := os.Getenv("BUILD_BUILDNUMBER") @@ -244,7 +334,8 @@ func getArchivePathRuntime(path string, ext string) (os string, arch string) { pathNoExt := path[0 : len(path)-len(ext)] firstRuntimeIndex := strings.LastIndex(pathNoExt, ".") + 1 osArch := strings.Split(pathNoExt[firstRuntimeIndex:], "-") - return osArch[0], osArch[1] + // "v6l" is added to the end of the "arm" arch filename, but is not part of the arch. Remove it. + return osArch[0], strings.TrimSuffix(osArch[1], "v6l") } // writeSHA256ChecksumFile reads the content of the file at the given path into a SHA256 hasher, and diff --git a/eng/_core/archive/writer.go b/eng/_core/archive/writer.go index fd2c55d5be1..3596ebdd0ce 100644 --- a/eng/_core/archive/writer.go +++ b/eng/_core/archive/writer.go @@ -88,14 +88,20 @@ func (a *tarGzArchiver) AddFile(filePath string, archivePath string) error { // tar.FileInfoHeader only takes base name, so set full path here. See FileInfoHeader doc. header.Name = archivePath + if stat.IsDir() { + header.Name += "/" + } if err := a.tarWriter.WriteHeader(header); err != nil { return err } - n, err := io.Copy(a.tarWriter, fileReader) - a.processedBytes += n - return err + if !stat.IsDir() { + n, err := io.Copy(a.tarWriter, fileReader) + a.processedBytes += n + return err + } + return nil } func (a *tarGzArchiver) Close() error { @@ -129,20 +135,36 @@ func newZipArchiver(path string) *zipArchiver { } func (a *zipArchiver) AddFile(filePath string, archivePath string) error { - archiveFileWriter, err := a.writer.Create(archivePath) + fileReader, err := os.Open(filePath) if err != nil { return err } + defer fileReader.Close() - fileReader, err := os.Open(filePath) + stat, err := fileReader.Stat() if err != nil { return err } - defer fileReader.Close() - n, err := io.Copy(archiveFileWriter, fileReader) - a.processedBytes += n - return err + // Upstream Go uses "/" for archive dir separators, even on Windows. + archivePath = filepath.ToSlash(archivePath) + + // Give dirs a trailing forward slash to indicate to the zip writer that it's a dir. + if stat.IsDir() { + archivePath += "/" + } + + archiveFileWriter, err := a.writer.Create(archivePath) + if err != nil { + return err + } + + if !stat.IsDir() { + n, err := io.Copy(archiveFileWriter, fileReader) + a.processedBytes += n + return err + } + return nil } func (a *zipArchiver) Close() error { diff --git a/eng/_core/buildutil/buildutil.go b/eng/_core/buildutil/buildutil.go new file mode 100644 index 00000000000..bd0b770d4de --- /dev/null +++ b/eng/_core/buildutil/buildutil.go @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package buildutil + +import ( + "fmt" + "log" + "os" + "strconv" +) + +// Retry runs f until it succeeds or the attempt limit is reached. +func Retry(attempts int, f func() error) error { + var i = 0 + for ; i < attempts; i++ { + if attempts > 1 { + fmt.Printf("---- Running attempt %v of %v...\n", i+1, attempts) + } + err := f() + if err != nil { + if i+1 < attempts { + fmt.Printf("---- Attempt failed with error: %v\n", err) + continue + } + fmt.Printf("---- Final attempt failed.\n") + return err + } + break + } + fmt.Printf("---- Successful on attempt %v of %v.\n", i+1, attempts) + return nil +} + +// MaxMakeRetryAttemptsOrExit returns max retry attempts for the Go build according to an env var. +func MaxMakeRetryAttemptsOrExit() int { + return maxAttemptsOrExit("GO_MAKE_MAX_RETRY_ATTEMPTS") +} + +// MaxTestRetryAttemptsOrExit returns the max test retry attempts according to an env var. Shared +// between the build command and run-builder command. +func MaxTestRetryAttemptsOrExit() int { + return maxAttemptsOrExit("GO_TEST_MAX_RETRY_ATTEMPTS") +} + +func maxAttemptsOrExit(varName string) int { + attempts, err := getEnvIntOrDefault(varName, 1) + if err != nil { + log.Fatal(err) + } + if attempts <= 0 { + log.Fatalf("Expected positive integer for environment variable %q, but found: %v\n", varName, attempts) + } + return attempts +} + +func getEnvIntOrDefault(varName string, defaultValue int) (int, error) { + a, err := GetEnvOrDefault(varName, strconv.Itoa(defaultValue)) + if err != nil { + return 0, err + } + i, err := strconv.Atoi(a) + if err != nil { + return 0, fmt.Errorf("env var %q is not an int: %w", varName, err) + } + return i, nil +} + +// GetEnvOrDefault find an environment variable with name varName and returns its value. If the env +// var is not set, returns defaultValue. +// +// If the env var is found and its value is empty string, returns an error. This can't happen on +// Windows because setting an env var to empty string deletes it. However, on Linux, it is possible. +// It's likely a mistake, so we let the user know what happened with an error. For example, the env +// var might be empty string because it was set by "example=$(someCommand)" and someCommand +// encountered an error and didn't send any output to stdout. +func GetEnvOrDefault(varName, defaultValue string) (string, error) { + v, ok := os.LookupEnv(varName) + if !ok { + return defaultValue, nil + } + if v == "" { + return "", fmt.Errorf( + "env var %q is empty, not a valid string. To use the default string %v, unset the env var", + varName, defaultValue) + } + return v, nil +} diff --git a/eng/_core/cmd/build/build.go b/eng/_core/cmd/build/build.go index 76153e8c1b5..a694217429d 100644 --- a/eng/_core/cmd/build/build.go +++ b/eng/_core/cmd/build/build.go @@ -11,9 +11,9 @@ import ( "os/exec" "path/filepath" "runtime" - "strconv" "github.com/microsoft/go/_core/archive" + "github.com/microsoft/go/_core/buildutil" "github.com/microsoft/go/_core/patch" "github.com/microsoft/go/_core/submodule" ) @@ -48,6 +48,9 @@ func main() { "Refresh Go submodule: clean untracked files, reset tracked files, and apply patches before building.\n"+ "For more refresh options, use the top level 'submodule-refresh' command instead of 'build'.") + o.MaxMakeAttempts = buildutil.MaxMakeRetryAttemptsOrExit() + o.MaxTestAttempts = buildutil.MaxTestRetryAttemptsOrExit() + flag.Usage = func() { fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n") flag.PrintDefaults() @@ -75,6 +78,9 @@ type options struct { JSON bool Pack bool Refresh bool + + MaxMakeAttempts int + MaxTestAttempts int } func build(o *options) error { @@ -105,6 +111,19 @@ func build(o *options) error { } } + // Get the target platform information. If the environment variable is different from the + // runtime value, this means we're doing a cross-compiled build. These values are used for + // capability checks and to make sure that if Pack is enabled, the output archive is formatted + // correctly and uses the right filename. + targetOS, err := buildutil.GetEnvOrDefault("GOOS", runtime.GOOS) + if err != nil { + return err + } + targetArch, err := buildutil.GetEnvOrDefault("GOARCH", runtime.GOARCH) + if err != nil { + return err + } + // The upstream build scripts in {repo-root}/src require your working directory to be src, or // they instantly fail. Change the current process dir so that we can run them. if err := os.Chdir("go/src"); err != nil { @@ -134,29 +153,18 @@ func build(o *options) error { return err } - maxAttempts, err := getMaxMakeRetryAttempts() - if err != nil { - return err - } - buildCommandLine := append(shellPrefix, "make"+scriptExtension) - for i := 0; i < maxAttempts; i++ { - if maxAttempts > 1 { - fmt.Printf("---- Running 'make' attempt %v of %v...\n", i+1, maxAttempts) - } - err := runCommandLine(buildCommandLine...) - if err != nil { - if i+1 < maxAttempts { - fmt.Printf("---- Build command failed with error: %v\n", err) - continue - } - return err - } - break + if err := buildutil.Retry(o.MaxMakeAttempts, func() error { + return runCommandLine(buildCommandLine...) + }); err != nil { + return err } - if os.Getenv("CGO_ENABLED") != "0" { + // The race runtime requires cgo. + // It isn't supported on arm. + // It's supported on arm64, but the official linux-arm64 distribution doesn't include it. + if os.Getenv("CGO_ENABLED") != "0" && targetArch != "arm" && targetArch != "arm64" { fmt.Println("---- Building race runtime...") err := runCommandLine( filepath.Join("..", "bin", "go"+executableExtension), @@ -194,33 +202,38 @@ func build(o *options) error { testCommandLine = append(testCommandLine, "-json") } - testCmd := exec.Command(testCommandLine[0], testCommandLine[1:]...) - testCmd.Stdout = os.Stdout - // Redirect stderr to stdout. We expect some lines of stderr to always show up during the - // test run, but "build"'s caller might not understand that. - // - // For example, if we're running in CI, gotestsum may be capturing our output to report in a - // JUnit file. If gotestsum detects output in stderr, it prints it in an error message. This - // error message stands out, and could mislead someone trying to diagnose a failed test run. - // Redirecting all stderr output avoids this scenario. (See /eng/_core/README.md for more - // info on why we may be wrapped by gotestsum.) - // - // An example of benign stderr output is when the tests check for machine capabilities. A - // Cgo static linking test emits "/usr/bin/ld: cannot find -lc" when it checks the - // capabilities of "ld" on the current system. - // - // The stderr output isn't used to determine whether the tests succeeded or not. (The - // redirect doesn't cause an issue where tests succeed that should have failed.) - testCmd.Stderr = os.Stdout + test := func() error { + testCmd := exec.Command(testCommandLine[0], testCommandLine[1:]...) + testCmd.Stdout = os.Stdout + // Redirect stderr to stdout. We expect some lines of stderr to always show up during the + // test run, but "build"'s caller might not understand that. + // + // For example, if we're running in CI, gotestsum may be capturing our output to report in a + // JUnit file. If gotestsum detects output in stderr, it prints it in an error message. This + // error message stands out, and could mislead someone trying to diagnose a failed test run. + // Redirecting all stderr output avoids this scenario. (See /eng/_core/README.md for more + // info on why we may be wrapped by gotestsum.) + // + // An example of benign stderr output is when the tests check for machine capabilities. A + // Cgo static linking test emits "/usr/bin/ld: cannot find -lc" when it checks the + // capabilities of "ld" on the current system. + // + // The stderr output isn't used to determine whether the tests succeeded or not. (The + // redirect doesn't cause an issue where tests succeed that should have failed.) + testCmd.Stderr = os.Stdout + + return runCmd(testCmd) + } - if err := runCmd(testCmd); err != nil { + if err := buildutil.Retry(o.MaxTestAttempts, test); err != nil { return err } } if o.Pack { goRootDir := filepath.Join(rootDir, "go") - if err := archive.CreateFromBuild(goRootDir, ""); err != nil { + output := archive.DefaultBuildOutputPath(goRootDir, targetOS, targetArch) + if err := archive.CreateFromBuild(goRootDir, output); err != nil { return err } } @@ -240,16 +253,3 @@ func runCmd(cmd *exec.Cmd) error { fmt.Printf("---- Running command: %v\n", cmd.Args) return cmd.Run() } - -func getMaxMakeRetryAttempts() (int, error) { - const retryEnvVarName = "GO_MAKE_MAX_RETRY_ATTEMPTS" - a := os.Getenv(retryEnvVarName) - if a == "" { - return 1, nil - } - i, err := strconv.Atoi(a) - if err != nil { - return 0, fmt.Errorf("env var '%v' is not an int: %w", retryEnvVarName, err) - } - return i, nil -} diff --git a/eng/_core/cmd/cmdscan/cmdscan.go b/eng/_core/cmd/cmdscan/cmdscan.go new file mode 100644 index 00000000000..cb87194dfee --- /dev/null +++ b/eng/_core/cmd/cmdscan/cmdscan.go @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "bufio" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log" + "os" + "os/exec" + "regexp" + "strings" +) + +const description = ` +Cmdscan runs the command given to it as args and monitors the output for text patterns that should +be elevated to AzDO Pipeline warnings using AzDO logging commands. Timeline events can be discovered +more easily in the UI and by automated tools like runfo. + +Uses the "log issue" pipeline logging command to create timeline warnings: +https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=bash#logissue-log-an-error-or-warning + +To specify the rules to match, pass something like "GO_CMDSCAN_RULE_" to envprefix. This detects any +env variables that start with that string and interprets their values as JSON specifying a rule. The +part of the env var after the prefix is what the rule should be called in logs. If parsing is not +successful, that rule is ignored. Cmdscan writes logs about what it finds. + +When using an AzDO pipeline, SHOUT_CASE is recommended so AzDO's env var naming conversion doesn't +change the result: +https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#environment-variables + +Use "--" to unambiguously separate the flag with the command to run. + +The format for a rule is a JSON object with regex string "pattern" and optionally a "url" string +that contains more information about the issue. For example: + + {"pattern": "(?i)Access is denied", "url": "https://github.com/microsoft/go/issues/241"} + +Example cmdscan call: + + pwsh eng/run.ps1 cmdscan -envprefix GoCmdscanRule -- pwsh eng/run.ps1 build -test +` + +type rule struct { + Pattern string `json:"pattern"` + URL string `json:"url"` +} + +type filter struct { + // name is a simple description of the detected error that should be uniquely searchable so + // runfo can keep track of this issue with "contains" searches on each timeline element. + name string + // url is an optional URL that links to more information about this error. + url string + // regexp is the regex that is matched against each line of output to find an issue. + regexp *regexp.Regexp +} + +var filters []filter + +func main() { + help := flag.Bool("h", false, "Print this help message.") + prefix := flag.String("envprefix", "", "The env var prefix to use to find scan rules.") + + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n") + flag.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) + } + + flag.Parse() + if *help { + flag.Usage() + return + } + + if *prefix != "" { + log.Printf("Searching for rules with env var prefix %v...\n", *prefix) + for _, e := range os.Environ() { + if envName, envValue, ok := strings.Cut(e, "="); ok { + if before, ruleName, ok := strings.Cut(envName, *prefix); ok && before == "" && ruleName != "" { + f, err := parseFilter(ruleName, envValue) + if err != nil { + log.Printf("Failed to parse rule %q: %v\n", envName, err) + continue + } + log.Printf("Parsed rule %q: %q (%v)\n", envName, f.regexp.String(), f.url) + filters = append(filters, *f) + } + } + } + log.Printf("Found %v rules defined by env vars.\n", len(filters)) + } + + if err := run(); err != nil { + log.Fatalln(err) + } +} + +func parseFilter(name, envValue string) (*filter, error) { + var r rule + + if err := json.Unmarshal([]byte(envValue), &r); err != nil { + return nil, err + } + if r.Pattern == "" { + return nil, errors.New("rule defines no pattern") + } + + exp, err := regexp.Compile(r.Pattern) + if err != nil { + return nil, fmt.Errorf("failed to parse rule regex: %v", err) + } + + return &filter{ + name: name, + url: r.URL, + regexp: exp, + }, nil +} + +func run() error { + cmd := exec.Command(flag.Args()[0], flag.Args()[1:]...) + log.Printf("Running: %v\n", cmd) + + outPipe, err := cmd.StdoutPipe() + if err != nil { + return err + } + + errPipe, err := cmd.StderrPipe() + if err != nil { + return err + } + + if err := cmd.Start(); err != nil { + return err + } + + go func() { + if err := scan(outPipe, os.Stdout, os.Stdout); err != nil { + log.Fatalf("Failed to scan stdout pipe: %v\n", err) + } + }() + go func() { + if err := scan(errPipe, os.Stdout, os.Stderr); err != nil { + log.Fatalf("Failed to scan stderr pipe: %v\n", err) + } + }() + + return cmd.Wait() +} + +func scan(r io.Reader, commands, echo *os.File) error { + s := bufio.NewScanner(r) + for s.Scan() { + fmt.Fprintf(echo, "%v\n", s.Text()) + for _, f := range filters { + if f.regexp.MatchString(s.Text()) { + fmt.Fprintf(echo, "Found pattern '%v'\n", f.regexp) + fmt.Fprintf(commands, warn(&f, s.Text())) + } + } + } + return s.Err() +} + +func warn(f *filter, line string) string { + var issueLink string + if f.url != "" { + issueLink = " (" + f.url + ")" + } + + return fmt.Sprintf("##vso[task.logissue type=warning]%q%v: %v\n", f.name, issueLink, line) +} diff --git a/eng/_core/cmd/pack/pack.go b/eng/_core/cmd/pack/pack.go index 0fc9cdbb103..ed106033aa3 100644 --- a/eng/_core/cmd/pack/pack.go +++ b/eng/_core/cmd/pack/pack.go @@ -16,6 +16,11 @@ import ( const description = ` This command packs a built Go directory into an archive file and produces a checksum file for the archive. It filters out the files that aren't necessary. + +Pack does not support packing cross-compiled Go directories. Use the "-pack" +argument with the build command for this, instead. The Pack command is intended +to repackage an extracted Go archive that was already in the correct format. +To re-run pack quickly on a cross-compiled build, use "build -skipbuild -pack". ` func main() { diff --git a/eng/_util/cmd/run-builder/run-builder.go b/eng/_util/cmd/run-builder/run-builder.go index fe9987925da..91166ea40d4 100644 --- a/eng/_util/cmd/run-builder/run-builder.go +++ b/eng/_util/cmd/run-builder/run-builder.go @@ -7,10 +7,13 @@ package main import ( "flag" "fmt" + "log" "os" "os/exec" + "strconv" "strings" + "github.com/microsoft/go/_core/buildutil" gotestsumcmd "gotest.tools/gotestsum/cmd" ) @@ -35,6 +38,7 @@ var dryRun = flag.Bool("n", false, "Enable dry run: print the commands that woul func main() { var builder = flag.String("builder", "", "[Required] Specify a builder to run. Note, this may be destructive!") + var fipsMode = flag.Bool("fipsmode", false, "Run the Go tests in FIPS mode.") var jUnitFile = flag.String("junitfile", "", "Write a JUnit XML file to this path if this builder runs tests.") var help = flag.Bool("h", false, "Print this help message.") @@ -68,6 +72,10 @@ func main() { runOrPanic("eng/workaround-install-mercurial.sh") } + maxTestRetries := buildutil.MaxTestRetryAttemptsOrExit() + // Scale this variable to increase timeout time based on scenario or builder speed. + timeoutScale := 1 + // Some builder configurations need extra env variables set up during the build, not just while // running tests: switch config { @@ -75,7 +83,7 @@ func main() { env("CC", "/usr/bin/clang-3.9") case "longtest": env("GO_TEST_SHORT", "false") - env("GO_TEST_TIMEOUT_SCALE", "5") + timeoutScale *= 5 case "nocgo": env("CGO_ENABLED", "0") case "noopt": @@ -88,6 +96,16 @@ func main() { env("GOEXPERIMENT", "staticlockranking") } + // Some Windows builders are slower than others and require more time for the runtime dist tests + // in "GOMAXPROCS=2 runtime -cpu=1,2,4 -quick" mode. https://github.com/microsoft/go/issues/700 + if goos == "windows" { + timeoutScale *= 2 + } + + if timeoutScale != 1 { + env("GO_TEST_TIMEOUT_SCALE", strconv.Itoa(timeoutScale)) + } + runOrPanic("pwsh", "eng/run.ps1", "build") // After the build completes, run builder-specific commands. @@ -102,6 +120,18 @@ func main() { default: // Most builder configurations use "bin/go tool dist test" directly, which is the default. + if *fipsMode { + env("GOFIPS", "true") + // Enable system-wide FIPS if supported by the host platform. + restore, err := enableSystemWideFIPS() + if err != nil { + log.Fatalf("Unable to enable system-wide FIPS: %v\n", err) + } + if restore != nil { + defer restore() + } + } + // The tests read GO_BUILDER_NAME and make decisions based on it. For some configurations, // we only need to set this env var. env("GO_BUILDER_NAME", *builder) @@ -138,7 +168,18 @@ func main() { ) } - runTest(cmdline, *jUnitFile) + err := buildutil.Retry(maxTestRetries, func() error { + return runTest(cmdline, *jUnitFile) + }) + // If we got an ExitError, the error message was already printed by the command. We just + // need to exit with the same exit code. + if exitErr, ok := err.(*exec.ExitError); ok { + os.Exit(exitErr.ExitCode()) + } + if err != nil { + // Something else happened: alert the user. + log.Fatal(err) + } } } @@ -174,7 +215,7 @@ func runOrPanic(cmdline ...string) { // runTest runs a testing command. If given a JUnit XML file path, runs the test command inside a // gotestsum command that converts the JSON output into JUnit XML and writes it to a file at this // path. -func runTest(cmdline []string, jUnitFile string) { +func runTest(cmdline []string, jUnitFile string) error { if jUnitFile != "" { // Emit verbose JSON results in stdout for conversion. cmdline = append(cmdline, "-json") @@ -182,11 +223,9 @@ func runTest(cmdline []string, jUnitFile string) { if *dryRun { fmt.Printf("---- Dry run. Would have run test command: %v\n", cmdline) - return + return nil } - var err error - if jUnitFile != "" { // Set up gotestsum args. We rely on gotestsum to run the command, capture its output, and // convert it to JUnit test result XML. @@ -231,19 +270,8 @@ func runTest(cmdline []string, jUnitFile string) { // binary and there is no actual 'gotestsum' program. We could pass run-builder's path, but // that would be misleading if it ever shows up in gotestsum's output unexpectedly. Instead, // pass an obvious placeholder. - err = gotestsumcmd.Run("ARG_0_PLACEHOLDER", gotestsumArgs) - } else { - // If we don't have a jUnitFile target, run the command normally. - err = run(cmdline...) - } - - if err != nil { - // If we got an ExitError, the error message was already printed by the command. We just - // need to exit with the same exit code. - if exitErr, ok := err.(*exec.ExitError); ok { - os.Exit(exitErr.ExitCode()) - } - // Something else happened: alert the user. - panic(err) + return gotestsumcmd.Run("ARG_0_PLACEHOLDER", gotestsumArgs) } + // If we don't have a jUnitFile target, run the command normally. + return run(cmdline...) } diff --git a/eng/_util/cmd/run-builder/systemfips_fallback.go b/eng/_util/cmd/run-builder/systemfips_fallback.go new file mode 100644 index 00000000000..8f6c89e557b --- /dev/null +++ b/eng/_util/cmd/run-builder/systemfips_fallback.go @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build !windows +// +build !windows + +package main + +import "log" + +// enableSystemWideFIPS fallback is a no-op because the current platform either doesn't support or +// doesn't require system-wide FIPS to be enabled to run tests. +func enableSystemWideFIPS() (restore func(), err error) { + log.Println("Using fallback (no-op) for enableSystemWideFIPS. It either isn't supported on this platform or isn't necessary.") + return nil, nil +} diff --git a/eng/_util/cmd/run-builder/systemfips_windows.go b/eng/_util/cmd/run-builder/systemfips_windows.go new file mode 100644 index 00000000000..64d03bd981c --- /dev/null +++ b/eng/_util/cmd/run-builder/systemfips_windows.go @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "fmt" + "log" + + "golang.org/x/sys/windows/registry" +) + +// enableSystemWideFIPS enables Windows system-wide FIPS and returns a state-restoring func in case +// the host will be used later by another process. If the host is simultaneously shared, enabling +// system-wide FIPS may interfere because this policy is a machine setting. +func enableSystemWideFIPS() (restore func(), err error) { + key, err := registry.OpenKey( + registry.LOCAL_MACHINE, + `SYSTEM\CurrentControlSet\Control\Lsa\FipsAlgorithmPolicy`, + registry.QUERY_VALUE|registry.SET_VALUE) + if err != nil { + return nil, err + } + + enabled, enabledType, err := key.GetIntegerValue("Enabled") + if err != nil { + return nil, err + } + + if enabledType != registry.DWORD { + return nil, fmt.Errorf("unexpected FIPS algorithm policy Enabled key type: %v", enabledType) + } + + if enabled == 1 { + log.Println("FIPS algorithm policy already enabled.") + return nil, nil + } + + log.Printf("Found FIPS algorithm policy Enabled value: %v\n", enabled) + if err := key.SetDWordValue("Enabled", 1); err != nil { + return nil, err + } + + log.Println("Enabled FIPS algorithm policy.") + + return func() { + defer key.Close() + err := key.SetDWordValue("Enabled", uint32(enabled)) + if err != nil { + log.Printf("Unable to set FIPS algorithm policy to original value %v: %v\n", enabled, err) + return + } + log.Printf("Successfully reset FIPS algorithm policy back to original value %v: %v\n", enabled, err) + }, nil +} diff --git a/eng/_util/go.mod b/eng/_util/go.mod index dc95b46f4ab..b7140503c5d 100644 --- a/eng/_util/go.mod +++ b/eng/_util/go.mod @@ -7,6 +7,10 @@ module github.com/microsoft/go/_util go 1.16 require ( - github.com/microsoft/go-infra v0.0.0-20220209233812-d528ea99adb8 + github.com/microsoft/go-infra v0.0.0-20220419195018-e437e0d7a6f9 + github.com/microsoft/go/_core v0.0.0 + golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 gotest.tools/gotestsum v1.6.5-0.20210515201937-ecb7c6956f6d ) + +replace github.com/microsoft/go/_core => ../_core diff --git a/eng/_util/go.sum b/eng/_util/go.sum index fa5aa4d8ca1..4963f26cc4f 100644 --- a/eng/_util/go.sum +++ b/eng/_util/go.sum @@ -4,6 +4,8 @@ github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -15,8 +17,8 @@ github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/microsoft/go-infra v0.0.0-20220209233812-d528ea99adb8 h1:2aNRJlGG6hOhHsQV3/5+udsetpNT24/5eeJBMWOmjDY= -github.com/microsoft/go-infra v0.0.0-20220209233812-d528ea99adb8/go.mod h1:3IVGTm7qFJldQHximiWLg2kYfmugjZMGNHnvUo5Mo5M= +github.com/microsoft/go-infra v0.0.0-20220419195018-e437e0d7a6f9 h1:TztudEX3sCZvuB9ZrKclZtxNRRE87GF5v7/U7DsLOVo= +github.com/microsoft/go-infra v0.0.0-20220419195018-e437e0d7a6f9/go.mod h1:eGPzRx36eMvtr8J3LLHYyP720NNE6kWLn78BcK/APOs= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 24cec0424e5..0ff81e6f3e7 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -59,7 +59,9 @@ jobs: - checkout: self clean: true - - template: /eng/common/templates/post-build/setup-maestro-vars.yml + # If the template caller didn't provide an AzDO parameter, set them all up as Maestro vars. + - ${{ if not(and(parameters.AzDOProjectName, parameters.AzDOPipelineId, parameters.AzDOBuildId)) }}: + - template: /eng/common/templates/post-build/setup-maestro-vars.yml - ${{ if ne(parameters.downloadArtifacts, 'false')}}: - ${{ if ne(parameters.artifactNames, '') }}: diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 5cd5325d7b4..88f6f75a622 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -46,7 +46,7 @@ jobs: # source-build builds run in Docker, including the default managed platform. pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore1ESPool-Public + name: NetCore-Public demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: NetCore1ESPool-Internal diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 4af724eb1a9..f6b80407eee 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -24,7 +24,7 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore1ESPool-Public + name: NetCore-Public demands: ImageOverride -equals Build.Server.Amd64.VS2019.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: NetCore1ESPool-Internal diff --git a/eng/compliance/Guardian/BinSkimConfig.xml b/eng/compliance/Guardian/BinSkimConfig.xml new file mode 100644 index 00000000000..c4f7fb472dc --- /dev/null +++ b/eng/compliance/Guardian/BinSkimConfig.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + diff --git a/eng/compliance/Guardian/CredScanSuppressions.json b/eng/compliance/Guardian/CredScanSuppressions.json new file mode 100644 index 00000000000..eacf393f498 --- /dev/null +++ b/eng/compliance/Guardian/CredScanSuppressions.json @@ -0,0 +1,17 @@ +{ + "tool": "Credential Scanner", + "suppressions": [ + { + "file": "example_test.go", + "_justification": "Public secret for testing purposes in an upstream source file." + }, + { + "file": "example-key.pem", + "_justification": "Public secret for testing purposes in an upstream source file." + }, + { + "file": "boring_test.go", + "_justification": "Public secret for testing purposes in an upstream source file." + } + ] +} \ No newline at end of file diff --git a/eng/compliance/Guardian/README.md b/eng/compliance/Guardian/README.md new file mode 100644 index 00000000000..ce6f2d2da4f --- /dev/null +++ b/eng/compliance/Guardian/README.md @@ -0,0 +1,32 @@ +# Guardian + +Guardian is an internal Microsoft tool written in .NET that runs a suite of SDL (Security Development Lifecycle) tools. It also runs PoliCheck, which is not an SDL tool, but it is convenient to let Guardian run it and report the results. + +Internal rolling builds run Guardian and report results. + +The microsoft/go implementation of Guardian execution is based on [dotnet/arcade](https://github.com/dotnet/arcade). See [HowToAddSDLRunToPipeline.md](https://github.com/dotnet/arcade/blob/main/Documentation/HowToAddSDLRunToPipeline.md). + +# Running Guardian locally on Windows + +Microsoft internal auth is necessary to download the SDL tools. + +1. Create a temporary folder, e.g. `C:\temp\sdl`. +1. Go to + https://dev.azure.com/SecurityTools/SecurityIntegration/_packaging?_a=package&feed=Guardian&package=Microsoft.Guardian.Cli&protocolType=NuGet + and download the desired version. +1. Extract the `nupkg` file (it's just a `zip`) to a known location like `C:\temp\guardian`. +1. Clone the Go repo into `C:\temp\sdl\src`. +1. Place artifacts to validate into `C:\temp\sdl\artifacts`. + 1. To validate a `zip` or `tar.gz`, extract it. +1. Open a powershell terminal. + 1. The build job uses `powershell`, not `pwsh`. +1. Set `$env:BUILD_ARTIFACTSTAGINGDIRECTORY = "C:\temp\sdl"` +1. Set `$env:BUILD_SOURCESDIRECTORY = "C:\temp\sdl\go"` +1. In `C:\temp\sdl`, run: + ```powershell + & go\eng\compliance\Guardian\execute-go-sdl-tools.ps1 ` + -GuardianCliLocation C:\temp\sdl\guardian\tools\guardian.cmd ` + -WorkingDirectory C:\temp\sdl + ``` + +Some steps (such as PoliCheck) may refuse to run locally due to lack of authentication, even if you have Microsoft internal auth. Those must be run in the internal rolling (official) build job. Running Guardian locally only confirms some basic functionality. diff --git a/eng/compliance/Guardian/execute-go-sdl-tools.ps1 b/eng/compliance/Guardian/execute-go-sdl-tools.ps1 new file mode 100644 index 00000000000..8440426a573 --- /dev/null +++ b/eng/compliance/Guardian/execute-go-sdl-tools.ps1 @@ -0,0 +1,98 @@ +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +$srcDir = $env:BUILD_SOURCESDIRECTORY +# Microsoft-specific engineering tools and configuration. +$engDirectory = Join-Path $srcDir "eng" +# Microsoft-specific GitHub and GitHub Actions configuration. +$dotGitHubDirectory = Join-Path $srcDir ".github" +# Official build artifacts, downloaded from the build job that completed earlier. +$downloadedArtifactsDirectory = Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY "artifacts" + +# Remove verison number from the artifact path to make path-based issue suppression more reliable. +foreach ($item in Get-ChildItem -Directory $downloadedArtifactsDirectory) +{ + if ($item.Name.StartsWith("go.") -and $item.Name.EndsWith(".extracted")) + { + $oldName = $item.FullName + $newName = $item.FullName -replace '\\go\.[0-9]+\.[0-9]+\.', '\go.' + if ($oldName -ne $newName) + { + Write-Host "Renaming '$oldName' to '$newName'" + Move-Item $oldName $newName + } + } +} + +# Create a file for PoliCheck's ListFile option. The extension must be ".txt", and this file must +# contain full paths, one per line, with no duplicates. The list should contain each microsoft/go +# file but no upstream files. Sort and print it for debug purposes. +$policheckFileList = (New-TemporaryFile).FullName + ".txt" +( + Get-ChildItem -File -Recurse $srcDir ` + | Where-Object { + # Submodule directory with upstream code. + -not $_.FullName.StartsWith((Join-Path $srcDir "go")) -and ` + # SDL NuGet packages: ignore, not part of our code. + -not $_.FullName.StartsWith((Join-Path $srcDir ".packages")) } ` + | ForEach-Object { $_.FullName } | Sort-Object ` +) -join "`r`n" > $policheckFileList + +Write-Host "--- List of files in PoliCheck file list:" +Get-Content $policheckFileList | Write-Host +Write-Host "---" + +& "$PSScriptRoot\..\..\common\sdl\execute-all-sdl-tools.ps1" ` + -SourceToolsList @( + @{ Name="credscan"; Scenario="source" } + ) ` + -ArtifactToolsList @( + @{ Name="credscan"; Scenario="artifacts" } + ) ` + -CrScanAdditionalRunConfigParams @( + "SuppressionsPath < $engDirectory\compliance\Guardian\CredScanSuppressions.json" + "SuppressAsError < false" + ) ` + -CustomToolsList @( + @{ + Name="binskim" + Args=@( + # Point binskim at the artifact directory. Pass everything to binskim and let it decide what + # it needs to scan. For more information about the glob format, see + # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1378/Glob-Format + # + # Exclude "testdata" binaries because they are only used during testing, they do not pass + # "binskim" for various reasons, and they are checked into the upstream Go repository. + # + # Exclude infra dependencies in ".gdn" dir. We are not distributing these. + # + # Exclude all ".exe" files. BinSkim strongly expects PDB files for each one, but they don't + # exist for Go. See https://github.com/microsoft/go/issues/114 + "Target < f|$downloadedArtifactsDirectory\**;-|**\testdata\*;-|.gdn\**;-|**\*.exe" + "ConfigPath < $engDirectory\compliance\Guardian\BinSkimConfig.xml" + ) + } + @{ + Name="codesign" + Args=@( + # Point codesign at the right location to find the artifacts that we've signed. However, we do + # not yet produce any artifacts that CodeSign knows how to verify, so don't fail if CodeSign + # fails to find anything. + "TargetDirectory < $downloadedArtifactsDirectory" + "targetFiles < f|**\*.dll;f|**\*.exe" + "failIfNoTargetsFound < false" + ) + } + # Only point PoliCheck at directories we control, not directories from the upstream repo. + @{ + Name="policheck" + Args=@( + # Target's default is ".", but we need to pass nothing instead. The Target and ListFile + # PoliCheck args are mutually exclusive. + "Target" + "ListFile < $policheckFileList" + ) + } + ) ` + @args diff --git a/eng/doc/fips/UserGuide.md b/eng/doc/fips/UserGuide.md index 156fee008e8..8bdf85ec2da 100644 --- a/eng/doc/fips/UserGuide.md +++ b/eng/doc/fips/UserGuide.md @@ -684,4 +684,4 @@ When using TLS in FIPS-only mode the TLS handshake has the following restriction - `tls.ECDSAWithP521AndSHA512` [EVP_EncryptUpdate]: https://www.openssl.org/docs/manmaster/man3/EVP_EncryptUpdate.html -[EVP_DecryptUpdate]: https://www.openssl.org/docs/manmaster/man3/EVP_DecryptUpdate.html \ No newline at end of file +[EVP_DecryptUpdate]: https://www.openssl.org/docs/manmaster/man3/EVP_DecryptUpdate.html diff --git a/eng/init-pwsh.sh b/eng/init-pwsh.sh index f13b4c39c1f..82123bce5fe 100755 --- a/eng/init-pwsh.sh +++ b/eng/init-pwsh.sh @@ -9,9 +9,29 @@ set -euo pipefail -pwsh_version='7.1.3' -pwsh_sha256='9f853fb8f7c7719005bd1054fa13ca4d925c519b893f439dd2574e84503e6a85' -pwsh_url="https://github.com/PowerShell/PowerShell/releases/download/v$pwsh_version/powershell-$pwsh_version-linux-x64.tar.gz" +# Default values for x64. +pwsh_version='7.2.1' +pwsh_sha256='337d9864799ad09b46d261071b9f835f69f078814409bc2681f4cc2857b6bda5' +pwsh_arch='x64' + +# 'uname' approach adapted from .NET install script: https://github.com/dotnet/install-scripts/blob/df8f863720a462448ad244f03ffeb619f0631bad/src/dotnet-install.sh#L295-L315 +if command -v uname > /dev/null; then + CPUName=$(uname -m) + case $CPUName in + armv*l) + echo "armv*l was detected, but it is not supported by the microsoft/go build infrastructure." + exit 1 + ;; + aarch64|arm64) + pwsh_sha256='f0d6c9c36d69e1466e5a9412085ef52cafd10b73f862d29479b806279a2975f4' + pwsh_arch='arm64' + ;; + esac +else + echo "uname command not detected. Assuming $pwsh_arch." +fi + +pwsh_url="https://github.com/PowerShell/PowerShell/releases/download/v$pwsh_version/powershell-$pwsh_version-linux-$pwsh_arch.tar.gz" # pwsh must be installed outside of the Go repo. If it's in the repo, longtest "TestAllDependencies" # fails. It tries to traverse the pwsh directory and can't handle the "no such file or directory" @@ -22,6 +42,7 @@ download_complete_indicator="$pwsh_dir/.downloaded" if [ ! -f "$download_complete_indicator" ]; then echo "Downloading PowerShell $pwsh_version and extracting to '$pwsh_dir' ..." + echo "URL: $pwsh_url" # Clear existing dir in case it's in a broken state. rm -rf "$pwsh_dir" diff --git a/eng/pipeline/README.md b/eng/pipeline/README.md index 06e34c85705..808391f0c88 100644 --- a/eng/pipeline/README.md +++ b/eng/pipeline/README.md @@ -6,17 +6,15 @@ Pipeline definitions currently using each YAML file are: * [`pr-pipeline.yml`](pr-pipeline.yml) - Required PR check. * (Public) [microsoft-go](https://dev.azure.com/dnceng/public/_build?definitionId=1099) -* [`pr-outerloop-pipeline.yml`](pr-outerloop-pipeline.yml) - Optional PR check. - Runs outerloop. +* [`pr-outerloop-pipeline.yml`](pr-outerloop-pipeline.yml) - Optional PR check. Runs outerloop. * (Public) [microsoft-go-outerloop](https://dev.azure.com/dnceng/public/_build/index?definitionId=1100) * Comment `/azp run microsoft-go-outerloop` on a PR to run. -* [`rolling-pipeline.yml`](rolling-pipeline.yml) - Triggers on merge, runs - innerloop + outerloop. +* [`rolling-pipeline.yml`](rolling-pipeline.yml) - Triggers on merge, runs innerloop + outerloop. * (Internal) [microsoft-go-rolling](https://dev.azure.com/dnceng/internal/_build?definitionId=987) -* [`rolling-internal-pipeline.yml`](rolling-internal-pipeline.yml) - Triggers on - merge. Builds, signs, and publishes. Runs innerloop, and won't publish if - innerloop fails. +* [`rolling-internal-pipeline.yml`](rolling-internal-pipeline.yml) - Triggers on merge. Builds, signs, and publishes. Runs innerloop, and won't publish if innerloop fails. * (Internal) [microsoft-go](https://dev.azure.com/dnceng/internal/_build?definitionId=958) +* [`rolling-internal-validation-pipeline.yml`](rolling-internal-validation-pipeline.yml) - Runs validation checks on internal build output. + * (Internal) [microsoft-go-validation](https://dev.azure.com/dnceng/internal/_build?definitionId=1166) The pipeline filenames are (mostly) based on the trigger scenario, not what they do. This means we can change their content later without worrying about @@ -25,6 +23,10 @@ to update the web-UI-based AzDO pipeline to point at the new file. Each web UI pipeline can only point at one YAML file, so this breaks old branches that haven't renamed the file. It would be nice to avoid this.) +For more information about the style of these pipeline and template YAML files +and the quirks involved with the way they're implemented, visit +[pipeline-yml-style.md in microsoft/go-infra](https://github.com/microsoft/go-infra/blob/main/docs/pipeline-yml-style.md). + ## Templates The subdirectories hold AzDO pipeline YAML templates, based on type of template. diff --git a/eng/pipeline/jobs/run-job.yml b/eng/pipeline/jobs/run-job.yml deleted file mode 100644 index cd6546aaabb..00000000000 --- a/eng/pipeline/jobs/run-job.yml +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# This job runs a builder for any OS. - -parameters: - # { id, os, arch, config, distro? } - builder: {} - createSourceArchive: false - -jobs: - - job: ${{ parameters.builder.id }} - workspace: - clean: all - - ${{ if eq(parameters.builder.os, 'windows') }}: - pool: - vmImage: windows-2019 - - ${{ if eq(parameters.builder.os, 'linux') }}: - pool: - # The VM image of the Docker host. This doesn't need to match the container image, but it may - # give slightly better coverage by matching the kernel version. - vmImage: ubuntu-18.04 - # The image used for the container this job runs in. The tests run in this container, so it - # should match what we support as closely as possible. - ${{ if not(parameters.builder.distro) }}: - container: golangpublicimages.azurecr.io/go-infra-images/prereqs:cbl-mariner-1.0.20211027-20211201-0cccc22 - ${{ if eq(parameters.builder.distro, 'ubuntu') }}: - container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20211022152710-047508b - - steps: - - ${{ if eq(parameters.builder.os, 'linux') }}: - - template: ../steps/checkout-unix-task.yml - - template: ../steps/init-pwsh-task.yml - - - ${{ if eq(parameters.builder.os, 'windows') }}: - - template: ../steps/checkout-windows-task.yml - - pwsh: | - Write-Host "Increasing max build retries to mitigate 'Access denied' flakiness during EXE copying on Windows." - Write-Host "##vso[task.setvariable variable=GO_MAKE_MAX_RETRY_ATTEMPTS]5" - displayName: Increase 'make' retry attempts - - # Initialize stage 0 toolset ahead of time so we can track timing data separately from the - # build operations. When we call this script again later, it won't download Go again. - - pwsh: | - . eng/utilities.ps1 - Get-Stage0GoRoot - displayName: Init stage 0 Go toolset - - - template: ../steps/init-submodule-task.yml - - # Create the source archive on one job only. The os choice is arbitrary. - - ${{ if and(eq(parameters.createSourceArchive, true), eq(parameters.builder.config, 'buildandpack'), eq(parameters.builder.os, 'linux')) }}: - - pwsh: | - git config --global user.name "microsoft-golang-bot" - git config --global user.email "microsoft-golang-bot@users.noreply.github.com" - - # Turn the patches into commits, so HEAD includes the changes. - eng/run.ps1 submodule-refresh -commits - eng/run.ps1 pack-source - displayName: Archive submodule source - - - pwsh: | - # Apply the patches as staged changes, so the HEAD commit is the same as upstream. - eng/run.ps1 submodule-refresh - displayName: Apply patches - - # Use build script directly for "buildandpack". If we used run-builder, we would need to - # download its external module dependencies. - - ${{ if eq(parameters.builder.config, 'buildandpack' ) }}: - - pwsh: | - eng/run.ps1 build -pack - displayName: Build and Pack - - - publish: eng/artifacts/bin - artifact: Binaries ${{ parameters.builder.id }} - displayName: Pipeline publish - condition: succeededOrFailed() - - # Use run-builder for any configuration that includes tests. run-builder uses the "gotestsum" - # module to convert test results to a JUnit file that Azure DevOps can understand. - - ${{ if ne(parameters.builder.config, 'buildandpack') }}: - - pwsh: | - if ($IsWindows) { - Write-Host "Removing Git usr\bin from PATH to avoid running a Linux test that would fail, 'TestScript/script_wait'..." - Write-Host $env:PATH - $env:PATH = ( - $env:PATH -split ';' | Where-Object { $_ -ne 'C:\Program Files\Git\usr\bin' } - ) -join ';' - } - - eng/run.ps1 run-builder ` - -builder '${{ parameters.builder.os }}-${{ parameters.builder.arch }}-${{ parameters.builder.config }}' ` - -junitfile '$(Build.SourcesDirectory)/eng/artifacts/TestResults.xml' - displayName: Run ${{ parameters.builder.config }} - - - task: PublishTestResults@2 - displayName: Publish test results - condition: succeededOrFailed() - inputs: - testResultsFormat: JUnit - testResultsFiles: $(Build.SourcesDirectory)/eng/artifacts/TestResults.xml - testRunTitle: ${{ parameters.builder.id }} - buildPlatform: ${{ parameters.builder.arch }} - buildConfiguration: ${{ parameters.builder.config }} - publishRunAttachments: true diff --git a/eng/pipeline/jobs/sign-job.yml b/eng/pipeline/jobs/sign-job.yml deleted file mode 100644 index 3ed566cbbae..00000000000 --- a/eng/pipeline/jobs/sign-job.yml +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# This job downloads Go binaries from the specified builders' pipeline artifacts, signs them, and -# publishes the signed files and signatures into a consolidated pipeline artifact. - -parameters: - # [] of { id, os, arch, config, distro? } - builders: [] - -jobs: - - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: - - job: Sign - # Depend on all build jobs that produced artifacts that need signing. - dependsOn: - - ${{ each builder in parameters.builders }}: - - ${{ builder.id }} - pool: - # This is a utility job that doesn't use Go: use a pool that supports signing. - name: NetCore1ESPool-Internal - demands: ImageOverride -equals build.windows.10.amd64.vs2019 - variables: - - name: TeamName - value: golang - workspace: - clean: all - steps: - - template: ../steps/checkout-windows-task.yml - - # Add MicroBuild, based on Arcade: https://github.com/dotnet/arcade/blob/927f8d4d5036f68a5fc6d042f336bc9458027208/eng/common/templates/job/job.yml#L106-L115 - - task: MicroBuildSigningPlugin@3 - displayName: Install MicroBuild plugin - inputs: - signType: $(SigningType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - env: - TeamName: $(TeamName) - condition: and(succeeded(), in(variables['SigningType'], 'real', 'test')) - - - ${{ each builder in parameters.builders }}: - - download: current - artifact: Binaries ${{ builder.id }} - displayName: 'Download: Binaries ${{ builder.id }}' - - - powershell: | - $flatDir = "$(Build.StagingDirectory)/ToSign" - New-Item $flatDir -ItemType Directory -ErrorAction Ignore - - Get-ChildItem -Recurse -File -Path @( - 'Binaries ${{ builder.id }}' - ) | %{ - if (Test-Path "$flatDir\$($_.Name)") { - throw "Duplicate filename, unable to flatten: $($_.FullName)" - } - Copy-Item $_.FullName $flatDir - } - displayName: 'Copy to flat dir: ${{ builder.id }}' - workingDirectory: '$(Pipeline.Workspace)' - - - task: DotNetCoreCLI@2 - displayName: 'Install Arcade SignTool' - inputs: - command: restore - projects: '$(Build.SourcesDirectory)/eng/signing/Sign.proj' - feedsToUse: config - nugetConfigPath: '$(Build.SourcesDirectory)/NuGet.config' - - - task: DotNetCoreCLI@2 - displayName: 'Sign Files' - inputs: - command: custom - projects: '$(Build.SourcesDirectory)/eng/signing/Sign.proj' - custom: msbuild - arguments: >- - /t:SignGoFiles - /p:SignFilesDir=$(Build.StagingDirectory)/ToSign - /p:SigningType=$(SigningType) - /bl:SignFiles.binlog - - - publish: $(Build.StagingDirectory)/ToSign - artifact: Binaries Signed - displayName: Publish Binaries Signed - condition: always() - - - publish: eng\signing - artifact: SigningDetails - displayName: Publish SigningDetails - condition: always() - - - publish: 'SignFiles.binlog' - artifact: SigningBinlog - displayName: Publish SigningBinlog - condition: always() - - - task: MicroBuildCleanup@1 - displayName: 'Clean up MicroBuild' - condition: always() diff --git a/eng/pipeline/pr-outerloop-pipeline.yml b/eng/pipeline/pr-outerloop-pipeline.yml index 8fe1ad4d114..53e9ee349bf 100644 --- a/eng/pipeline/pr-outerloop-pipeline.yml +++ b/eng/pipeline/pr-outerloop-pipeline.yml @@ -16,7 +16,7 @@ pr: - microsoft/* - dev/* -jobs: - - template: jobs/go-builder-matrix-jobs.yml +stages: + - template: stages/go-builder-matrix-stages.yml parameters: outerloop: true diff --git a/eng/pipeline/pr-pipeline.yml b/eng/pipeline/pr-pipeline.yml index 0ebef7d40ba..ed0896682b6 100644 --- a/eng/pipeline/pr-pipeline.yml +++ b/eng/pipeline/pr-pipeline.yml @@ -9,7 +9,7 @@ pr: - microsoft/* - dev/* -jobs: - - template: jobs/go-builder-matrix-jobs.yml +stages: + - template: stages/go-builder-matrix-stages.yml parameters: innerloop: true diff --git a/eng/pipeline/rolling-internal-pipeline.yml b/eng/pipeline/rolling-internal-pipeline.yml index 028cdba95c8..e023ed0e5aa 100644 --- a/eng/pipeline/rolling-internal-pipeline.yml +++ b/eng/pipeline/rolling-internal-pipeline.yml @@ -14,89 +14,23 @@ trigger: pr: none stages: - - stage: Build - jobs: - - template: jobs/go-builder-matrix-jobs.yml + - template: stages/go-builder-matrix-stages.yml + parameters: + innerloop: true + sign: true + createSourceArchive: true + + - ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/')) }}: + - template: stages/pool.yml + parameters: + inner: + template: publish-stage.yml + parameters: + public: true + + - template: stages/pool.yml + parameters: + inner: + template: publish-stage.yml parameters: - innerloop: true - sign: true - createSourceArchive: true - - - stage: Publish - dependsOn: Build - jobs: - - job: Publish - pool: - # This is a utility job: use generic recent LTS for publishing. - vmImage: ubuntu-20.04 - variables: - - name: blobDestinationUrl - value: 'https://dotnetbuildoutput.blob.core.windows.net/golang/microsoft/$(PublishBranchAlias)/$(Build.BuildNumber)' - - group: go-storage - workspace: - clean: all - steps: - - template: steps/checkout-unix-task.yml - - template: steps/init-pwsh-task.yml - - template: steps/init-submodule-task.yml - - - pwsh: | - function TrimStart($s, $prefix) { - if ($s.StartsWith($prefix)) { - return $s.Substring($prefix.Length) - } - return $s - } - $branch = "$(Build.SourceBranch)" - Write-Host "For Build.SourceBranch '$branch'," - - # $(Build.SourceBranchName) only gives us "feature" for "refs/heads/dev/feature". We - # want to publish this as "dev/feature". So, use $(Build.SourceBranch) and figure it - # out ourselves. - $branch = TrimStart $branch "refs/heads/" - - # Don't include "microsoft/" virtual directory prefix: we are already in the - # "microsoft" container, so this would result in "microsoft/microsoft/main". - $branch = TrimStart $branch "microsoft/" - - Write-Host "PublishBranchAlias is: $branch" - Write-Host "##vso[task.setvariable variable=PublishBranchAlias;]$branch" - displayName: Find publish branch alias - - - download: current - artifact: Binaries Signed - displayName: 'Download: Binaries Signed' - - - pwsh: | - eng/run.ps1 createbuildassetjson ` - -artifacts-dir '$(Pipeline.Workspace)/Binaries Signed/' ` - -source-dir '$(Build.SourcesDirectory)' ` - -destination-url '$(blobDestinationUrl)' ` - -branch '$(PublishBranchAlias)' ` - -o '$(Pipeline.Workspace)/Binaries Signed/assets.json' - displayName: 'Create build asset JSON' - - - publish: $(Pipeline.Workspace)/Binaries Signed/assets.json - artifact: BuildAssets - displayName: Publish build asset JSON - - - task: AzureCLI@2 - displayName: Upload to blob storage - inputs: - azureSubscription: GoLang - scriptType: bash - scriptLocation: inlineScript - # Send literal '*' to az: it handles the wildcard itself. Az copy only accepts one - # "from" argument, so we can't use the shell's wildcard expansion. - inlineScript: | - az storage copy -s '*' -d '$(blobDestinationUrl)' --sas-token '$(dotnetbuildoutput-golang-write-sas-query)' - workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/' - - - script: | - echo 'Generated links to artifacts in blob storage:' - echo '' - for f in *; do - echo "$(blobDestinationUrl)/$f" - done - displayName: Show uploaded URLs - workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/' + public: false diff --git a/eng/pipeline/rolling-internal-validation-pipeline.yml b/eng/pipeline/rolling-internal-validation-pipeline.yml new file mode 100644 index 00000000000..dff015add76 --- /dev/null +++ b/eng/pipeline/rolling-internal-validation-pipeline.yml @@ -0,0 +1,76 @@ +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This pipeline runs after each internal rolling build of Go and validates compliance. It runs the +# automated CodeQL scan and SDL validation with internal-only tooling. It uses a job template from +# dotnet/arcade that runs the Guardian suite of tools and reports the results to TSA (Trust Services +# Automation). +# +# For more information, see: +# CodeQL: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/codeql-semmle +# SDL: https://microsoft.sharepoint.com/teams/managedlanguages/_layouts/OneNote.aspx?id=%2Fteams%2Fmanagedlanguages%2Ffiles%2FTeam%20Notebook%2FGoLang%20Team&wd=target%28Main.one%7C62B655D4-14E7-41D6-A063-0869C28D63FC%2FSDL%20Tools%7C3908F727-3751-4ACC-8C71-6CEB2DF277B4%2F%29 + +trigger: none +pr: none + +resources: + pipelines: + - pipeline: build + # The rolling pipeline and this validation pipeline share the same source repository. AzDO + # sees this and makes this pipeline's "checkout" steps download the same source code that was + # built by the microsoft-go pipeline: + # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/resources?view=azure-devops&tabs=schema#define-a-pipelines-resource + # + # This means we can have SDL scan the currently-checked-out source code as the way to scan the + # source code of the internal rolling build. + source: microsoft-go + trigger: + branches: + include: + # Validate all branches that may be released. + - microsoft/main + - microsoft/release-branch.* + - microsoft/dev.boringcrypto.go* + +stages: + - template: stages/shorthand-builders-to-builders.yml + parameters: + jobsTemplate: builders-to-stages.yml + shorthandBuilders: + - { os: linux, arch: amd64, config: codeql } + + - stage: SDLValidate + dependsOn: [] + variables: + # TSA variables. + - group: go-sdl-validation + jobs: + # Run SDL validation tooling on sources and signed/complete artifacts. + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + # Don't download any build artifacts: only pipeline artifacts. + downloadArtifacts: false + pipelineArtifactNames: + - Binaries Signed + extractArchiveArtifacts: true + enable: true + publishGuardianDirectoryToPipeline: true + # Specify that artifacts should be downloaded from the build that triggered this one. + AzDOProjectName: $(resources.pipeline.build.projectID) + AzDOPipelineId: $(resources.pipeline.build.pipelineID) + AzDOBuildId: $(resources.pipeline.build.runID) + # Use a wrapper script for the SDL tools to pass the Go-specific configuration. + executeAllSdlToolsScript: eng/compliance/Guardian/execute-go-sdl-tools.ps1 + # Set up TSA publish and build break condition. + additionalParameters: >- + -TsaInstanceURL "$(TsaInstanceURL)" + -TsaProjectName "$(TsaProjectName)" + -TsaNotificationEmail "$(TsaNotificationEmail)" + -TsaCodebaseAdmin "$(TsaCodebaseAdmin)" + -TsaBugAreaPath "$(TsaBugAreaPath)" + -TsaIterationPath "$(TsaIterationPath)" + -TsaRepositoryName "$(TsaRepositoryName)" + -TsaCodebaseName "$(TsaCodebaseName)" + -TsaPublish $true + -BreakOnFailure $true diff --git a/eng/pipeline/rolling-pipeline.yml b/eng/pipeline/rolling-pipeline.yml index 6e006554961..3d2c2ba9a87 100644 --- a/eng/pipeline/rolling-pipeline.yml +++ b/eng/pipeline/rolling-pipeline.yml @@ -17,8 +17,8 @@ trigger: - dev/official/* pr: none -jobs: - - template: jobs/go-builder-matrix-jobs.yml +stages: + - template: stages/go-builder-matrix-stages.yml parameters: innerloop: true outerloop: true diff --git a/eng/pipeline/jobs/builders-to-jobs.yml b/eng/pipeline/stages/builders-to-stages.yml similarity index 54% rename from eng/pipeline/jobs/builders-to-jobs.yml rename to eng/pipeline/stages/builders-to-stages.yml index 05d7a751e9a..4109532f0a7 100644 --- a/eng/pipeline/jobs/builders-to-jobs.yml +++ b/eng/pipeline/stages/builders-to-stages.yml @@ -5,7 +5,7 @@ # This template expands a list of builders into a list of jobs. parameters: - # [] of { id, os, arch, config, distro? } + # [] of { id, os, arch, hostarch, config, distro? } builders: [] # If true, include a signing job that depends on all 'buildandpack' builder jobs finishing. This # lets us start the lengthy tasks of signing and testing in parallel. @@ -13,17 +13,23 @@ parameters: # If true, generate source archive tarballs, and sign them if signing is enabled. createSourceArchive: false -jobs: +stages: - ${{ each builder in parameters.builders }}: - - template: run-job.yml + - template: pool.yml parameters: - builder: ${{ builder }} - createSourceArchive: ${{ parameters.createSourceArchive }} + inner: + template: run-stage.yml + parameters: + builder: ${{ builder }} + createSourceArchive: ${{ parameters.createSourceArchive }} - ${{ if eq(parameters.sign, true) }}: - - template: sign-job.yml + - template: pool.yml parameters: - builders: - - ${{ each builder in parameters.builders }}: - - ${{ if eq(builder.config, 'buildandpack') }}: - - ${{ builder }} + inner: + template: sign-stage.yml + parameters: + builders: + - ${{ each builder in parameters.builders }}: + - ${{ if eq(builder.config, 'buildandpack') }}: + - ${{ builder }} diff --git a/eng/pipeline/jobs/go-builder-matrix-jobs.yml b/eng/pipeline/stages/go-builder-matrix-stages.yml similarity index 75% rename from eng/pipeline/jobs/go-builder-matrix-jobs.yml rename to eng/pipeline/stages/go-builder-matrix-stages.yml index 588287276bf..3e333156d31 100644 --- a/eng/pipeline/jobs/go-builder-matrix-jobs.yml +++ b/eng/pipeline/stages/go-builder-matrix-stages.yml @@ -11,10 +11,10 @@ parameters: sign: false createSourceArchive: false -jobs: +stages: - template: shorthand-builders-to-builders.yml parameters: - jobsTemplate: builders-to-jobs.yml + jobsTemplate: builders-to-stages.yml jobsParameters: sign: ${{ parameters.sign }} createSourceArchive: ${{ parameters.createSourceArchive }} @@ -24,9 +24,16 @@ jobs: - { os: linux, arch: amd64, config: devscript } - { os: linux, arch: amd64, config: test } - { os: linux, arch: amd64, config: test, distro: ubuntu } + - { os: linux, arch: amd64, config: test, fips: true } - { os: windows, arch: amd64, config: buildandpack } - { os: windows, arch: amd64, config: devscript } - { os: windows, arch: amd64, config: test } + - { os: windows, arch: amd64, config: test, fips: true } + - { os: linux, arch: arm, hostArch: amd64, config: buildandpack } + # Only build arm64 if we're running a signed (internal, rolling) build. Avoid contention + # with other projects' builds that use the same limited-capacity pool of arm64 agents. + - ${{ if eq(parameters.sign, true) }}: + - { os: linux, arch: arm64, config: buildandpack } - ${{ if eq(parameters.outerloop, true) }}: # Upstream builders. - { os: linux, arch: amd64, config: clang } diff --git a/eng/pipeline/stages/pool-core.yml b/eng/pipeline/stages/pool-core.yml new file mode 100644 index 00000000000..1682a99124b --- /dev/null +++ b/eng/pipeline/stages/pool-core.yml @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This template defines common pool names for dnceng agents for various platforms. + +parameters: + - name: public + type: boolean + - name: servicing + type: boolean + # The inner template: { template string, parameters object } + - name: inner + type: object + +stages: + - template: ${{ parameters.inner.template }} + parameters: + ${{ insert }}: ${{ parameters.inner.parameters }} + + dncengPool: + # Pick pool. https://github.com/dotnet/arcade/blob/0db07252ccb18afdf94820ba6125da6de729ec04/Documentation/AzureDevOps/AzureDevOpsOnboarding.md#agent-queues + ${{ if parameters.public }}: + ${{ if parameters.servicing }}: + name: NetCore-Svc-Public + ${{ else }}: + name: NetCore-Public + ${{ else }}: + ${{ if parameters.servicing }}: + name: NetCore1ESPool-Svc-Internal + ${{ else }}: + name: NetCore1ESPool-Internal + + # Pick images. https://helix.dot.net/#1esPools + demands: + ${{ if parameters.public }}: + windows: ImageOverride -equals 1es-windows-2019-open + ${{ else }}: + windows: ImageOverride -equals 1es-windows-2019 + + ${{ if parameters.public }}: + linux: ImageOverride -equals 1es-ubuntu-2004-open + ${{ else }}: + linux: ImageOverride -equals 1es-ubuntu-2004 diff --git a/eng/pipeline/stages/pool.yml b/eng/pipeline/stages/pool.yml new file mode 100644 index 00000000000..b726cf225b0 --- /dev/null +++ b/eng/pipeline/stages/pool.yml @@ -0,0 +1,24 @@ +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This template determines the correct pool names and agent demands based on the current branch and +# AzDO project and passes them into the provided inner stages template. +# +# The "public" (vs. internal) and "servicing" (vs. R&D) conditions are evaluated here and then +# passed into an inner template so the values can be reused to determine the pools to use. +# +# See pool-core.yml for the list of pool names and demands and the reasons behind the choices. + +parameters: + # The inner template: { template string, parameters object } + # Note: the template path is relative to this file (inside "stages/"), not the caller's file path. + - name: inner + type: object + +stages: + - template: pool-core.yml + parameters: + public: ${{ eq(variables['System.TeamProject'], 'public') }} + servicing: ${{ startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/release-branch') }} + inner: ${{ parameters.inner }} diff --git a/eng/pipeline/stages/publish-stage.yml b/eng/pipeline/stages/publish-stage.yml new file mode 100644 index 00000000000..6b3963cf46d --- /dev/null +++ b/eng/pipeline/stages/publish-stage.yml @@ -0,0 +1,113 @@ +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Create a build asset JSON file as a pipeline artifact and publish build artifacts to blob storage. + +parameters: + - name: public + type: boolean + - name: dncengPool + type: object + +stages: + - stage: Publish${{ parameters.public }} + ${{ if parameters.public }}: + displayName: Publish Public + ${{ else }}: + displayName: Publish Internal + dependsOn: Sign + jobs: + - job: Publish + pool: + name: ${{ parameters.dncengPool.name }} + demands: ${{ parameters.dncengPool.demands.linux }} + + variables: + - ${{ if parameters.public }}: + - name: blobContainer + value: 'https://dotnetbuildoutput.blob.core.windows.net/golang/microsoft' + - name: blobSASArg + value: --sas-token '$(dotnetbuildoutput-golang-write-sas-query)' + - ${{ else }}: + - name: blobContainer + value: 'https://golangartifacts.blob.core.windows.net/microsoft' + - name: blobSASArg + value: '' # golangartifacts is set up with service connection auth. + + - name: blobDestinationUrl + value: '$(blobContainer)/$(PublishBranchAlias)/$(Build.BuildNumber)' + + - group: go-storage + + workspace: + clean: all + + steps: + - template: ../steps/checkout-unix-task.yml + - template: ../steps/init-pwsh-task.yml + - template: ../steps/init-submodule-task.yml + + - pwsh: | + function TrimStart($s, $prefix) { + if ($s.StartsWith($prefix)) { + return $s.Substring($prefix.Length) + } + return $s + } + $branch = "$(Build.SourceBranch)" + Write-Host "For Build.SourceBranch '$branch'," + + # $(Build.SourceBranchName) only gives us "feature" for "refs/heads/dev/feature". We + # want to publish this as "dev/feature". So, use $(Build.SourceBranch) and figure it + # out ourselves. + $branch = TrimStart $branch "refs/heads/" + + # Don't include "microsoft/" virtual directory prefix: we are already in the + # "microsoft" container, so this would result in "microsoft/microsoft/main". + $branch = TrimStart $branch "microsoft/" + + Write-Host "PublishBranchAlias is: $branch" + Write-Host "##vso[task.setvariable variable=PublishBranchAlias;]$branch" + displayName: Find publish branch alias + + - download: current + artifact: Binaries Signed + displayName: 'Download: Binaries Signed' + + - pwsh: | + eng/run.ps1 createbuildassetjson ` + -artifacts-dir '$(Pipeline.Workspace)/Binaries Signed/' ` + -source-dir '$(Build.SourcesDirectory)' ` + -destination-url '$(blobDestinationUrl)' ` + -branch '$(PublishBranchAlias)' ` + -o '$(Pipeline.Workspace)/Binaries Signed/assets.json' + displayName: 'Create build asset JSON' + + - publish: $(Pipeline.Workspace)/Binaries Signed/assets.json + displayName: Publish build asset JSON to pipeline + ${{ if parameters.public }}: + artifact: BuildAssets + ${{ else }}: + artifact: BuildAssetsInternal + + - task: AzureCLI@2 + displayName: Upload to blob storage + inputs: + azureSubscription: GoLang + scriptType: bash + scriptLocation: inlineScript + # Send literal '*' to az: it handles the wildcard itself. Az copy only accepts one + # "from" argument, so we can't use the shell's wildcard expansion. + inlineScript: | + az storage copy -s '*' -d '$(blobDestinationUrl)' $(blobSASArg) + workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/' + + - script: | + echo 'Generated links to artifacts in blob storage:' + echo '' + for f in *; do + echo "$(blobDestinationUrl)/$f" + done + displayName: Show uploaded URLs + workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/' diff --git a/eng/pipeline/stages/run-stage.yml b/eng/pipeline/stages/run-stage.yml new file mode 100644 index 00000000000..f216be42846 --- /dev/null +++ b/eng/pipeline/stages/run-stage.yml @@ -0,0 +1,198 @@ +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This job runs a builder for any OS. + +parameters: + # { id, os, arch, hostArch, config, distro?, fips? } + builder: {} + createSourceArchive: false + +stages: + - stage: ${{ parameters.builder.id }} + # For display name, try for readability. Use some parameters set by + # shorthand-builders-to-builders.yml that let us add some formatting. + displayName: ${{ parameters.builder.os }}-${{ parameters.builder.arch }} ${{ parameters.builder.hostParens}} ${{ parameters.builder.config }} ${{ parameters.builder.distroParens}} ${{ parameters.builder.fipsAcronym }} + dependsOn: [] + jobs: + - job: ${{ parameters.builder.id }} + displayName: ${{ parameters.builder.os }}-${{ parameters.builder.arch }} ${{ parameters.builder.hostParens}} ${{ parameters.builder.config }} ${{ parameters.builder.distroParens}} ${{ parameters.builder.fipsAcronym }} + workspace: + clean: all + + ${{ if eq(parameters.builder.config, 'longtest') }}: + # longtest has been seen to succeed after 53 minutes. Give around 3x headroom. In the future, + # we should also give the tests a shorter timeout to make sure this doesn't balloon too far: + # https://github.com/microsoft/go/issues/568 + timeoutInMinutes: 180 + + ${{ if eq(parameters.builder.config, 'codeql') }}: + # Allow CodeQL to take a while. https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/configuring-codeql3000-ado-pipelines#other-issues + timeoutInMinutes: 360 + + ${{ if eq(parameters.builder.os, 'windows') }}: + pool: + name: ${{ parameters.dncengPool.name }} + demands: ${{ parameters.dncengPool.demands.windows }} + + ${{ if eq(parameters.builder.os, 'linux') }}: + ${{ if eq(parameters.builder.hostArch, 'amd64') }}: + # Docker host agent. + pool: + name: ${{ parameters.dncengPool.name }} + demands: ${{ parameters.dncengPool.demands.linux }} + # The image used for the container this job runs in. The tests run in this container, so + # it should match what we support as closely as possible. Don't use a container for the + # CodeQL scan build to avoid interfering with the LD_PRELOAD hook. + ${{ if ne(parameters.builder.config, 'codeql') }}: + ${{ if eq(parameters.builder.distro, 'ubuntu') }}: + container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20211022152710-047508b + ${{ else }}: + container: golangpublicimages.azurecr.io/go-infra-images/prereqs:cbl-mariner-1.0.20211027-20211201-0cccc22 + ${{ if eq(parameters.builder.hostArch, 'arm64') }}: + pool: + name: Docker-Linux-Arm-Internal + ${{ if not(parameters.builder.distro) }}: + container: golangpublicimages.azurecr.io/go-infra-images/prereqs:cbl-mariner-arm64-1.0-20220314-a003148 + + variables: + - group: go-cmdscan-rules + - ${{ if eq(parameters.builder.config, 'codeql') }}: + # Enable CodeQL scan and configure options. + # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/configuring-codeql3000-ado-pipelines#additional-options + - name: Codeql.Enabled + value: true + - name: Codeql.Language + value: go,cpp + # Always scan. This way we don't miss out on a release branch build, for example. + - name: Codeql.Cadence + value: 0 + + steps: + - ${{ if eq(parameters.builder.os, 'linux') }}: + # AzDO builds don't seem to set user ID in the running container, so files from a previous + # build might be owned by root and unable to be cleaned up by AzDO's cleanup step. Clean up + # the build dirs ourselves in another Docker container to avoid failures. + - script: | + set -x + echo 'Cleaning old build dirs with sudo in case of root ownership.' + sudo rm -v -rf a b s + mkdir a b s + workingDirectory: $(Agent.BuildDirectory) + displayName: Cleanup + + - template: ../steps/checkout-unix-task.yml + - template: ../steps/init-pwsh-task.yml + + - ${{ if eq(parameters.builder.os, 'windows') }}: + - template: ../steps/checkout-windows-task.yml + - pwsh: | + Write-Host "Increasing max build retries to mitigate 'Access denied' flakiness during EXE copying on Windows." + Write-Host "##vso[task.setvariable variable=GO_MAKE_MAX_RETRY_ATTEMPTS]5" + Write-Host "Increasing max test retries to mitigate access denied issues as well as general test flakiness on Windows." + Write-Host "##vso[task.setvariable variable=GO_TEST_MAX_RETRY_ATTEMPTS]5" + displayName: Increase 'make' retry attempts + + # Initialize stage 0 toolset ahead of time so we can track timing data separately from the + # build operations. When we call this script again later, it won't download Go again. + - pwsh: | + . eng/utilities.ps1 + Get-Stage0GoRoot + displayName: Init stage 0 Go toolset + + - template: ../steps/init-submodule-task.yml + + # Create the source archive on one job only. The os choice is arbitrary. + - ${{ if and(eq(parameters.createSourceArchive, true), eq(parameters.builder.config, 'buildandpack'), eq(parameters.builder.os, 'linux'), eq(parameters.builder.arch, 'amd64')) }}: + - pwsh: | + git config --global user.name "microsoft-golang-bot" + git config --global user.email "microsoft-golang-bot@users.noreply.github.com" + + # Turn the patches into commits, so HEAD includes the changes. + eng/run.ps1 submodule-refresh -commits + eng/run.ps1 pack-source + displayName: Archive submodule source + + # Manually init (and finalize, later) the CodeQL3000 extension so that it will run on any + # branch, not just the default. + # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/configuring-codeql3000-ado-pipelines#how-does-this-extension-work + - ${{ if eq(parameters.builder.config, 'codeql') }}: + - task: CodeQL3000Init@0 + + - pwsh: | + # Apply the patches as staged changes, so the HEAD commit is the same as upstream. + eng/run.ps1 submodule-refresh + displayName: Apply patches + + - ${{ if ne(parameters.builder.hostArch, parameters.builder.arch) }}: + - pwsh: Write-Host "##vso[task.setvariable variable=GOARCH]${{ parameters.builder.arch }}" + displayName: Set GOARCH for cross-compile + - ${{ if eq(parameters.builder.arch, 'arm') }}: + - pwsh: Write-Host "##vso[task.setvariable variable=GOARM]6" + displayName: Set GOARM for cross-compile + + # Use build script directly for "buildandpack". If we used run-builder, we would need to + # download its external module dependencies. + - ${{ if eq(parameters.builder.config, 'buildandpack' ) }}: + - pwsh: | + eng/run.ps1 cmdscan -envprefix GO_CMDSCAN_RULE_ -- ` + pwsh eng/run.ps1 build -pack + displayName: Build and Pack + + - publish: eng/artifacts/bin + artifact: Binaries ${{ parameters.builder.id }} + displayName: Pipeline publish + condition: succeededOrFailed() + + # CodeQL plugs into the compiler to find the code. Just build. + - ${{ elseif eq(parameters.builder.config, 'codeql' ) }}: + - pwsh: | + eng/run.ps1 cmdscan -envprefix GO_CMDSCAN_RULE_ -- ` + pwsh eng/run.ps1 build + displayName: Build + + # Use run-builder for any configuration that includes tests. run-builder uses the "gotestsum" + # module to convert test results to a JUnit file that Azure DevOps can understand. + - ${{ else }}: + - pwsh: | + if ($IsWindows) { + Write-Host "Removing Git usr\bin from PATH to avoid running a Linux test that would fail, 'TestScript/script_wait'..." + Write-Host $env:PATH + $env:PATH = ( + $env:PATH -split ';' | Where-Object { $_ -ne 'C:\Program Files\Git\usr\bin' } + ) -join ';' + } + + eng/run.ps1 cmdscan -envprefix GO_CMDSCAN_RULE_ -- ` + pwsh eng/run.ps1 run-builder ` + -builder '${{ parameters.builder.os }}-${{ parameters.builder.arch }}-${{ parameters.builder.config }}' ` + $(if ('${{ parameters.builder.fips }}') { '-fipsmode' }) ` + -junitfile '$(Build.SourcesDirectory)/eng/artifacts/TestResults.xml' + displayName: Run ${{ parameters.builder.config }} + + - task: PublishTestResults@2 + displayName: Publish test results + condition: succeededOrFailed() + inputs: + testResultsFormat: JUnit + testResultsFiles: $(Build.SourcesDirectory)/eng/artifacts/TestResults.xml + testRunTitle: $(System.JobDisplayName) + buildPlatform: ${{ parameters.builder.arch }} + buildConfiguration: ${{ parameters.builder.config }} + publishRunAttachments: true + + - ${{ if eq(parameters.builder.os, 'linux') }}: + # Files may be owned by root because builds don't set user ID. If this build is running on a + # persistent machine, later builds may fail to clean up this build's directory as as + # result--even if it also uses a build container. This step prevents that kind of failure by + # using chown to make sure the machine's agent user can access/delete the files. + - script: | + sudo chown -R $(id -u):$(id -g) * + workingDirectory: $(Agent.BuildDirectory) + displayName: Update file ownership from root to build agent account + continueOnError: true + condition: succeededOrFailed() + + - ${{ if eq(parameters.builder.config, 'codeql') }}: + - task: CodeQL3000Finalize@0 diff --git a/eng/pipeline/jobs/shorthand-builders-to-builders.yml b/eng/pipeline/stages/shorthand-builders-to-builders.yml similarity index 54% rename from eng/pipeline/jobs/shorthand-builders-to-builders.yml rename to eng/pipeline/stages/shorthand-builders-to-builders.yml index c1f7fe7baf6..6c1242c97f9 100644 --- a/eng/pipeline/jobs/shorthand-builders-to-builders.yml +++ b/eng/pipeline/stages/shorthand-builders-to-builders.yml @@ -1,34 +1,42 @@ -# Copyright (c) Microsoft Corporation. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# This template expands each "shorthand" builder in a list of builders into a builder object that -# includes its job ID. The list of builders is then passed into a given jobs template. This way, we -# don't need to repeat the ID evaluation template expression everywhere the value is needed. -# -# If any other builder-specific calculated value is needed based on shorthand properties, it can be -# added to this file. Passing data through a template like this one is the only way to share values -# to be used by template expressions, as of writing. - -parameters: - # [] of { os, arch, config, distro? } - shorthandBuilders: [] - # The inner jobs template to pass the filed-out builders into. - # - # It should accept parameter "builders", [] of { id, os, arch, config, distro? } - jobsTemplate: "" - jobsParameters: {} - -jobs: - - template: ${{ parameters.jobsTemplate }} - parameters: - ${{ insert }}: ${{ parameters.jobsParameters }} - builders: - - ${{ each builder in parameters.shorthandBuilders }}: - - ${{ insert }}: ${{ builder }} - ${{ if builder.distro }}: - id: ${{ builder.os }}_${{ builder.distro }}_${{ builder.arch }}_${{ builder.config }} - ${{ if not(builder.distro) }}: - id: ${{ builder.os }}_${{ builder.arch }}_${{ builder.config }} - - +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This template expands each "shorthand" builder in a list of builders into a builder object that +# includes its job ID. The list of builders is then passed into a given jobs template. This way, we +# don't need to repeat the ID evaluation template expression everywhere the value is needed. +# +# If any other builder-specific calculated value is needed based on shorthand properties, it can be +# added to this file. Passing data through a template like this one is the only way to share values +# to be used by template expressions, as of writing. + +parameters: + # [] of { os, arch, hostArch, config, distro? } + # If hostArch is not defined, defaults to the arch value. + shorthandBuilders: [] + # The inner jobs template to pass the filed-out builders into. + # + # It should accept parameter "builders", [] of { id, os, arch, hostArch, config, distro?, fips? } + jobsTemplate: "" + jobsParameters: {} + +stages: + - template: ${{ parameters.jobsTemplate }} + parameters: + ${{ insert }}: ${{ parameters.jobsParameters }} + builders: + - ${{ each builder in parameters.shorthandBuilders }}: + - ${{ insert }}: ${{ builder }} + # Use 'default' in place of null to define ID. This value just needs to be unique and + # only contain "[A-z_]+". + id: ${{ builder.os }}_${{ coalesce(builder.distro, 'default') }}_${{ coalesce(builder.hostArch, 'default') }}_${{ builder.arch }}_${{ builder.config }}_${{ coalesce(builder.fips, false) }} + ${{ if not(builder.hostArch) }}: + hostArch: ${{ builder.arch }} + # Set up some parameters that are for display purposes. AzDO YAML expressions can't + # branch, so we must to prepare these values here rather than where they're used. + ${{ if builder.distro }}: + distroParens: (${{ builder.distro }}) + ${{ if builder.hostArch }}: + hostParens: (${{ builder.hostArch }} host) + ${{ if builder.fips }}: + fipsAcronym: 'FIPS' diff --git a/eng/pipeline/stages/sign-stage.yml b/eng/pipeline/stages/sign-stage.yml new file mode 100644 index 00000000000..e176c3fa1c2 --- /dev/null +++ b/eng/pipeline/stages/sign-stage.yml @@ -0,0 +1,101 @@ +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This job downloads Go binaries from the specified builders' pipeline artifacts, signs them, and +# publishes the signed files and signatures into a consolidated pipeline artifact. + +parameters: + # [] of { id, os, arch, config, distro? } + builders: [] + +stages: + - stage: Sign + # Depend on all build stages that produced artifacts that need signing. + dependsOn: + - ${{ each builder in parameters.builders }}: + - ${{ builder.id }} + jobs: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: + - job: Sign + pool: + name: ${{ parameters.dncengPool.name }} + # This is a utility job that doesn't use Go, but must support signing. + demands: ${{ parameters.dncengPool.demands.windows }} + variables: + - name: TeamName + value: golang + workspace: + clean: all + steps: + - template: ../steps/checkout-windows-task.yml + + # Add MicroBuild, based on Arcade: https://github.com/dotnet/arcade/blob/927f8d4d5036f68a5fc6d042f336bc9458027208/eng/common/templates/job/job.yml#L106-L115 + - task: MicroBuildSigningPlugin@3 + displayName: Install MicroBuild plugin + inputs: + signType: $(SigningType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(TeamName) + condition: and(succeeded(), in(variables['SigningType'], 'real', 'test')) + + - ${{ each builder in parameters.builders }}: + - download: current + artifact: Binaries ${{ builder.id }} + displayName: 'Download: Binaries ${{ builder.id }}' + + - powershell: | + $flatDir = "$(Build.StagingDirectory)/ToSign" + New-Item $flatDir -ItemType Directory -ErrorAction Ignore + + Get-ChildItem -Recurse -File -Path @( + 'Binaries ${{ builder.id }}' + ) | %{ + if (Test-Path "$flatDir\$($_.Name)") { + throw "Duplicate filename, unable to flatten: $($_.FullName)" + } + Copy-Item $_.FullName $flatDir + } + displayName: 'Copy to flat dir: ${{ builder.id }}' + workingDirectory: '$(Pipeline.Workspace)' + + - task: DotNetCoreCLI@2 + displayName: 'Install Arcade SignTool' + inputs: + command: restore + projects: '$(Build.SourcesDirectory)/eng/signing/Sign.proj' + feedsToUse: config + nugetConfigPath: '$(Build.SourcesDirectory)/NuGet.config' + + - task: DotNetCoreCLI@2 + displayName: 'Sign Files' + inputs: + command: custom + projects: '$(Build.SourcesDirectory)/eng/signing/Sign.proj' + custom: msbuild + arguments: >- + /t:SignGoFiles + /p:SignFilesDir=$(Build.StagingDirectory)/ToSign + /p:SigningType=$(SigningType) + /bl:SignFiles.binlog + + - publish: $(Build.StagingDirectory)/ToSign + artifact: Binaries Signed + displayName: Publish Binaries Signed + condition: always() + + - publish: eng\signing + artifact: SigningDetails + displayName: Publish SigningDetails + condition: always() + + - publish: 'SignFiles.binlog' + artifact: SigningBinlog + displayName: Publish SigningBinlog + condition: always() + + - task: MicroBuildCleanup@1 + displayName: 'Clean up MicroBuild' + condition: always() diff --git a/eng/pipeline/steps/init-pwsh-task.yml b/eng/pipeline/steps/init-pwsh-task.yml index f8621029dc3..e3a903aa8ba 100644 --- a/eng/pipeline/steps/init-pwsh-task.yml +++ b/eng/pipeline/steps/init-pwsh-task.yml @@ -12,4 +12,6 @@ steps: . eng/init-pwsh.sh echo "##vso[task.prependpath]$pwsh_dir" + # Enable invariant mode to make .NET/PowerShell work without libicu installed. + echo "##vso[task.setvariable variable=DOTNET_SYSTEM_GLOBALIZATION_INVARIANT]1" displayName: Init PowerShell diff --git a/eng/pipeline/steps/init-submodule-task.yml b/eng/pipeline/steps/init-submodule-task.yml index 8e370d3b8ab..3ad5392f146 100644 --- a/eng/pipeline/steps/init-submodule-task.yml +++ b/eng/pipeline/steps/init-submodule-task.yml @@ -22,5 +22,5 @@ steps: env: FETCH_BEARER_TOKEN: $(System.AccessToken) displayName: Set up submodule from internal mirror - ${{ if eq(variables['System.TeamProject'], 'public') }}: + ${{ else }}: displayName: Set up submodule diff --git a/eng/run.ps1 b/eng/run.ps1 index 237ab6cc176..a12878de720 100644 --- a/eng/run.ps1 +++ b/eng/run.ps1 @@ -99,12 +99,18 @@ try { # Use a module-local path so Go resolves imports correctly. $module_local_script_path = Join-Path "." "cmd" "$tool" - Write-Host "In '$tool_module', building '$module_local_script_path' -> $tool_output" - & (Join-Path $stage0_goroot "bin" "go") build -o $tool_output $module_local_script_path - if ($LASTEXITCODE) { - Write-Host "Failed to build tool." - exit 1 + # The caller may have passed in GOOS/GOARCH to cross-compile Go. We can't use those values here: + # we need to be able to run the tool on the host, so we must always target the host OS/ARCH. Clear + # out the GOOS/GOARCH values (empty string) to detect host OS/ARCH automatically for the tools. + Invoke-CrossGoBlock "" "" { + Write-Host "In '$tool_module', building '$module_local_script_path' -> $tool_output" + & (Join-Path $stage0_goroot "bin" "go") build -o $tool_output $module_local_script_path + if ($LASTEXITCODE) { + Write-Host "Failed to build tool." + exit 1 + } } + Write-Host "Building done." } finally { Pop-Location diff --git a/eng/utilities.ps1 b/eng/utilities.ps1 index 70b21be4add..baff8efde17 100644 --- a/eng/utilities.ps1 +++ b/eng/utilities.ps1 @@ -24,14 +24,33 @@ function Get-Stage0GoRoot() { # pre-installed. This CI script installs a consistent, official version of Go to a directory in # $HOME to handle this. This also makes it easier to locally repro issues in CI that involve a # specific version of Go. The downloaded copy of Go is called the "stage 0" version. - $stage0_go_version = '1.16.5' + $stage0_go_version = '1.18' + $proc_arch = ([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture).ToString().ToLowerInvariant() if ($IsWindows) { - $stage0_go_sha256 = '0a3fa279ae5b91bc8c88017198c8f1ba5d9925eb6e5d7571316e567c73add39d' - $stage0_go_suffix = 'windows-amd64.zip' + switch ($proc_arch) { + 'x64' { + $stage0_go_sha256 = '65c5c0c709a7ca1b357091b10b795b439d8b50e579d3893edab4c7e9b384f435' + $stage0_go_suffix = 'windows-amd64.zip' + } + 'arm64' { + $stage0_go_sha256 = '1c454eb60c64d481965a165c623ff1ed6cf32d68c6b31f36069c8768d908f093' + $stage0_go_suffix = 'windows-arm64.zip' + } + Default { throw "Unable to match Windows '$proc_arch' to an architecture supported by the Microsoft scripts to build Go." } + } } elseif ($IsLinux) { - $stage0_go_sha256 = 'b12c23023b68de22f74c0524f10b753e7b08b1504cb7e417eccebdd3fae49061' - $stage0_go_suffix = 'linux-amd64.tar.gz' + switch ($proc_arch) { + 'x64' { + $stage0_go_sha256 = 'e85278e98f57cdb150fe8409e6e5df5343ecb13cebf03a5d5ff12bd55a80264f' + $stage0_go_suffix = 'linux-amd64.tar.gz' + } + 'arm64' { + $stage0_go_sha256 = '7ac7b396a691e588c5fb57687759e6c4db84a2a3bbebb0765f4b38e5b1c5b00e' + $stage0_go_suffix = 'linux-arm64.tar.gz' + } + Default { throw "Unable to match Linux '$proc_arch' to an architecture supported by the Microsoft scripts to build Go." } + } } else { throw "Current OS/Platform is not supported by the Microsoft scripts to build Go." } @@ -107,6 +126,20 @@ function Invoke-WithRetry([ScriptBlock]$ScriptBlock, [int]$MaxAttempts = 3, [int } } +function Invoke-CrossGoBlock([string] $GOOS, [string] $GOARCH, [ScriptBlock] $block) { + $oldGOOS = $env:GOOS + $oldGOARCH = $env:GOARCH + + try { + $env:GOOS = $GOOS + $env:GOARCH = $GOARCH + & $block + } finally { + $env:GOOS = $oldGOOS + $env:GOARCH = $oldGOARCH + } +} + # Utility method to unzip a file to a specific path. function Extract-Zip([string] $file, [string] $destination) { Add-Type -AssemblyName System.IO.Compression.FileSystem diff --git a/go b/go index 7900576bac4..7058c2cb413 160000 --- a/go +++ b/go @@ -1 +1 @@ -Subproject commit 7900576bac4630bbeec7f4f1aa4b1cb0d51bd8a1 +Subproject commit 7058c2cb4132291e1b6c9498a9a6f13a9a70ff85 diff --git a/patches/0001-cmd-dist-add-JSON-output-support-for-some-tests.patch b/patches/0001-cmd-dist-add-JSON-output-support-for-some-tests.patch index a9bcff182ae..f947b43a036 100644 --- a/patches/0001-cmd-dist-add-JSON-output-support-for-some-tests.patch +++ b/patches/0001-cmd-dist-add-JSON-output-support-for-some-tests.patch @@ -1,5 +1,5 @@ -From be418f20052ddea746451bafe21fdbf2bb1c915d Mon Sep 17 00:00:00 2001 -From: Davis Goodin +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: microsoft-golang-bot Date: Tue, 4 Jan 2022 11:23:27 -0600 Subject: [PATCH] cmd/dist: add JSON output support for some tests @@ -17,7 +17,7 @@ in JSON format, and the ordinary logs are still important. 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go -index 50a2e5936c..0436c63063 100644 +index f40fa926df..f76ecbe28c 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -29,6 +29,7 @@ func cmdtest() { @@ -36,7 +36,7 @@ index 50a2e5936c..0436c63063 100644 rebuild bool failed bool keepGoing bool -@@ -294,9 +296,13 @@ func short() string { +@@ -286,9 +288,13 @@ func short() string { // Callers should use goTest and then pass flags overriding these // defaults as later arguments in the command line. func (t *tester) goTest() []string { @@ -51,7 +51,7 @@ index 50a2e5936c..0436c63063 100644 } func (t *tester) tags() string { -@@ -379,6 +385,9 @@ func (t *tester) registerStdTest(pkg string, useG3 bool) { +@@ -371,6 +377,9 @@ func (t *tester) registerStdTest(pkg string, useG3 bool) { t.timeout(timeoutSec), "-gcflags=all=" + gcflags, } diff --git a/patches/0002-net-Skip-TestDialCancel-on-linux-arm64.patch b/patches/0002-net-Skip-TestDialCancel-on-linux-arm64.patch new file mode 100644 index 00000000000..0b6144107b3 --- /dev/null +++ b/patches/0002-net-Skip-TestDialCancel-on-linux-arm64.patch @@ -0,0 +1,25 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: microsoft-golang-bot +Date: Thu, 24 Feb 2022 17:57:24 -0600 +Subject: [PATCH] net: Skip TestDialCancel on linux-arm64 + +The test is flaky on our linux-arm64 builder and gets "network is unreachable". See https://github.com/golang/go/issues/37330 +--- + src/net/dial_test.go | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/src/net/dial_test.go b/src/net/dial_test.go +index 3cce444e5c..d001fc36af 100644 +--- a/src/net/dial_test.go ++++ b/src/net/dial_test.go +@@ -758,6 +758,10 @@ func TestDialerKeepAlive(t *testing.T) { + func TestDialCancel(t *testing.T) { + mustHaveExternalNetwork(t) + ++ if strings.HasPrefix(testenv.Builder(), "linux-arm64") { ++ t.Skip("skipping on linux-arm64-*; incompatible network config? issue 37330") ++ } ++ + if strings.HasPrefix(testenv.Builder(), "darwin-arm64") { + // The darwin-arm64 machines run in an environment that's not + // compatible with this test. diff --git a/patches/0100-Add-OpenSSL-crypto-module.patch b/patches/0100-Add-OpenSSL-crypto-module.patch new file mode 100644 index 00000000000..a50cfc60d0b --- /dev/null +++ b/patches/0100-Add-OpenSSL-crypto-module.patch @@ -0,0 +1,437 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: microsoft-golang-bot +Date: Thu, 31 Mar 2022 13:09:04 -0500 +Subject: [PATCH] Add OpenSSL crypto module + +github.com/microsoft/go-infra/cmd/git-go-patch command: patch number 0100 +--- + src/crypto/internal/backend/backend_test.go | 30 ++++ + src/crypto/internal/backend/common.go | 35 +++++ + src/crypto/internal/backend/dummy.s | 10 ++ + src/crypto/internal/backend/iscgo.go | 10 ++ + src/crypto/internal/backend/nobackend.go | 112 +++++++++++++++ + src/crypto/internal/backend/nocgo.go | 10 ++ + src/crypto/internal/backend/openssl_linux.go | 141 +++++++++++++++++++ + src/go.mod | 1 + + src/go.sum | 2 + + 9 files changed, 351 insertions(+) + create mode 100644 src/crypto/internal/backend/backend_test.go + create mode 100644 src/crypto/internal/backend/common.go + create mode 100644 src/crypto/internal/backend/dummy.s + create mode 100644 src/crypto/internal/backend/iscgo.go + create mode 100644 src/crypto/internal/backend/nobackend.go + create mode 100644 src/crypto/internal/backend/nocgo.go + create mode 100644 src/crypto/internal/backend/openssl_linux.go + +diff --git a/src/crypto/internal/backend/backend_test.go b/src/crypto/internal/backend/backend_test.go +new file mode 100644 +index 00000000000000..c2c06d3bff8c74 +--- /dev/null ++++ b/src/crypto/internal/backend/backend_test.go +@@ -0,0 +1,30 @@ ++// Copyright 2017 The Go Authors. All rights reserved. ++// Use of this source code is governed by a BSD-style ++// license that can be found in the LICENSE file. ++ ++package backend ++ ++import ( ++ "testing" ++) ++ ++// Test that Unreachable panics. ++func TestUnreachable(t *testing.T) { ++ defer func() { ++ if Enabled { ++ if err := recover(); err == nil { ++ t.Fatal("expected Unreachable to panic") ++ } ++ } else { ++ if err := recover(); err != nil { ++ t.Fatalf("expected Unreachable to be a no-op") ++ } ++ } ++ }() ++ Unreachable() ++} ++ ++// Test that UnreachableExceptTests does not panic (this is a test). ++func TestUnreachableExceptTests(t *testing.T) { ++ UnreachableExceptTests() ++} +diff --git a/src/crypto/internal/backend/common.go b/src/crypto/internal/backend/common.go +new file mode 100644 +index 00000000000000..9d39130dba8f43 +--- /dev/null ++++ b/src/crypto/internal/backend/common.go +@@ -0,0 +1,35 @@ ++// Copyright 2022 The Go Authors. All rights reserved. ++// Use of this source code is governed by a BSD-style ++// license that can be found in the LICENSE file. ++ ++//go:build !cmd_go_bootstrap && !msan && !gocrypt ++// +build !cmd_go_bootstrap,!msan,!gocrypt ++ ++package backend ++ ++import ( ++ "runtime" ++ "syscall" ++) ++ ++func init() { ++ if v, _ := envGoFIPS(); v == "1" { ++ if runtime.GOOS != "linux" { ++ panic("FIPS mode requested (GOFIPS=1) but not supported on " + runtime.GOOS) ++ } else if !iscgo { ++ panic("FIPS mode requested (GOFIPS=1) but not supported with cgo disabled") ++ } ++ } ++} ++ ++func envGoFIPS() (string, bool) { ++ if v, ok := syscall.Getenv("GOFIPS"); ok { ++ return v, true ++ } ++ // TODO: Decide which environment variable to use. ++ // See https://github.com/microsoft/go/issues/397. ++ if v, ok := syscall.Getenv("GOLANG_FIPS"); ok { ++ return v, true ++ } ++ return "", false ++} +diff --git a/src/crypto/internal/backend/dummy.s b/src/crypto/internal/backend/dummy.s +new file mode 100644 +index 00000000000000..157adeeeb37b21 +--- /dev/null ++++ b/src/crypto/internal/backend/dummy.s +@@ -0,0 +1,10 @@ ++// Copyright 2017 The Go Authors. All rights reserved. ++// Use of this source code is governed by a BSD-style ++// license that can be found in the LICENSE file. ++ ++// runtime_arg0 is declared in openssl_linux.go without a body. ++// It's provided by package runtime, ++// but the go command doesn't know that. ++// Having this assembly file keeps the go command ++// from complaining about the missing body ++// (because the implementation might be here). +diff --git a/src/crypto/internal/backend/iscgo.go b/src/crypto/internal/backend/iscgo.go +new file mode 100644 +index 00000000000000..1e0d3cf8c18b55 +--- /dev/null ++++ b/src/crypto/internal/backend/iscgo.go +@@ -0,0 +1,10 @@ ++// Copyright 2022 The Go Authors. All rights reserved. ++// Use of this source code is governed by a BSD-style ++// license that can be found in the LICENSE file. ++ ++//go:build cgo ++// +build cgo ++ ++package backend ++ ++const iscgo = true +diff --git a/src/crypto/internal/backend/nobackend.go b/src/crypto/internal/backend/nobackend.go +new file mode 100644 +index 00000000000000..4de8d404f8682c +--- /dev/null ++++ b/src/crypto/internal/backend/nobackend.go +@@ -0,0 +1,112 @@ ++// Copyright 2017 The Go Authors. All rights reserved. ++// Use of this source code is governed by a BSD-style ++// license that can be found in the LICENSE file. ++ ++//go:build !linux || !cgo || android || cmd_go_bootstrap || msan || gocrypt ++// +build !linux !cgo android cmd_go_bootstrap msan gocrypt ++ ++package backend ++ ++import ( ++ "crypto" ++ "crypto/cipher" ++ "crypto/internal/boring/sig" ++ "hash" ++ "math/big" ++) ++ ++const Enabled = false ++ ++// Unreachable marks code that should be unreachable ++// when OpenSSLCrypto is in use. It is a no-op without OpenSSLCrypto. ++func Unreachable() { ++ // Code that's unreachable when using OpenSSLCrypto ++ // is exactly the code we want to detect for reporting ++ // standard Go crypto. ++ sig.StandardCrypto() ++} ++ ++// UnreachableExceptTests marks code that should be unreachable ++// when OpenSSLCrypto is in use. It is a no-op without OpenSSLCrypto. ++func UnreachableExceptTests() {} ++ ++type randReader int ++ ++func (randReader) Read(b []byte) (int, error) { panic("opensslcrypto: not available") } ++ ++const RandReader = randReader(0) ++ ++func NewSHA1() hash.Hash { panic("opensslcrypto: not available") } ++func NewSHA224() hash.Hash { panic("opensslcrypto: not available") } ++func NewSHA256() hash.Hash { panic("opensslcrypto: not available") } ++func NewSHA384() hash.Hash { panic("opensslcrypto: not available") } ++func NewSHA512() hash.Hash { panic("opensslcrypto: not available") } ++ ++func NewHMAC(h func() hash.Hash, key []byte) hash.Hash { panic("opensslcrypto: not available") } ++ ++func NewAESCipher(key []byte) (cipher.Block, error) { panic("opensslcrypto: not available") } ++ ++type PublicKeyECDSA struct{ _ int } ++type PrivateKeyECDSA struct{ _ int } ++ ++func GenerateKeyECDSA(curve string) (X, Y, D *big.Int, err error) { ++ panic("opensslcrypto: not available") ++} ++func NewPrivateKeyECDSA(curve string, X, Y, D *big.Int) (*PrivateKeyECDSA, error) { ++ panic("opensslcrypto: not available") ++} ++func NewPublicKeyECDSA(curve string, X, Y *big.Int) (*PublicKeyECDSA, error) { ++ panic("opensslcrypto: not available") ++} ++func SignECDSA(priv *PrivateKeyECDSA, hash []byte) (r, s *big.Int, err error) { ++ panic("opensslcrypto: not available") ++} ++func SignMarshalECDSA(priv *PrivateKeyECDSA, hash []byte) ([]byte, error) { ++ panic("opensslcrypto: not available") ++} ++func VerifyECDSA(pub *PublicKeyECDSA, hash []byte, r, s *big.Int) bool { ++ panic("opensslcrypto: not available") ++} ++ ++type PublicKeyRSA struct{ _ int } ++type PrivateKeyRSA struct{ _ int } ++ ++func DecryptRSAOAEP(h hash.Hash, priv *PrivateKeyRSA, ciphertext, label []byte) ([]byte, error) { ++ panic("opensslcrypto: not available") ++} ++func DecryptRSAPKCS1(priv *PrivateKeyRSA, ciphertext []byte) ([]byte, error) { ++ panic("opensslcrypto: not available") ++} ++func DecryptRSANoPadding(priv *PrivateKeyRSA, ciphertext []byte) ([]byte, error) { ++ panic("opensslcrypto: not available") ++} ++func EncryptRSAOAEP(h hash.Hash, pub *PublicKeyRSA, msg, label []byte) ([]byte, error) { ++ panic("opensslcrypto: not available") ++} ++func EncryptRSAPKCS1(pub *PublicKeyRSA, msg []byte) ([]byte, error) { ++ panic("opensslcrypto: not available") ++} ++func EncryptRSANoPadding(pub *PublicKeyRSA, msg []byte) ([]byte, error) { ++ panic("opensslcrypto: not available") ++} ++func GenerateKeyRSA(bits int) (N, E, D, P, Q, Dp, Dq, Qinv *big.Int, err error) { ++ panic("opensslcrypto: not available") ++} ++func NewPrivateKeyRSA(N, E, D, P, Q, Dp, Dq, Qinv *big.Int) (*PrivateKeyRSA, error) { ++ panic("opensslcrypto: not available") ++} ++func NewPublicKeyRSA(N, E *big.Int) (*PublicKeyRSA, error) { ++ panic("opensslcrypto: not available") ++} ++func SignRSAPKCS1v15(priv *PrivateKeyRSA, h crypto.Hash, hashed []byte) ([]byte, error) { ++ panic("opensslcrypto: not available") ++} ++func SignRSAPSS(priv *PrivateKeyRSA, h crypto.Hash, hashed []byte, saltLen int) ([]byte, error) { ++ panic("opensslcrypto: not available") ++} ++func VerifyRSAPKCS1v15(pub *PublicKeyRSA, h crypto.Hash, hashed, sig []byte) error { ++ panic("opensslcrypto: not available") ++} ++func VerifyRSAPSS(pub *PublicKeyRSA, h crypto.Hash, hashed, sig []byte, saltLen int) error { ++ panic("opensslcrypto: not available") ++} +diff --git a/src/crypto/internal/backend/nocgo.go b/src/crypto/internal/backend/nocgo.go +new file mode 100644 +index 00000000000000..63c13fc4b01cba +--- /dev/null ++++ b/src/crypto/internal/backend/nocgo.go +@@ -0,0 +1,10 @@ ++// Copyright 2022 The Go Authors. All rights reserved. ++// Use of this source code is governed by a BSD-style ++// license that can be found in the LICENSE file. ++ ++//go:build !cgo ++// +build !cgo ++ ++package backend ++ ++const iscgo = false +diff --git a/src/crypto/internal/backend/openssl_linux.go b/src/crypto/internal/backend/openssl_linux.go +new file mode 100644 +index 00000000000000..659104006e84da +--- /dev/null ++++ b/src/crypto/internal/backend/openssl_linux.go +@@ -0,0 +1,141 @@ ++// Copyright 2017 The Go Authors. All rights reserved. ++// Use of this source code is governed by a BSD-style ++// license that can be found in the LICENSE file. ++ ++//go:build linux && cgo && !android && !gocrypt && !cmd_go_bootstrap && !msan ++// +build linux,cgo,!android,!gocrypt,!cmd_go_bootstrap,!msan ++ ++// Package openssl provides access to OpenSSLCrypto implementation functions. ++// Check the variable Enabled to find out whether OpenSSLCrypto is available. ++// If OpenSSLCrypto is not available, the functions in this package all panic. ++package backend ++ ++import ( ++ "crypto/internal/boring/sig" ++ "syscall" ++ ++ "github.com/microsoft/go-crypto-openssl/openssl" ++) ++ ++// Enabled controls whether FIPS crypto is enabled. ++var Enabled = false ++ ++func init() { ++ if !needFIPS() { ++ return ++ } ++ err := openssl.Init() ++ if err != nil { ++ panic(err) ++ } ++ ++ if !openssl.FIPS() { ++ if err = openssl.SetFIPS(true); err != nil { ++ panic(err) ++ } ++ } ++ ++ Enabled = true ++ sig.BoringCrypto() ++} ++ ++func needFIPS() bool { ++ if v, ok := envGoFIPS(); ok { ++ return v != "0" ++ } ++ var fd int ++ for { ++ var err error ++ fd, err = syscall.Open("/proc/sys/crypto/fips_enabled", syscall.O_RDONLY, 0) ++ if err == nil { ++ break ++ } ++ switch err { ++ case syscall.EINTR: ++ continue ++ case syscall.ENOENT: ++ return false ++ default: ++ // If there is an error reading we could either panic or assume FIPS is not enabled. ++ // Panicking would be too disruptive for apps that don't require FIPS. ++ // If an app wants to be 100% sure that is running in FIPS mode ++ // it should use boring.Enabled() or GOFIPS=1. ++ return false ++ } ++ } ++ defer syscall.Close(fd) ++ var tmp [1]byte ++ n, err := syscall.Read(fd, tmp[:]) ++ if n != 1 || err != nil { ++ // We return false instead of panicing for the same reason as before. ++ return false ++ } ++ // fips_enabled can be either '0' or '1'. ++ return tmp[0] == '1' ++} ++ ++// Unreachable marks code that should be unreachable ++// when OpenSSLCrypto is in use. It panics only when ++// the system is in FIPS mode. ++func Unreachable() { ++ if Enabled { ++ panic("opensslcrypto: invalid code execution") ++ } ++} ++ ++// Provided by runtime.crypto_backend_runtime_arg0 to avoid os import. ++func runtime_arg0() string ++ ++func hasSuffix(s, t string) bool { ++ return len(s) > len(t) && s[len(s)-len(t):] == t ++} ++ ++// UnreachableExceptTests marks code that should be unreachable ++// when OpenSSLCrypto is in use. It panics. ++func UnreachableExceptTests() { ++ name := runtime_arg0() ++ // If OpenSSLCrypto ran on Windows we'd need to allow _test.exe and .test.exe as well. ++ if Enabled && !hasSuffix(name, "_test") && !hasSuffix(name, ".test") { ++ println("opensslcrypto: unexpected code execution in", name) ++ panic("opensslcrypto: invalid code execution") ++ } ++} ++ ++const RandReader = openssl.RandReader ++ ++var NewSHA1 = openssl.NewSHA1 ++var NewSHA224 = openssl.NewSHA224 ++var NewSHA256 = openssl.NewSHA256 ++var NewSHA384 = openssl.NewSHA384 ++var NewSHA512 = openssl.NewSHA512 ++ ++var NewHMAC = openssl.NewHMAC ++ ++var NewAESCipher = openssl.NewAESCipher ++ ++type PublicKeyECDSA = openssl.PublicKeyECDSA ++type PrivateKeyECDSA = openssl.PrivateKeyECDSA ++ ++var GenerateKeyECDSA = openssl.GenerateKeyECDSA ++var NewPrivateKeyECDSA = openssl.NewPrivateKeyECDSA ++var NewPublicKeyECDSA = openssl.NewPublicKeyECDSA ++var SignECDSA = openssl.SignECDSA ++var SignMarshalECDSA = openssl.SignMarshalECDSA ++var VerifyECDSA = openssl.VerifyECDSA ++ ++type PublicKeyRSA = openssl.PublicKeyRSA ++type PrivateKeyRSA = openssl.PrivateKeyRSA ++ ++var DecryptRSAOAEP = openssl.DecryptRSAOAEP ++var DecryptRSAPKCS1 = openssl.DecryptRSAPKCS1 ++var DecryptRSANoPadding = openssl.DecryptRSANoPadding ++var EncryptRSAOAEP = openssl.EncryptRSAOAEP ++var EncryptRSAPKCS1 = openssl.EncryptRSAPKCS1 ++var EncryptRSANoPadding = openssl.EncryptRSANoPadding ++var GenerateKeyRSA = openssl.GenerateKeyRSA ++var NewPrivateKeyRSA = openssl.NewPrivateKeyRSA ++var NewPublicKeyRSA = openssl.NewPublicKeyRSA ++var SignRSAPKCS1v15 = openssl.SignRSAPKCS1v15 ++var SignRSAPSS = openssl.SignRSAPSS ++var VerifyRSAPKCS1v15 = openssl.VerifyRSAPKCS1v15 ++var VerifyRSAPSS = openssl.VerifyRSAPSS +diff --git a/src/go.mod b/src/go.mod +index e11c6d7df80053..5a46c4c944ce92 100644 +--- a/src/go.mod ++++ b/src/go.mod +@@ -3,6 +3,7 @@ module std + go 1.18 + + require ( ++ github.com/microsoft/go-crypto-openssl v0.1.2 + golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 + golang.org/x/net v0.0.0-20221214163811-6143a133e5c9 + ) +diff --git a/src/go.sum b/src/go.sum +index d2bfed64dc9738..46ee4c4442223c 100644 +--- a/src/go.sum ++++ b/src/go.sum +@@ -1,3 +1,5 @@ ++github.com/microsoft/go-crypto-openssl v0.1.2 h1:1JO/O62aLFCGpKWKrxdxzIITNJJ00hkBL0v/LLpEZUM= ++github.com/microsoft/go-crypto-openssl v0.1.2/go.mod h1:rC+rtBU3m60UCQifBmpWII0VETfu78w6YGZQvVc0rd4= + golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M= + golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= + golang.org/x/net v0.0.0-20221214163811-6143a133e5c9 h1:gcbGP3ZkgsHGpX/48qvg7Q/YmTtzZRWc/zpvN8XGSBg= diff --git a/patches/0101-Integrate-OpenSSL-module.patch b/patches/0101-Integrate-OpenSSL-module.patch new file mode 100644 index 00000000000..e22fb58837f --- /dev/null +++ b/patches/0101-Integrate-OpenSSL-module.patch @@ -0,0 +1,321 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: microsoft-golang-bot +Date: Thu, 31 Mar 2022 13:10:24 -0500 +Subject: [PATCH] Integrate OpenSSL module + +--- + src/cmd/link/internal/ld/lib.go | 2 +- + src/crypto/aes/cipher.go | 2 +- + src/crypto/aes/cipher_asm.go | 2 +- + src/crypto/boring/boring.go | 2 +- + src/crypto/ecdsa/boring.go | 2 +- + src/crypto/ecdsa/ecdsa.go | 2 +- + src/crypto/hmac/hmac.go | 2 +- + src/crypto/hmac/hmac_test.go | 2 +- + src/crypto/rand/rand_unix.go | 2 +- + src/crypto/rsa/boring.go | 2 +- + src/crypto/rsa/pkcs1v15.go | 2 +- + src/crypto/rsa/pss.go | 2 +- + src/crypto/rsa/rsa.go | 2 +- + src/crypto/rsa/rsa_test.go | 2 +- + src/crypto/sha1/boring.go | 4 ++-- + src/crypto/sha1/sha1_test.go | 2 +- + src/crypto/sha256/sha256.go | 2 +- + src/crypto/sha256/sha256_test.go | 2 +- + src/crypto/sha512/sha512.go | 2 +- + src/crypto/sha512/sha512_test.go | 2 +- + src/crypto/tls/cipher_suites.go | 2 +- + src/runtime/runtime_boring.go | 5 +++++ + 22 files changed, 27 insertions(+), 22 deletions(-) + +diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go +index 5b82dc287d..0e88b3ac94 100644 +--- a/src/cmd/link/internal/ld/lib.go ++++ b/src/cmd/link/internal/ld/lib.go +@@ -1015,7 +1015,7 @@ var hostobj []Hostobj + // These packages can use internal linking mode. + // Others trigger external mode. + var internalpkg = []string{ +- "crypto/internal/boring", ++ "vendor/github.com/microsoft/go-crypto-openssl/openssl", + "crypto/x509", + "net", + "os/user", +diff --git a/src/crypto/aes/cipher.go b/src/crypto/aes/cipher.go +index 29d01796eb..f3680ad6b4 100644 +--- a/src/crypto/aes/cipher.go ++++ b/src/crypto/aes/cipher.go +@@ -10,7 +10,7 @@ import ( + "strconv" + ) + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + // The AES block size in bytes. + const BlockSize = 16 +diff --git a/src/crypto/aes/cipher_asm.go b/src/crypto/aes/cipher_asm.go +index ecc6ccbbfb..a8860527fa 100644 +--- a/src/crypto/aes/cipher_asm.go ++++ b/src/crypto/aes/cipher_asm.go +@@ -12,7 +12,7 @@ import ( + "internal/cpu" + ) + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + // defined in asm_*.s + +diff --git a/src/crypto/boring/boring.go b/src/crypto/boring/boring.go +index 19e2a0876f..2829231f4a 100644 +--- a/src/crypto/boring/boring.go ++++ b/src/crypto/boring/boring.go +@@ -11,7 +11,7 @@ + // is satisfied, so that applications can tag files that use this package. + package boring + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + // Enabled reports whether BoringCrypto handles supported crypto operations. + func Enabled() bool { +diff --git a/src/crypto/ecdsa/boring.go b/src/crypto/ecdsa/boring.go +index fa15ecb850..92c42e28d5 100644 +--- a/src/crypto/ecdsa/boring.go ++++ b/src/crypto/ecdsa/boring.go +@@ -5,7 +5,7 @@ + package ecdsa + + import ( +- "crypto/internal/boring" ++ boring "crypto/internal/backend" + "math/big" + "sync/atomic" + "unsafe" +diff --git a/src/crypto/ecdsa/ecdsa.go b/src/crypto/ecdsa/ecdsa.go +index c1dd32a2d8..1471ec44fa 100644 +--- a/src/crypto/ecdsa/ecdsa.go ++++ b/src/crypto/ecdsa/ecdsa.go +@@ -35,7 +35,7 @@ import ( + ) + + import ( +- "crypto/internal/boring" ++ boring "crypto/internal/backend" + "unsafe" + ) + +diff --git a/src/crypto/hmac/hmac.go b/src/crypto/hmac/hmac.go +index 34805765d5..79fd58d0da 100644 +--- a/src/crypto/hmac/hmac.go ++++ b/src/crypto/hmac/hmac.go +@@ -26,7 +26,7 @@ import ( + "hash" + ) + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + // FIPS 198-1: + // https://csrc.nist.gov/publications/fips/fips198-1/FIPS-198-1_final.pdf +diff --git a/src/crypto/hmac/hmac_test.go b/src/crypto/hmac/hmac_test.go +index 55415abf02..904925377b 100644 +--- a/src/crypto/hmac/hmac_test.go ++++ b/src/crypto/hmac/hmac_test.go +@@ -6,7 +6,7 @@ package hmac + + import ( + "bytes" +- "crypto/internal/boring" ++ boring "crypto/internal/backend" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" +diff --git a/src/crypto/rand/rand_unix.go b/src/crypto/rand/rand_unix.go +index 28f2f5b58b..8112183bfa 100644 +--- a/src/crypto/rand/rand_unix.go ++++ b/src/crypto/rand/rand_unix.go +@@ -22,7 +22,7 @@ import ( + "time" + ) + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + const urandomDevice = "/dev/urandom" + +diff --git a/src/crypto/rsa/boring.go b/src/crypto/rsa/boring.go +index 0f362a2f16..856bc26aea 100644 +--- a/src/crypto/rsa/boring.go ++++ b/src/crypto/rsa/boring.go +@@ -5,7 +5,7 @@ + package rsa + + import ( +- "crypto/internal/boring" ++ boring "crypto/internal/backend" + "math/big" + "sync/atomic" + "unsafe" +diff --git a/src/crypto/rsa/pkcs1v15.go b/src/crypto/rsa/pkcs1v15.go +index 213ddb4add..5a44b4a71c 100644 +--- a/src/crypto/rsa/pkcs1v15.go ++++ b/src/crypto/rsa/pkcs1v15.go +@@ -14,7 +14,7 @@ import ( + "crypto/internal/randutil" + ) + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + // This file implements encryption and decryption using PKCS #1 v1.5 padding. + +diff --git a/src/crypto/rsa/pss.go b/src/crypto/rsa/pss.go +index 16ebc0e6a7..54afa8992e 100644 +--- a/src/crypto/rsa/pss.go ++++ b/src/crypto/rsa/pss.go +@@ -15,7 +15,7 @@ import ( + "math/big" + ) + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + // Per RFC 8017, Section 9.1 + // +diff --git a/src/crypto/rsa/rsa.go b/src/crypto/rsa/rsa.go +index eef967f826..550e66fdd9 100644 +--- a/src/crypto/rsa/rsa.go ++++ b/src/crypto/rsa/rsa.go +@@ -36,7 +36,7 @@ import ( + ) + + import ( +- "crypto/internal/boring" ++ boring "crypto/internal/backend" + "unsafe" + ) + +diff --git a/src/crypto/rsa/rsa_test.go b/src/crypto/rsa/rsa_test.go +index 766d9a954f..f2602b94ab 100644 +--- a/src/crypto/rsa/rsa_test.go ++++ b/src/crypto/rsa/rsa_test.go +@@ -15,7 +15,7 @@ import ( + "testing" + ) + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + func TestKeyGeneration(t *testing.T) { + for _, size := range []int{128, 1024, 2048, 3072} { +diff --git a/src/crypto/sha1/boring.go b/src/crypto/sha1/boring.go +index 1cacf93f9b..0d4305f379 100644 +--- a/src/crypto/sha1/boring.go ++++ b/src/crypto/sha1/boring.go +@@ -12,11 +12,11 @@ + package sha1 + + import ( +- "crypto/internal/boring" ++ boring "crypto/internal/backend" + "hash" + ) + +-const boringEnabled = boring.Enabled ++var boringEnabled = boring.Enabled + + func boringNewSHA1() hash.Hash { return boring.NewSHA1() } + +diff --git a/src/crypto/sha1/sha1_test.go b/src/crypto/sha1/sha1_test.go +index f1a5448dd2..f5a8fe053d 100644 +--- a/src/crypto/sha1/sha1_test.go ++++ b/src/crypto/sha1/sha1_test.go +@@ -16,7 +16,7 @@ import ( + "testing" + ) + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + type sha1Test struct { + out string +diff --git a/src/crypto/sha256/sha256.go b/src/crypto/sha256/sha256.go +index 465d00e0e6..cec4a49c8c 100644 +--- a/src/crypto/sha256/sha256.go ++++ b/src/crypto/sha256/sha256.go +@@ -13,7 +13,7 @@ import ( + "hash" + ) + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + func init() { + crypto.RegisterHash(crypto.SHA224, New224) +diff --git a/src/crypto/sha256/sha256_test.go b/src/crypto/sha256/sha256_test.go +index a762afc4d9..a30ad62f89 100644 +--- a/src/crypto/sha256/sha256_test.go ++++ b/src/crypto/sha256/sha256_test.go +@@ -16,7 +16,7 @@ import ( + "testing" + ) + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + type sha256Test struct { + out string +diff --git a/src/crypto/sha512/sha512.go b/src/crypto/sha512/sha512.go +index 1285cca7ee..4ac54ad5aa 100644 +--- a/src/crypto/sha512/sha512.go ++++ b/src/crypto/sha512/sha512.go +@@ -17,7 +17,7 @@ import ( + "hash" + ) + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + func init() { + crypto.RegisterHash(crypto.SHA384, New384) +diff --git a/src/crypto/sha512/sha512_test.go b/src/crypto/sha512/sha512_test.go +index 99d1423527..9042fc5337 100644 +--- a/src/crypto/sha512/sha512_test.go ++++ b/src/crypto/sha512/sha512_test.go +@@ -17,7 +17,7 @@ import ( + "testing" + ) + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + type sha512Test struct { + out string +diff --git a/src/crypto/tls/cipher_suites.go b/src/crypto/tls/cipher_suites.go +index 76312984ab..98bbeb9c47 100644 +--- a/src/crypto/tls/cipher_suites.go ++++ b/src/crypto/tls/cipher_suites.go +@@ -4,7 +4,7 @@ + + package tls + +-import "crypto/internal/boring" ++import boring "crypto/internal/backend" + + import ( + "crypto" +diff --git a/src/runtime/runtime_boring.go b/src/runtime/runtime_boring.go +index 5a98b20253..9042f2c279 100644 +--- a/src/runtime/runtime_boring.go ++++ b/src/runtime/runtime_boring.go +@@ -17,3 +17,8 @@ func boring_runtime_arg0() string { + + //go:linkname fipstls_runtime_arg0 crypto/internal/boring/fipstls.runtime_arg0 + func fipstls_runtime_arg0() string { return boring_runtime_arg0() } ++ ++//go:linkname crypto_backend_runtime_arg0 crypto/internal/backend.runtime_arg0 ++func crypto_backend_runtime_arg0() string { ++ return boring_runtime_arg0() ++} diff --git a/patches/0102-Vendor-OpenSSL-crypto-library.patch b/patches/0102-Vendor-OpenSSL-crypto-library.patch new file mode 100644 index 00000000000..d4099c05aed --- /dev/null +++ b/patches/0102-Vendor-OpenSSL-crypto-library.patch @@ -0,0 +1,2997 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: microsoft-golang-bot +Date: Thu, 31 Mar 2022 13:10:45 -0500 +Subject: [PATCH] Vendor OpenSSL crypto library + +To reproduce, run 'go mod vendor' in 'go/src'. +--- + .../microsoft/go-crypto-openssl/LICENSE | 21 + + .../go-crypto-openssl/openssl/aes.go | 466 +++++++++++++++ + .../go-crypto-openssl/openssl/ecdsa.go | 214 +++++++ + .../go-crypto-openssl/openssl/evpkey.go | 279 +++++++++ + .../go-crypto-openssl/openssl/goopenssl.c | 153 +++++ + .../go-crypto-openssl/openssl/goopenssl.h | 133 +++++ + .../go-crypto-openssl/openssl/hmac.go | 148 +++++ + .../openssl/internal/subtle/aliasing.go | 32 ++ + .../go-crypto-openssl/openssl/openssl.go | 260 +++++++++ + .../go-crypto-openssl/openssl/openssl_funcs.h | 242 ++++++++ + .../openssl/openssl_lock_setup.c | 53 ++ + .../go-crypto-openssl/openssl/rand.go | 24 + + .../go-crypto-openssl/openssl/rsa.go | 303 ++++++++++ + .../go-crypto-openssl/openssl/sha.go | 534 ++++++++++++++++++ + src/vendor/modules.txt | 4 + + 15 files changed, 2866 insertions(+) + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/LICENSE + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/aes.go + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/ecdsa.go + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/evpkey.go + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/goopenssl.c + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/goopenssl.h + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/hmac.go + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/internal/subtle/aliasing.go + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/openssl.go + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/openssl_funcs.h + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/openssl_lock_setup.c + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/rand.go + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/rsa.go + create mode 100644 src/vendor/github.com/microsoft/go-crypto-openssl/openssl/sha.go + +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/LICENSE b/src/vendor/github.com/microsoft/go-crypto-openssl/LICENSE +new file mode 100644 +index 00000000000000..9e841e7a26e4eb +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/LICENSE +@@ -0,0 +1,21 @@ ++ MIT License ++ ++ Copyright (c) Microsoft Corporation. ++ ++ Permission is hereby granted, free of charge, to any person obtaining a copy ++ of this software and associated documentation files (the "Software"), to deal ++ in the Software without restriction, including without limitation the rights ++ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell ++ copies of the Software, and to permit persons to whom the Software is ++ furnished to do so, subject to the following conditions: ++ ++ The above copyright notice and this permission notice shall be included in all ++ copies or substantial portions of the Software. ++ ++ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ++ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ SOFTWARE +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/aes.go b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/aes.go +new file mode 100644 +index 00000000000000..3177c0d37c8e5f +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/aes.go +@@ -0,0 +1,466 @@ ++// Copyright (c) Microsoft Corporation. ++// Licensed under the MIT License. ++ ++//go:build linux && !android ++// +build linux,!android ++ ++package openssl ++ ++// #include "goopenssl.h" ++import "C" ++import ( ++ "crypto/cipher" ++ "errors" ++ "runtime" ++ "strconv" ++ "unsafe" ++ ++ "github.com/microsoft/go-crypto-openssl/openssl/internal/subtle" ++) ++ ++type aesKeySizeError int ++ ++func (k aesKeySizeError) Error() string { ++ return "crypto/aes: invalid key size " + strconv.Itoa(int(k)) ++} ++ ++const aesBlockSize = 16 ++ ++type aesCipher struct { ++ key []byte ++ enc_ctx C.GO_EVP_CIPHER_CTX_PTR ++ dec_ctx C.GO_EVP_CIPHER_CTX_PTR ++ cipher C.GO_EVP_CIPHER_PTR ++} ++ ++type extraModes interface { ++ // Copied out of crypto/aes/modes.go. ++ NewCBCEncrypter(iv []byte) cipher.BlockMode ++ NewCBCDecrypter(iv []byte) cipher.BlockMode ++ NewCTR(iv []byte) cipher.Stream ++ NewGCM(nonceSize, tagSize int) (cipher.AEAD, error) ++ ++ // Invented for BoringCrypto. ++ NewGCMTLS() (cipher.AEAD, error) ++} ++ ++var _ extraModes = (*aesCipher)(nil) ++ ++func NewAESCipher(key []byte) (cipher.Block, error) { ++ c := &aesCipher{key: make([]byte, len(key))} ++ copy(c.key, key) ++ ++ switch len(c.key) * 8 { ++ case 128: ++ c.cipher = C.go_openssl_EVP_aes_128_ecb() ++ case 192: ++ c.cipher = C.go_openssl_EVP_aes_192_ecb() ++ case 256: ++ c.cipher = C.go_openssl_EVP_aes_256_ecb() ++ default: ++ return nil, errors.New("crypto/cipher: Invalid key size") ++ } ++ ++ runtime.SetFinalizer(c, (*aesCipher).finalize) ++ ++ return c, nil ++} ++ ++func (c *aesCipher) finalize() { ++ if c.enc_ctx != nil { ++ C.go_openssl_EVP_CIPHER_CTX_free(c.enc_ctx) ++ } ++ if c.dec_ctx != nil { ++ C.go_openssl_EVP_CIPHER_CTX_free(c.dec_ctx) ++ } ++} ++ ++func (c *aesCipher) BlockSize() int { return aesBlockSize } ++ ++func (c *aesCipher) Encrypt(dst, src []byte) { ++ if subtle.InexactOverlap(dst, src) { ++ panic("crypto/cipher: invalid buffer overlap") ++ } ++ if len(src) < aesBlockSize { ++ panic("crypto/aes: input not full block") ++ } ++ if len(dst) < aesBlockSize { ++ panic("crypto/aes: output not full block") ++ } ++ ++ if c.enc_ctx == nil { ++ var err error ++ c.enc_ctx, err = newCipherCtx(c.cipher, C.GO_AES_ENCRYPT, c.key, nil) ++ if err != nil { ++ panic(err) ++ } ++ } ++ ++ C.go_openssl_EVP_EncryptUpdate_wrapper(c.enc_ctx, base(dst), base(src), aesBlockSize) ++ runtime.KeepAlive(c) ++} ++ ++func (c *aesCipher) Decrypt(dst, src []byte) { ++ if subtle.InexactOverlap(dst, src) { ++ panic("crypto/cipher: invalid buffer overlap") ++ } ++ if len(src) < aesBlockSize { ++ panic("crypto/aes: input not full block") ++ } ++ if len(dst) < aesBlockSize { ++ panic("crypto/aes: output not full block") ++ } ++ if c.dec_ctx == nil { ++ var err error ++ c.dec_ctx, err = newCipherCtx(c.cipher, C.GO_AES_DECRYPT, c.key, nil) ++ if err != nil { ++ panic(err) ++ } ++ } ++ ++ C.go_openssl_EVP_DecryptUpdate_wrapper(c.dec_ctx, base(dst), base(src), aesBlockSize) ++ runtime.KeepAlive(c) ++} ++ ++type aesCBC struct { ++ ctx C.GO_EVP_CIPHER_CTX_PTR ++} ++ ++func (x *aesCBC) BlockSize() int { return aesBlockSize } ++ ++func (x *aesCBC) CryptBlocks(dst, src []byte) { ++ if subtle.InexactOverlap(dst, src) { ++ panic("crypto/cipher: invalid buffer overlap") ++ } ++ if len(src)%aesBlockSize != 0 { ++ panic("crypto/cipher: input not full blocks") ++ } ++ if len(dst) < len(src) { ++ panic("crypto/cipher: output smaller than input") ++ } ++ if len(src) > 0 { ++ if C.go_openssl_EVP_CipherUpdate_wrapper(x.ctx, base(dst), base(src), C.int(len(src))) != 1 { ++ panic("crypto/cipher: CipherUpdate failed") ++ } ++ runtime.KeepAlive(x) ++ } ++} ++ ++func (x *aesCBC) SetIV(iv []byte) { ++ if len(iv) != aesBlockSize { ++ panic("cipher: incorrect length IV") ++ } ++ if C.go_openssl_EVP_CipherInit_ex(x.ctx, nil, nil, nil, base(iv), -1) != 1 { ++ panic("cipher: unable to initialize EVP cipher ctx") ++ } ++} ++ ++func (c *aesCipher) NewCBCEncrypter(iv []byte) cipher.BlockMode { ++ x := new(aesCBC) ++ ++ var cipher C.GO_EVP_CIPHER_PTR ++ switch len(c.key) * 8 { ++ case 128: ++ cipher = C.go_openssl_EVP_aes_128_cbc() ++ case 192: ++ cipher = C.go_openssl_EVP_aes_192_cbc() ++ case 256: ++ cipher = C.go_openssl_EVP_aes_256_cbc() ++ default: ++ panic("openssl: unsupported key length") ++ } ++ var err error ++ x.ctx, err = newCipherCtx(cipher, C.GO_AES_ENCRYPT, c.key, iv) ++ if err != nil { ++ panic(err) ++ } ++ ++ runtime.SetFinalizer(x, (*aesCBC).finalize) ++ ++ return x ++} ++ ++func (c *aesCBC) finalize() { ++ C.go_openssl_EVP_CIPHER_CTX_free(c.ctx) ++} ++ ++func (c *aesCipher) NewCBCDecrypter(iv []byte) cipher.BlockMode { ++ x := new(aesCBC) ++ ++ var cipher C.GO_EVP_CIPHER_PTR ++ switch len(c.key) * 8 { ++ case 128: ++ cipher = C.go_openssl_EVP_aes_128_cbc() ++ case 192: ++ cipher = C.go_openssl_EVP_aes_192_cbc() ++ case 256: ++ cipher = C.go_openssl_EVP_aes_256_cbc() ++ default: ++ panic("openssl: unsupported key length") ++ } ++ ++ var err error ++ x.ctx, err = newCipherCtx(cipher, C.GO_AES_DECRYPT, c.key, iv) ++ if err != nil { ++ panic(err) ++ } ++ if C.go_openssl_EVP_CIPHER_CTX_set_padding(x.ctx, 0) != 1 { ++ panic("cipher: unable to set padding") ++ } ++ ++ runtime.SetFinalizer(x, (*aesCBC).finalize) ++ return x ++} ++ ++type aesCTR struct { ++ ctx C.GO_EVP_CIPHER_CTX_PTR ++} ++ ++func (x *aesCTR) XORKeyStream(dst, src []byte) { ++ if subtle.InexactOverlap(dst, src) { ++ panic("crypto/cipher: invalid buffer overlap") ++ } ++ if len(dst) < len(src) { ++ panic("crypto/cipher: output smaller than input") ++ } ++ if len(src) == 0 { ++ return ++ } ++ C.go_openssl_EVP_EncryptUpdate_wrapper(x.ctx, base(dst), base(src), C.int(len(src))) ++ runtime.KeepAlive(x) ++} ++ ++func (c *aesCipher) NewCTR(iv []byte) cipher.Stream { ++ x := new(aesCTR) ++ ++ var cipher C.GO_EVP_CIPHER_PTR ++ switch len(c.key) * 8 { ++ case 128: ++ cipher = C.go_openssl_EVP_aes_128_ctr() ++ case 192: ++ cipher = C.go_openssl_EVP_aes_192_ctr() ++ case 256: ++ cipher = C.go_openssl_EVP_aes_256_ctr() ++ default: ++ panic("openssl: unsupported key length") ++ } ++ var err error ++ x.ctx, err = newCipherCtx(cipher, C.GO_AES_ENCRYPT, c.key, iv) ++ if err != nil { ++ panic(err) ++ } ++ ++ runtime.SetFinalizer(x, (*aesCTR).finalize) ++ ++ return x ++} ++ ++func (c *aesCTR) finalize() { ++ C.go_openssl_EVP_CIPHER_CTX_free(c.ctx) ++} ++ ++type aesGCM struct { ++ ctx C.GO_EVP_CIPHER_CTX_PTR ++ tls bool ++ minNextNonce uint64 ++} ++ ++const ( ++ gcmTagSize = 16 ++ gcmStandardNonceSize = 12 ++ gcmTlsAddSize = 13 ++ gcmTlsFixedNonceSize = 4 ++) ++ ++type aesNonceSizeError int ++ ++func (n aesNonceSizeError) Error() string { ++ return "crypto/aes: invalid GCM nonce size " + strconv.Itoa(int(n)) ++} ++ ++type noGCM struct { ++ cipher.Block ++} ++ ++func (c *aesCipher) NewGCM(nonceSize, tagSize int) (cipher.AEAD, error) { ++ if nonceSize != gcmStandardNonceSize && tagSize != gcmTagSize { ++ return nil, errors.New("crypto/aes: GCM tag and nonce sizes can't be non-standard at the same time") ++ } ++ // Fall back to standard library for GCM with non-standard nonce or tag size. ++ if nonceSize != gcmStandardNonceSize { ++ return cipher.NewGCMWithNonceSize(&noGCM{c}, nonceSize) ++ } ++ if tagSize != gcmTagSize { ++ return cipher.NewGCMWithTagSize(&noGCM{c}, tagSize) ++ } ++ return c.newGCM(false) ++} ++ ++func (c *aesCipher) NewGCMTLS() (cipher.AEAD, error) { ++ return c.newGCM(true) ++} ++ ++func (c *aesCipher) newGCM(tls bool) (cipher.AEAD, error) { ++ var cipher C.GO_EVP_CIPHER_PTR ++ switch len(c.key) * 8 { ++ case 128: ++ cipher = C.go_openssl_EVP_aes_128_gcm() ++ case 192: ++ cipher = C.go_openssl_EVP_aes_192_gcm() ++ case 256: ++ cipher = C.go_openssl_EVP_aes_256_gcm() ++ default: ++ panic("openssl: unsupported key length") ++ } ++ ctx, err := newCipherCtx(cipher, -1, c.key, nil) ++ if err != nil { ++ return nil, err ++ } ++ g := &aesGCM{ctx: ctx, tls: tls} ++ runtime.SetFinalizer(g, (*aesGCM).finalize) ++ return g, nil ++} ++ ++func (g *aesGCM) finalize() { ++ C.go_openssl_EVP_CIPHER_CTX_free(g.ctx) ++} ++ ++func (g *aesGCM) NonceSize() int { ++ return gcmStandardNonceSize ++} ++ ++func (g *aesGCM) Overhead() int { ++ return gcmTagSize ++} ++ ++// base returns the address of the underlying array in b, ++// being careful not to panic when b has zero length. ++func base(b []byte) *C.uchar { ++ if len(b) == 0 { ++ return nil ++ } ++ return (*C.uchar)(unsafe.Pointer(&b[0])) ++} ++ ++func (g *aesGCM) Seal(dst, nonce, plaintext, additionalData []byte) []byte { ++ if len(nonce) != gcmStandardNonceSize { ++ panic("cipher: incorrect nonce length given to GCM") ++ } ++ if uint64(len(plaintext)) > ((1<<32)-2)*aesBlockSize || len(plaintext)+gcmTagSize < len(plaintext) { ++ panic("cipher: message too large for GCM") ++ } ++ if len(dst)+len(plaintext)+gcmTagSize < len(dst) { ++ panic("cipher: message too large for buffer") ++ } ++ if g.tls { ++ if len(additionalData) != gcmTlsAddSize { ++ panic("cipher: incorrect additional data length given to GCM TLS") ++ } ++ // BoringCrypto enforces strictly monotonically increasing explicit nonces ++ // and to fail after 2^64 - 1 keys as per FIPS 140-2 IG A.5, ++ // but OpenSSL does not perform this check, so it is implemented here. ++ const maxUint64 = 1<<64 - 1 ++ counter := bigUint64(nonce[gcmTlsFixedNonceSize:]) ++ if counter == maxUint64 { ++ panic("cipher: nonce counter must be less than 2^64 - 1") ++ } ++ if counter < g.minNextNonce { ++ panic("cipher: nonce counter must be strictly monotonically increasing") ++ } ++ defer func() { ++ g.minNextNonce = counter + 1 ++ }() ++ } ++ ++ // Make room in dst to append plaintext+overhead. ++ ret, out := sliceForAppend(dst, len(plaintext)+gcmTagSize) ++ ++ // Check delayed until now to make sure len(dst) is accurate. ++ if subtle.InexactOverlap(out, plaintext) { ++ panic("cipher: invalid buffer overlap") ++ } ++ ++ // Encrypt additional data. ++ // When sealing a TLS payload, OpenSSL app sets the additional data using ++ // 'EVP_CIPHER_CTX_ctrl(g.ctx, C.EVP_CTRL_AEAD_TLS1_AAD, C.EVP_AEAD_TLS1_AAD_LEN, base(additionalData))'. ++ // This makes the explicit nonce component to monotonically increase on every Seal operation without ++ // relying in the explicit nonce being securely set externally, ++ // and it also gives some interesting speed gains. ++ // Unfortunately we can't use it because Go expects AEAD.Seal to honor the provided nonce. ++ if C.go_openssl_EVP_CIPHER_CTX_seal_wrapper(g.ctx, base(out), base(nonce), ++ base(plaintext), C.int(len(plaintext)), ++ base(additionalData), C.int(len(additionalData))) != 1 { ++ ++ panic(fail("EVP_CIPHER_CTX_seal")) ++ } ++ runtime.KeepAlive(g) ++ return ret ++} ++ ++var errOpen = errors.New("cipher: message authentication failed") ++ ++func (g *aesGCM) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) { ++ if len(nonce) != gcmStandardNonceSize { ++ panic("cipher: incorrect nonce length given to GCM") ++ } ++ if len(ciphertext) < gcmTagSize { ++ return nil, errOpen ++ } ++ if uint64(len(ciphertext)) > ((1<<32)-2)*aesBlockSize+gcmTagSize { ++ return nil, errOpen ++ } ++ // BoringCrypto does not do any TLS check when decrypting, neither do we. ++ ++ tag := ciphertext[len(ciphertext)-gcmTagSize:] ++ ciphertext = ciphertext[:len(ciphertext)-gcmTagSize] ++ ++ // Make room in dst to append ciphertext without tag. ++ ret, out := sliceForAppend(dst, len(ciphertext)) ++ ++ // Check delayed until now to make sure len(dst) is accurate. ++ if subtle.InexactOverlap(out, ciphertext) { ++ panic("cipher: invalid buffer overlap") ++ } ++ ++ if C.go_openssl_EVP_CIPHER_CTX_open_wrapper(g.ctx, base(out), base(nonce), ++ base(ciphertext), C.int(len(ciphertext)), ++ base(additionalData), C.int(len(additionalData)), base(tag)) != 1 { ++ ++ for i := range out { ++ out[i] = 0 ++ } ++ return nil, errOpen ++ } ++ runtime.KeepAlive(g) ++ return ret, nil ++} ++ ++// sliceForAppend is a mirror of crypto/cipher.sliceForAppend. ++func sliceForAppend(in []byte, n int) (head, tail []byte) { ++ if total := len(in) + n; cap(in) >= total { ++ head = in[:total] ++ } else { ++ head = make([]byte, total) ++ copy(head, in) ++ } ++ tail = head[len(in):] ++ return ++} ++ ++func newCipherCtx(cipher C.GO_EVP_CIPHER_PTR, mode C.int, key, iv []byte) (C.GO_EVP_CIPHER_CTX_PTR, error) { ++ ctx := C.go_openssl_EVP_CIPHER_CTX_new() ++ if ctx == nil { ++ return nil, fail("unable to create EVP cipher ctx") ++ } ++ if C.go_openssl_EVP_CipherInit_ex(ctx, cipher, nil, base(key), base(iv), mode) != 1 { ++ C.go_openssl_EVP_CIPHER_CTX_free(ctx) ++ return nil, fail("unable to initialize EVP cipher ctx") ++ } ++ return ctx, nil ++} ++ ++func bigUint64(b []byte) uint64 { ++ _ = b[7] // bounds check hint to compiler; see go.dev/issue/14808 ++ return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | ++ uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 ++} +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/ecdsa.go b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/ecdsa.go +new file mode 100644 +index 00000000000000..84f82e903ce185 +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/ecdsa.go +@@ -0,0 +1,214 @@ ++// Copyright (c) Microsoft Corporation. ++// Licensed under the MIT License. ++ ++//go:build linux && !android ++// +build linux,!android ++ ++package openssl ++ ++// #include "goopenssl.h" ++import "C" ++import ( ++ "encoding/asn1" ++ "errors" ++ "math/big" ++ "runtime" ++ "unsafe" ++) ++ ++type ecdsaSignature struct { ++ R, S *big.Int ++} ++ ++type PrivateKeyECDSA struct { ++ // _pkey MUST NOT be accessed directly. Instead, use the withKey method. ++ _pkey C.GO_EVP_PKEY_PTR ++} ++ ++func (k *PrivateKeyECDSA) finalize() { ++ C.go_openssl_EVP_PKEY_free(k._pkey) ++} ++ ++func (k *PrivateKeyECDSA) withKey(f func(C.GO_EVP_PKEY_PTR) C.int) C.int { ++ defer runtime.KeepAlive(k) ++ return f(k._pkey) ++} ++ ++type PublicKeyECDSA struct { ++ // _pkey MUST NOT be accessed directly. Instead, use the withKey method. ++ _pkey C.GO_EVP_PKEY_PTR ++} ++ ++func (k *PublicKeyECDSA) finalize() { ++ C.go_openssl_EVP_PKEY_free(k._pkey) ++} ++ ++func (k *PublicKeyECDSA) withKey(f func(C.GO_EVP_PKEY_PTR) C.int) C.int { ++ defer runtime.KeepAlive(k) ++ return f(k._pkey) ++} ++ ++var errUnknownCurve = errors.New("openssl: unknown elliptic curve") ++var errUnsupportedCurve = errors.New("openssl: unsupported elliptic curve") ++ ++func curveNID(curve string) (C.int, error) { ++ switch curve { ++ case "P-224": ++ return C.GO_NID_secp224r1, nil ++ case "P-256": ++ return C.GO_NID_X9_62_prime256v1, nil ++ case "P-384": ++ return C.GO_NID_secp384r1, nil ++ case "P-521": ++ return C.GO_NID_secp521r1, nil ++ } ++ return 0, errUnknownCurve ++} ++ ++func NewPublicKeyECDSA(curve string, X, Y *big.Int) (*PublicKeyECDSA, error) { ++ pkey, err := newECKey(curve, X, Y, nil) ++ if err != nil { ++ return nil, err ++ } ++ k := &PublicKeyECDSA{_pkey: pkey} ++ // Note: Because of the finalizer, any time k.key is passed to cgo, ++ // that call must be followed by a call to runtime.KeepAlive(k), ++ // to make sure k is not collected (and finalized) before the cgo ++ // call returns. ++ runtime.SetFinalizer(k, (*PublicKeyECDSA).finalize) ++ return k, nil ++} ++ ++func newECKey(curve string, X, Y, D *big.Int) (pkey C.GO_EVP_PKEY_PTR, err error) { ++ var nid C.int ++ if nid, err = curveNID(curve); err != nil { ++ return nil, err ++ } ++ var bx, by C.GO_BIGNUM_PTR ++ var key C.GO_EC_KEY_PTR ++ defer func() { ++ if bx != nil { ++ C.go_openssl_BN_free(bx) ++ } ++ if by != nil { ++ C.go_openssl_BN_free(by) ++ } ++ if err != nil { ++ if key != nil { ++ C.go_openssl_EC_KEY_free(key) ++ } ++ if pkey != nil { ++ C.go_openssl_EVP_PKEY_free(pkey) ++ // pkey is a named return, so in case of error ++ // it have to be cleared before returing. ++ pkey = nil ++ } ++ } ++ }() ++ bx = bigToBN(X) ++ by = bigToBN(Y) ++ if bx == nil || by == nil { ++ return nil, newOpenSSLError("BN_bin2bn failed") ++ } ++ if key = C.go_openssl_EC_KEY_new_by_curve_name(nid); key == nil { ++ return nil, newOpenSSLError("EC_KEY_new_by_curve_name failed") ++ } ++ if C.go_openssl_EC_KEY_set_public_key_affine_coordinates(key, bx, by) != 1 { ++ return nil, newOpenSSLError("EC_KEY_set_public_key_affine_coordinates failed") ++ } ++ if D != nil { ++ bd := bigToBN(D) ++ if bd == nil { ++ return nil, newOpenSSLError("BN_bin2bn failed") ++ } ++ defer C.go_openssl_BN_free(bd) ++ if C.go_openssl_EC_KEY_set_private_key(key, bd) != 1 { ++ return nil, newOpenSSLError("EC_KEY_set_private_key failed") ++ } ++ } ++ if pkey = C.go_openssl_EVP_PKEY_new(); pkey == nil { ++ return nil, newOpenSSLError("EVP_PKEY_new failed") ++ } ++ if C.go_openssl_EVP_PKEY_assign(pkey, C.GO_EVP_PKEY_EC, (unsafe.Pointer)(key)) != 1 { ++ return nil, newOpenSSLError("EVP_PKEY_assign failed") ++ } ++ return pkey, nil ++} ++ ++func NewPrivateKeyECDSA(curve string, X, Y *big.Int, D *big.Int) (*PrivateKeyECDSA, error) { ++ pkey, err := newECKey(curve, X, Y, D) ++ if err != nil { ++ return nil, err ++ } ++ k := &PrivateKeyECDSA{_pkey: pkey} ++ // Note: Because of the finalizer, any time k.key is passed to cgo, ++ // that call must be followed by a call to runtime.KeepAlive(k), ++ // to make sure k is not collected (and finalized) before the cgo ++ // call returns. ++ runtime.SetFinalizer(k, (*PrivateKeyECDSA).finalize) ++ return k, nil ++} ++ ++func SignECDSA(priv *PrivateKeyECDSA, hash []byte) (r, s *big.Int, err error) { ++ // We could use ECDSA_do_sign instead but would need to convert ++ // the resulting BIGNUMs to *big.Int form. If we're going to do a ++ // conversion, converting the ASN.1 form is more convenient and ++ // likely not much more expensive. ++ sig, err := SignMarshalECDSA(priv, hash) ++ if err != nil { ++ return nil, nil, err ++ } ++ var esig ecdsaSignature ++ if _, err := asn1.Unmarshal(sig, &esig); err != nil { ++ return nil, nil, err ++ } ++ return esig.R, esig.S, nil ++} ++ ++func SignMarshalECDSA(priv *PrivateKeyECDSA, hash []byte) ([]byte, error) { ++ return evpSign(priv.withKey, 0, 0, 0, hash) ++} ++ ++func VerifyECDSA(pub *PublicKeyECDSA, hash []byte, r, s *big.Int) bool { ++ // We could use ECDSA_do_verify instead but would need to convert ++ // r and s to BIGNUM form. If we're going to do a conversion, marshaling ++ // to ASN.1 is more convenient and likely not much more expensive. ++ sig, err := asn1.Marshal(ecdsaSignature{r, s}) ++ if err != nil { ++ return false ++ } ++ return evpVerify(pub.withKey, 0, 0, 0, sig, hash) == nil ++} ++ ++func GenerateKeyECDSA(curve string) (X, Y, D *big.Int, err error) { ++ pkey, err := generateEVPPKey(C.GO_EVP_PKEY_EC, 0, curve) ++ if err != nil { ++ return nil, nil, nil, err ++ } ++ defer C.go_openssl_EVP_PKEY_free(pkey) ++ key := C.go_openssl_EVP_PKEY_get1_EC_KEY(pkey) ++ if key == nil { ++ return nil, nil, nil, newOpenSSLError("EVP_PKEY_get1_EC_KEY failed") ++ } ++ defer C.go_openssl_EC_KEY_free(key) ++ group := C.go_openssl_EC_KEY_get0_group(key) ++ pt := C.go_openssl_EC_KEY_get0_public_key(key) ++ bd := C.go_openssl_EC_KEY_get0_private_key(key) ++ if pt == nil || bd == nil { ++ return nil, nil, nil, newOpenSSLError("EC_KEY_get0_private_key failed") ++ } ++ bx := C.go_openssl_BN_new() ++ if bx == nil { ++ return nil, nil, nil, newOpenSSLError("BN_new failed") ++ } ++ defer C.go_openssl_BN_free(bx) ++ by := C.go_openssl_BN_new() ++ if by == nil { ++ return nil, nil, nil, newOpenSSLError("BN_new failed") ++ } ++ defer C.go_openssl_BN_free(by) ++ if C.go_openssl_EC_POINT_get_affine_coordinates_GFp(group, pt, bx, by, nil) == 0 { ++ return nil, nil, nil, newOpenSSLError("EC_POINT_get_affine_coordinates_GFp failed") ++ } ++ return bnToBig(bx), bnToBig(by), bnToBig(bd), nil ++} +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/evpkey.go b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/evpkey.go +new file mode 100644 +index 00000000000000..03a652aef13931 +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/evpkey.go +@@ -0,0 +1,279 @@ ++// Copyright (c) Microsoft Corporation. ++// Licensed under the MIT License. ++ ++//go:build linux && !android ++// +build linux,!android ++ ++package openssl ++ ++// #include "goopenssl.h" ++import "C" ++import ( ++ "crypto" ++ "errors" ++ "hash" ++ "unsafe" ++) ++ ++// hashToMD converts a hash.Hash implementation from this package to a GO_EVP_MD_PTR. ++func hashToMD(h hash.Hash) C.GO_EVP_MD_PTR { ++ switch h.(type) { ++ case *sha1Hash: ++ return C.go_openssl_EVP_sha1() ++ case *sha224Hash: ++ return C.go_openssl_EVP_sha224() ++ case *sha256Hash: ++ return C.go_openssl_EVP_sha256() ++ case *sha384Hash: ++ return C.go_openssl_EVP_sha384() ++ case *sha512Hash: ++ return C.go_openssl_EVP_sha512() ++ } ++ return nil ++} ++ ++// cryptoHashToMD converts a crypto.Hash to a GO_EVP_MD_PTR. ++func cryptoHashToMD(ch crypto.Hash) C.GO_EVP_MD_PTR { ++ switch ch { ++ case crypto.MD5: ++ return C.go_openssl_EVP_md5() ++ case crypto.MD5SHA1: ++ if vMajor == 1 && vMinor == 0 { ++ // MD5SHA1 is not implemented in OpenSSL 1.0.2. ++ // It is implemented in higher versions but without FIPS support. ++ // It is considered a deprecated digest, not approved by FIPS 140-2 ++ // and only used in pre-TLS 1.2, so we would rather not support it ++ // if using 1.0.2 than than implement something that is not properly validated. ++ return nil ++ } ++ return C.go_openssl_EVP_md5_sha1() ++ case crypto.SHA1: ++ return C.go_openssl_EVP_sha1() ++ case crypto.SHA224: ++ return C.go_openssl_EVP_sha224() ++ case crypto.SHA256: ++ return C.go_openssl_EVP_sha256() ++ case crypto.SHA384: ++ return C.go_openssl_EVP_sha384() ++ case crypto.SHA512: ++ return C.go_openssl_EVP_sha512() ++ } ++ return nil ++} ++ ++func generateEVPPKey(id C.int, bits int, curve string) (C.GO_EVP_PKEY_PTR, error) { ++ if (bits == 0 && curve == "") || (bits != 0 && curve != "") { ++ return nil, fail("incorrect generateEVPPKey parameters") ++ } ++ ctx := C.go_openssl_EVP_PKEY_CTX_new_id(id, nil) ++ if ctx == nil { ++ return nil, newOpenSSLError("EVP_PKEY_CTX_new_id failed") ++ } ++ defer C.go_openssl_EVP_PKEY_CTX_free(ctx) ++ if C.go_openssl_EVP_PKEY_keygen_init(ctx) != 1 { ++ return nil, newOpenSSLError("EVP_PKEY_keygen_init failed") ++ } ++ if bits != 0 { ++ if C.go_openssl_EVP_PKEY_CTX_ctrl(ctx, id, -1, C.GO_EVP_PKEY_CTRL_RSA_KEYGEN_BITS, C.int(bits), nil) != 1 { ++ return nil, newOpenSSLError("EVP_PKEY_CTX_ctrl failed") ++ } ++ } ++ if curve != "" { ++ nid, err := curveNID(curve) ++ if err != nil { ++ return nil, err ++ } ++ if C.go_openssl_EVP_PKEY_CTX_ctrl(ctx, id, -1, C.GO_EVP_PKEY_CTRL_EC_PARAMGEN_CURVE_NID, nid, nil) != 1 { ++ return nil, newOpenSSLError("EVP_PKEY_CTX_ctrl failed") ++ } ++ } ++ var pkey C.GO_EVP_PKEY_PTR ++ if C.go_openssl_EVP_PKEY_keygen(ctx, &pkey) != 1 { ++ return nil, newOpenSSLError("EVP_PKEY_keygen failed") ++ } ++ return pkey, nil ++} ++ ++type withKeyFunc func(func(C.GO_EVP_PKEY_PTR) C.int) C.int ++type initFunc func(C.GO_EVP_PKEY_CTX_PTR) C.int ++type cryptFunc func(C.GO_EVP_PKEY_CTX_PTR, *C.uchar, *C.size_t, *C.uchar, C.size_t) C.int ++type verifyFunc func(C.GO_EVP_PKEY_CTX_PTR, *C.uchar, C.size_t, *C.uchar, C.size_t) C.int ++ ++func setupEVP(withKey withKeyFunc, padding C.int, ++ h hash.Hash, label []byte, saltLen int, ch crypto.Hash, ++ init initFunc) (ctx C.GO_EVP_PKEY_CTX_PTR, err error) { ++ defer func() { ++ if err != nil { ++ if ctx != nil { ++ C.go_openssl_EVP_PKEY_CTX_free(ctx) ++ ctx = nil ++ } ++ } ++ }() ++ ++ withKey(func(pkey C.GO_EVP_PKEY_PTR) C.int { ++ ctx = C.go_openssl_EVP_PKEY_CTX_new(pkey, nil) ++ return 1 ++ }) ++ if ctx == nil { ++ return nil, newOpenSSLError("EVP_PKEY_CTX_new failed") ++ } ++ if init(ctx) != 1 { ++ return nil, newOpenSSLError("EVP_PKEY_operation_init failed") ++ } ++ if padding == 0 { ++ return ctx, nil ++ } ++ // Each padding type has its own requirements in terms of when to apply the padding, ++ // so it can't be just set at this point. ++ setPadding := func() error { ++ if C.go_openssl_EVP_PKEY_CTX_ctrl(ctx, C.GO_EVP_PKEY_RSA, -1, C.GO_EVP_PKEY_CTRL_RSA_PADDING, padding, nil) != 1 { ++ return newOpenSSLError("EVP_PKEY_CTX_ctrl failed") ++ } ++ return nil ++ } ++ switch padding { ++ case C.GO_RSA_PKCS1_OAEP_PADDING: ++ md := hashToMD(h) ++ if md == nil { ++ return nil, errors.New("crypto/rsa: unsupported hash function") ++ } ++ // setPadding must happen before setting EVP_PKEY_CTRL_RSA_OAEP_MD. ++ if err := setPadding(); err != nil { ++ return nil, err ++ } ++ if C.go_openssl_EVP_PKEY_CTX_ctrl(ctx, C.GO_EVP_PKEY_RSA, -1, C.GO_EVP_PKEY_CTRL_RSA_OAEP_MD, 0, unsafe.Pointer(md)) != 1 { ++ return nil, newOpenSSLError("EVP_PKEY_CTX_ctrl failed") ++ } ++ // ctx takes ownership of label, so malloc a copy for OpenSSL to free. ++ // OpenSSL 1.1.1 and higher does not take ownership of the label if the length is zero, ++ // so better avoid the allocation. ++ var clabel *C.uchar ++ if len(label) > 0 { ++ // Go guarantees C.malloc never returns nil. ++ clabel = (*C.uchar)(C.malloc(C.size_t(len(label)))) ++ copy((*[1 << 30]byte)(unsafe.Pointer(clabel))[:len(label)], label) ++ } ++ if C.go_openssl_EVP_PKEY_CTX_ctrl(ctx, C.GO_EVP_PKEY_RSA, -1, C.GO_EVP_PKEY_CTRL_RSA_OAEP_LABEL, C.int(len(label)), unsafe.Pointer(clabel)) != 1 { ++ if clabel != nil { ++ C.free(unsafe.Pointer(clabel)) ++ } ++ return nil, newOpenSSLError("EVP_PKEY_CTX_ctrl failed") ++ } ++ case C.GO_RSA_PKCS1_PSS_PADDING: ++ md := cryptoHashToMD(ch) ++ if md == nil { ++ return nil, errors.New("crypto/rsa: unsupported hash function") ++ } ++ if C.go_openssl_EVP_PKEY_CTX_ctrl(ctx, C.GO_EVP_PKEY_RSA, -1, C.GO_EVP_PKEY_CTRL_MD, 0, unsafe.Pointer(md)) != 1 { ++ return nil, newOpenSSLError("EVP_PKEY_CTX_ctrl failed") ++ } ++ // setPadding must happen after setting EVP_PKEY_CTRL_MD. ++ if err := setPadding(); err != nil { ++ return nil, err ++ } ++ if saltLen != 0 { ++ if C.go_openssl_EVP_PKEY_CTX_ctrl(ctx, C.GO_EVP_PKEY_RSA, -1, C.GO_EVP_PKEY_CTRL_RSA_PSS_SALTLEN, C.int(saltLen), nil) != 1 { ++ return nil, newOpenSSLError("EVP_PKEY_CTX_ctrl failed") ++ } ++ } ++ ++ case C.GO_RSA_PKCS1_PADDING: ++ if ch != 0 { ++ // We support unhashed messages. ++ md := cryptoHashToMD(ch) ++ if md == nil { ++ return nil, errors.New("crypto/rsa: unsupported hash function") ++ } ++ if C.go_openssl_EVP_PKEY_CTX_ctrl(ctx, -1, -1, C.GO_EVP_PKEY_CTRL_MD, 0, unsafe.Pointer(md)) != 1 { ++ return nil, newOpenSSLError("EVP_PKEY_CTX_ctrl failed") ++ } ++ if err := setPadding(); err != nil { ++ return nil, err ++ } ++ } ++ default: ++ if err := setPadding(); err != nil { ++ return nil, err ++ } ++ } ++ return ctx, nil ++} ++ ++func cryptEVP(withKey withKeyFunc, padding C.int, ++ h hash.Hash, label []byte, saltLen int, ch crypto.Hash, ++ init initFunc, crypt cryptFunc, in []byte) ([]byte, error) { ++ ++ ctx, err := setupEVP(withKey, padding, h, label, saltLen, ch, init) ++ if err != nil { ++ return nil, err ++ } ++ defer C.go_openssl_EVP_PKEY_CTX_free(ctx) ++ pkeySize := withKey(func(pkey C.GO_EVP_PKEY_PTR) C.int { ++ return C.go_openssl_EVP_PKEY_get_size(pkey) ++ }) ++ outLen := C.size_t(pkeySize) ++ out := make([]byte, pkeySize) ++ if crypt(ctx, base(out), &outLen, base(in), C.size_t(len(in))) != 1 { ++ return nil, newOpenSSLError("EVP_PKEY_decrypt/encrypt failed") ++ } ++ // The size returned by EVP_PKEY_get_size() is only preliminary and not exact, ++ // so the final contents of the out buffer may be smaller. ++ return out[:outLen], nil ++} ++ ++func verifyEVP(withKey withKeyFunc, padding C.int, ++ h hash.Hash, label []byte, saltLen int, ch crypto.Hash, ++ init initFunc, verify verifyFunc, ++ sig, in []byte) error { ++ ++ ctx, err := setupEVP(withKey, padding, h, label, saltLen, ch, init) ++ if err != nil { ++ return err ++ } ++ defer C.go_openssl_EVP_PKEY_CTX_free(ctx) ++ if verify(ctx, base(sig), C.size_t(len(sig)), base(in), C.size_t(len(in))) != 1 { ++ return newOpenSSLError("EVP_PKEY_decrypt/encrypt failed") ++ } ++ return nil ++} ++ ++func evpEncrypt(withKey withKeyFunc, padding C.int, h hash.Hash, label, msg []byte) ([]byte, error) { ++ encryptInit := func(ctx C.GO_EVP_PKEY_CTX_PTR) C.int { ++ return C.go_openssl_EVP_PKEY_encrypt_init(ctx) ++ } ++ encrypt := func(ctx C.GO_EVP_PKEY_CTX_PTR, out *C.uchar, outLen *C.size_t, in *C.uchar, inLen C.size_t) C.int { ++ return C.go_openssl_EVP_PKEY_encrypt(ctx, out, outLen, in, inLen) ++ } ++ return cryptEVP(withKey, padding, h, label, 0, 0, encryptInit, encrypt, msg) ++} ++ ++func evpDecrypt(withKey withKeyFunc, padding C.int, h hash.Hash, label, msg []byte) ([]byte, error) { ++ decryptInit := func(ctx C.GO_EVP_PKEY_CTX_PTR) C.int { ++ return C.go_openssl_EVP_PKEY_decrypt_init(ctx) ++ } ++ decrypt := func(ctx C.GO_EVP_PKEY_CTX_PTR, out *C.uchar, outLen *C.size_t, in *C.uchar, inLen C.size_t) C.int { ++ return C.go_openssl_EVP_PKEY_decrypt(ctx, out, outLen, in, inLen) ++ } ++ return cryptEVP(withKey, padding, h, label, 0, 0, decryptInit, decrypt, msg) ++} ++ ++func evpSign(withKey withKeyFunc, padding C.int, saltLen int, h crypto.Hash, hashed []byte) ([]byte, error) { ++ signtInit := func(ctx C.GO_EVP_PKEY_CTX_PTR) C.int { ++ return C.go_openssl_EVP_PKEY_sign_init(ctx) ++ } ++ sign := func(ctx C.GO_EVP_PKEY_CTX_PTR, out *C.uchar, outLen *C.size_t, in *C.uchar, inLen C.size_t) C.int { ++ return C.go_openssl_EVP_PKEY_sign(ctx, out, outLen, in, inLen) ++ } ++ return cryptEVP(withKey, padding, nil, nil, saltLen, h, signtInit, sign, hashed) ++} ++ ++func evpVerify(withKey withKeyFunc, padding C.int, saltLen int, h crypto.Hash, sig, hashed []byte) error { ++ verifyInit := func(ctx C.GO_EVP_PKEY_CTX_PTR) C.int { ++ return C.go_openssl_EVP_PKEY_verify_init(ctx) ++ } ++ verify := func(ctx C.GO_EVP_PKEY_CTX_PTR, out *C.uchar, outLen C.size_t, in *C.uchar, inLen C.size_t) C.int { ++ return C.go_openssl_EVP_PKEY_verify(ctx, out, outLen, in, inLen) ++ } ++ return verifyEVP(withKey, padding, nil, nil, saltLen, h, verifyInit, verify, sig, hashed) ++} +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/goopenssl.c b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/goopenssl.c +new file mode 100644 +index 00000000000000..7bbf74185128dd +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/goopenssl.c +@@ -0,0 +1,153 @@ ++// Copyright (c) Microsoft Corporation. ++// Licensed under the MIT License. ++ ++//go:build linux && !android ++// +build linux,!android ++ ++#include "goopenssl.h" ++ ++#include ++#include ++ ++static unsigned long ++version_num(void* handle) ++{ ++ unsigned long (*fn)(void); ++ // OPENSSL_version_num is defined in OpenSSL 1.1.0 and 1.1.1. ++ fn = (unsigned long (*)(void))dlsym(handle, "OpenSSL_version_num"); ++ if (fn != NULL) ++ return fn(); ++ ++ // SSLeay is defined in OpenSSL 1.0.2. ++ fn = (unsigned long (*)(void))dlsym(handle, "SSLeay"); ++ if (fn != NULL) ++ return fn(); ++ ++ return 0; ++} ++ ++int ++go_openssl_version_major(void* handle) ++{ ++ unsigned int (*fn)(void); ++ // OPENSSL_version_major is supported since OpenSSL 3. ++ fn = (unsigned int (*)(void))dlsym(handle, "OPENSSL_version_major"); ++ if (fn != NULL) ++ return (int)fn(); ++ ++ // If OPENSSL_version_major is not defined, try with OpenSSL 1 functions. ++ unsigned long num = version_num(handle); ++ if (num < 0x10000000L || num >= 0x20000000L) ++ return -1; ++ ++ return 1; ++} ++ ++int ++go_openssl_version_minor(void* handle) ++{ ++ unsigned int (*fn)(void); ++ // OPENSSL_version_major is supported since OpenSSL 3. ++ fn = (unsigned int (*)(void))dlsym(handle, "OPENSSL_version_minor"); ++ if (fn != NULL) ++ return (int)fn(); ++ ++ // If OPENSSL_version_major is not defined, try with OpenSSL 1 functions. ++ unsigned long num = version_num(handle); ++ // OpenSSL version number follows this schema: ++ // MNNFFPPS: major minor fix patch status. ++ if (num < 0x10000000L || num >= 0x10200000L) ++ { ++ // We only support minor version 0 and 1, ++ // so there is no need to implement an algorithm ++ // that decodes the version number into individual components. ++ return -1; ++ } ++ ++ if (num >= 0x10100000L) ++ return 1; ++ ++ return 0; ++} ++ ++// Approach taken from .Net System.Security.Cryptography.Native ++// https://github.com/dotnet/runtime/blob/f64246ce08fb7a58221b2b7c8e68f69c02522b0d/src/libraries/Native/Unix/System.Security.Cryptography.Native/opensslshim.c ++ ++#define DEFINEFUNC(ret, func, args, argscall) ret (*_g_##func)args; ++#define DEFINEFUNC_LEGACY_1_0(ret, func, args, argscall) DEFINEFUNC(ret, func, args, argscall) ++#define DEFINEFUNC_LEGACY_1(ret, func, args, argscall) DEFINEFUNC(ret, func, args, argscall) ++#define DEFINEFUNC_1_1(ret, func, args, argscall) DEFINEFUNC(ret, func, args, argscall) ++#define DEFINEFUNC_3_0(ret, func, args, argscall) DEFINEFUNC(ret, func, args, argscall) ++#define DEFINEFUNC_RENAMED_1_1(ret, func, oldfunc, args, argscall) DEFINEFUNC(ret, func, args, argscall) ++#define DEFINEFUNC_RENAMED_3_0(ret, func, oldfunc, args, argscall) DEFINEFUNC(ret, func, args, argscall) ++ ++FOR_ALL_OPENSSL_FUNCTIONS ++ ++#undef DEFINEFUNC ++#undef DEFINEFUNC_LEGACY_1_0 ++#undef DEFINEFUNC_LEGACY_1 ++#undef DEFINEFUNC_1_1 ++#undef DEFINEFUNC_3_0 ++#undef DEFINEFUNC_RENAMED_1_1 ++#undef DEFINEFUNC_RENAMED_3_0 ++ ++// Load all the functions stored in FOR_ALL_OPENSSL_FUNCTIONS ++// and assign them to their corresponding function pointer ++// defined in goopenssl.h. ++void ++go_openssl_load_functions(void* handle, int major, int minor) ++{ ++#define DEFINEFUNC_INTERNAL(name, func) \ ++ _g_##name = dlsym(handle, func); \ ++ if (_g_##name == NULL) { fprintf(stderr, "Cannot get required symbol " #func " from libcrypto version %d.%d\n", major, minor); abort(); } ++#define DEFINEFUNC(ret, func, args, argscall) \ ++ DEFINEFUNC_INTERNAL(func, #func) ++#define DEFINEFUNC_LEGACY_1_0(ret, func, args, argscall) \ ++ if (major == 1 && minor == 0) \ ++ { \ ++ DEFINEFUNC_INTERNAL(func, #func) \ ++ } ++#define DEFINEFUNC_LEGACY_1(ret, func, args, argscall) \ ++ if (major == 1) \ ++ { \ ++ DEFINEFUNC_INTERNAL(func, #func) \ ++ } ++#define DEFINEFUNC_1_1(ret, func, args, argscall) \ ++ if (major == 3 || (major == 1 && minor == 1)) \ ++ { \ ++ DEFINEFUNC_INTERNAL(func, #func) \ ++ } ++#define DEFINEFUNC_3_0(ret, func, args, argscall) \ ++ if (major == 3) \ ++ { \ ++ DEFINEFUNC_INTERNAL(func, #func) \ ++ } ++#define DEFINEFUNC_RENAMED_1_1(ret, func, oldfunc, args, argscall) \ ++ if (major == 1 && minor == 0) \ ++ { \ ++ DEFINEFUNC_INTERNAL(func, #oldfunc) \ ++ } \ ++ else \ ++ { \ ++ DEFINEFUNC_INTERNAL(func, #func) \ ++ } ++#define DEFINEFUNC_RENAMED_3_0(ret, func, oldfunc, args, argscall) \ ++ if (major == 1) \ ++ { \ ++ DEFINEFUNC_INTERNAL(func, #oldfunc) \ ++ } \ ++ else \ ++ { \ ++ DEFINEFUNC_INTERNAL(func, #func) \ ++ } ++ ++FOR_ALL_OPENSSL_FUNCTIONS ++ ++#undef DEFINEFUNC ++#undef DEFINEFUNC_LEGACY_1_0 ++#undef DEFINEFUNC_LEGACY_1 ++#undef DEFINEFUNC_1_1 ++#undef DEFINEFUNC_3_0 ++#undef DEFINEFUNC_RENAMED_1_1 ++#undef DEFINEFUNC_RENAMED_3_0 ++} +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/goopenssl.h b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/goopenssl.h +new file mode 100644 +index 00000000000000..9191b512e3192d +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/goopenssl.h +@@ -0,0 +1,133 @@ ++// Copyright (c) Microsoft Corporation. ++// Licensed under the MIT License. ++ ++// This header file describes the OpenSSL ABI as built for use in Go. ++ ++#include // size_t ++ ++#include "openssl_funcs.h" ++ ++int go_openssl_version_major(void* handle); ++int go_openssl_version_minor(void* handle); ++int go_openssl_thread_setup(void); ++void go_openssl_load_functions(void* handle, int major, int minor); ++ ++// Define pointers to all the used OpenSSL functions. ++// Calling C function pointers from Go is currently not supported. ++// It is possible to circumvent this by using a C function wrapper. ++// https://pkg.go.dev/cmd/cgo ++#define DEFINEFUNC(ret, func, args, argscall) \ ++ extern ret (*_g_##func)args; \ ++ static inline ret go_openssl_##func args \ ++ { \ ++ return _g_##func argscall; \ ++ } ++#define DEFINEFUNC_LEGACY_1_0(ret, func, args, argscall) \ ++ DEFINEFUNC(ret, func, args, argscall) ++#define DEFINEFUNC_LEGACY_1(ret, func, args, argscall) \ ++ DEFINEFUNC(ret, func, args, argscall) ++#define DEFINEFUNC_1_1(ret, func, args, argscall) \ ++ DEFINEFUNC(ret, func, args, argscall) ++#define DEFINEFUNC_3_0(ret, func, args, argscall) \ ++ DEFINEFUNC(ret, func, args, argscall) ++#define DEFINEFUNC_RENAMED_1_1(ret, func, oldfunc, args, argscall) \ ++ DEFINEFUNC(ret, func, args, argscall) ++#define DEFINEFUNC_RENAMED_3_0(ret, func, oldfunc, args, argscall) \ ++ DEFINEFUNC(ret, func, args, argscall) ++ ++FOR_ALL_OPENSSL_FUNCTIONS ++ ++#undef DEFINEFUNC ++#undef DEFINEFUNC_LEGACY_1_0 ++#undef DEFINEFUNC_LEGACY_1 ++#undef DEFINEFUNC_1_1 ++#undef DEFINEFUNC_3_0 ++#undef DEFINEFUNC_RENAMED_1_1 ++#undef DEFINEFUNC_RENAMED_3_0 ++ ++// These wrappers allocate out_len on the C stack to avoid having to pass a pointer from Go, which would escape to the heap. ++// Use them only in situations where the output length can be safely discarded. ++static inline int ++go_openssl_EVP_EncryptUpdate_wrapper(GO_EVP_CIPHER_CTX_PTR ctx, unsigned char *out, const unsigned char *in, int in_len) ++{ ++ int len; ++ return go_openssl_EVP_EncryptUpdate(ctx, out, &len, in, in_len); ++} ++ ++static inline int ++go_openssl_EVP_DecryptUpdate_wrapper(GO_EVP_CIPHER_CTX_PTR ctx, unsigned char *out, const unsigned char *in, int in_len) ++{ ++ int len; ++ return go_openssl_EVP_DecryptUpdate(ctx, out, &len, in, in_len); ++} ++ ++static inline int ++go_openssl_EVP_CipherUpdate_wrapper(GO_EVP_CIPHER_CTX_PTR ctx, unsigned char *out, const unsigned char *in, int in_len) ++{ ++ int len; ++ return go_openssl_EVP_CipherUpdate(ctx, out, &len, in, in_len); ++} ++ ++ ++// These wrappers allocate out_len on the C stack, and check that it matches the expected ++// value, to avoid having to pass a pointer from Go, which would escape to the heap. ++ ++static inline int ++go_openssl_EVP_CIPHER_CTX_seal_wrapper(const GO_EVP_CIPHER_CTX_PTR ctx, ++ unsigned char *out, ++ const unsigned char *nonce, ++ const unsigned char *in, int in_len, ++ const unsigned char *aad, int aad_len) ++{ ++ if (in_len == 0) in = ""; ++ if (aad_len == 0) aad = ""; ++ ++ if (go_openssl_EVP_CipherInit_ex(ctx, NULL, NULL, NULL, nonce, GO_AES_ENCRYPT) != 1) ++ return 0; ++ ++ int discard_len, out_len; ++ if (go_openssl_EVP_EncryptUpdate(ctx, NULL, &discard_len, aad, aad_len) != 1 ++ || go_openssl_EVP_EncryptUpdate(ctx, out, &out_len, in, in_len) != 1 ++ || go_openssl_EVP_EncryptFinal_ex(ctx, out + out_len, &discard_len) != 1) ++ { ++ return 0; ++ } ++ ++ if (in_len != out_len) ++ return 0; ++ ++ return go_openssl_EVP_CIPHER_CTX_ctrl(ctx, GO_EVP_CTRL_GCM_GET_TAG, 16, out + out_len); ++}; ++ ++static inline int ++go_openssl_EVP_CIPHER_CTX_open_wrapper(const GO_EVP_CIPHER_CTX_PTR ctx, ++ unsigned char *out, ++ const unsigned char *nonce, ++ const unsigned char *in, int in_len, ++ const unsigned char *aad, int aad_len, ++ const unsigned char *tag) ++{ ++ if (in_len == 0) in = ""; ++ if (aad_len == 0) aad = ""; ++ ++ if (go_openssl_EVP_CipherInit_ex(ctx, NULL, NULL, NULL, nonce, GO_AES_DECRYPT) != 1) ++ return 0; ++ ++ int discard_len, out_len; ++ if (go_openssl_EVP_DecryptUpdate(ctx, NULL, &discard_len, aad, aad_len) != 1 ++ || go_openssl_EVP_DecryptUpdate(ctx, out, &out_len, in, in_len) != 1) ++ { ++ return 0; ++ } ++ ++ if (go_openssl_EVP_CIPHER_CTX_ctrl(ctx, GO_EVP_CTRL_GCM_SET_TAG, 16, (unsigned char *)(tag)) != 1) ++ return 0; ++ ++ if (go_openssl_EVP_DecryptFinal_ex(ctx, out + out_len, &discard_len) != 1) ++ return 0; ++ ++ if (out_len != in_len) ++ return 0; ++ ++ return 1; ++}; +\ No newline at end of file +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/hmac.go b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/hmac.go +new file mode 100644 +index 00000000000000..80f320b041f7e0 +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/hmac.go +@@ -0,0 +1,148 @@ ++// Copyright (c) Microsoft Corporation. ++// Licensed under the MIT License. ++ ++//go:build linux && !android ++// +build linux,!android ++ ++package openssl ++ ++// #include "goopenssl.h" ++import "C" ++import ( ++ "hash" ++ "runtime" ++ "unsafe" ++) ++ ++// NewHMAC returns a new HMAC using OpenSSL. ++// The function h must return a hash implemented by ++// OpenSSL (for example, h could be openssl.NewSHA256). ++// If h is not recognized, NewHMAC returns nil. ++func NewHMAC(h func() hash.Hash, key []byte) hash.Hash { ++ ch := h() ++ md := hashToMD(ch) ++ if md == nil { ++ return nil ++ } ++ ++ var hkey []byte ++ if len(key) > 0 { ++ // Note: Could hash down long keys here using EVP_Digest. ++ hkey = make([]byte, len(key)) ++ copy(hkey, key) ++ } else { ++ // This is supported in OpenSSL/Standard lib and as such ++ // we must support it here. When using HMAC with a null key ++ // HMAC_Init will try and reuse the key from the ctx. This is ++ // not the bahavior previously implemented, so as a workaround ++ // we pass an "empty" key. ++ hkey = make([]byte, C.GO_EVP_MAX_MD_SIZE) ++ } ++ hmac := &opensslHMAC{ ++ md: md, ++ size: ch.Size(), ++ blockSize: ch.BlockSize(), ++ key: hkey, ++ ctx: hmacCtxNew(), ++ } ++ runtime.SetFinalizer(hmac, (*opensslHMAC).finalize) ++ hmac.Reset() ++ return hmac ++} ++ ++type opensslHMAC struct { ++ md C.GO_EVP_MD_PTR ++ ctx C.GO_HMAC_CTX_PTR ++ size int ++ blockSize int ++ key []byte ++ sum []byte ++} ++ ++func (h *opensslHMAC) Reset() { ++ hmacCtxReset(h.ctx) ++ ++ if C.go_openssl_HMAC_Init_ex(h.ctx, unsafe.Pointer(&h.key[0]), C.int(len(h.key)), h.md, nil) == 0 { ++ panic("openssl: HMAC_Init failed") ++ } ++ if size := C.go_openssl_EVP_MD_get_size(h.md); size != C.int(h.size) { ++ println("openssl: HMAC size:", size, "!=", h.size) ++ panic("openssl: HMAC size mismatch") ++ } ++ runtime.KeepAlive(h) // Next line will keep h alive too; just making doubly sure. ++ h.sum = nil ++} ++ ++func (h *opensslHMAC) finalize() { ++ hmacCtxFree(h.ctx) ++} ++ ++func (h *opensslHMAC) Write(p []byte) (int, error) { ++ if len(p) > 0 { ++ C.go_openssl_HMAC_Update(h.ctx, base(p), C.size_t(len(p))) ++ } ++ runtime.KeepAlive(h) ++ return len(p), nil ++} ++ ++func (h *opensslHMAC) Size() int { ++ return h.size ++} ++ ++func (h *opensslHMAC) BlockSize() int { ++ return h.blockSize ++} ++ ++func (h *opensslHMAC) Sum(in []byte) []byte { ++ if h.sum == nil { ++ size := h.Size() ++ h.sum = make([]byte, size) ++ } ++ // Make copy of context because Go hash.Hash mandates ++ // that Sum has no effect on the underlying stream. ++ // In particular it is OK to Sum, then Write more, then Sum again, ++ // and the second Sum acts as if the first didn't happen. ++ ctx2 := hmacCtxNew() ++ defer hmacCtxFree(ctx2) ++ if C.go_openssl_HMAC_CTX_copy(ctx2, h.ctx) == 0 { ++ panic("openssl: HMAC_CTX_copy failed") ++ } ++ C.go_openssl_HMAC_Final(ctx2, base(h.sum), nil) ++ return append(in, h.sum...) ++} ++ ++func hmacCtxNew() C.GO_HMAC_CTX_PTR { ++ if vMajor == 1 && vMinor == 0 { ++ // 0x120 is the sizeof value when building against OpenSSL 1.0.2 on Ubuntu 16.04. ++ ctx := (C.GO_HMAC_CTX_PTR)(C.malloc(0x120)) ++ if ctx != nil { ++ C.go_openssl_HMAC_CTX_init(ctx) ++ } ++ return ctx ++ } ++ return C.go_openssl_HMAC_CTX_new() ++} ++ ++func hmacCtxReset(ctx C.GO_HMAC_CTX_PTR) { ++ if ctx == nil { ++ return ++ } ++ if vMajor == 1 && vMinor == 0 { ++ C.go_openssl_HMAC_CTX_cleanup(ctx) ++ C.go_openssl_HMAC_CTX_init(ctx) ++ return ++ } ++ C.go_openssl_HMAC_CTX_reset(ctx) ++} ++ ++func hmacCtxFree(ctx C.GO_HMAC_CTX_PTR) { ++ if ctx == nil { ++ return ++ } ++ if vMajor == 1 && vMinor == 0 { ++ C.go_openssl_HMAC_CTX_cleanup(ctx) ++ C.free(unsafe.Pointer(ctx)) ++ return ++ } ++ C.go_openssl_HMAC_CTX_free(ctx) ++} +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/internal/subtle/aliasing.go b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/internal/subtle/aliasing.go +new file mode 100644 +index 00000000000000..db09e4aae64f8c +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/internal/subtle/aliasing.go +@@ -0,0 +1,32 @@ ++// Copyright 2018 The Go Authors. All rights reserved. ++// Use of this source code is governed by a BSD-style ++// license that can be found in the LICENSE file. ++ ++// Package subtle implements functions that are often useful in cryptographic ++// code but require careful thought to use correctly. ++// ++// This is a mirror of golang.org/x/crypto/internal/subtle. ++package subtle ++ ++import "unsafe" ++ ++// AnyOverlap reports whether x and y share memory at any (not necessarily ++// corresponding) index. The memory beyond the slice length is ignored. ++func AnyOverlap(x, y []byte) bool { ++ return len(x) > 0 && len(y) > 0 && ++ uintptr(unsafe.Pointer(&x[0])) <= uintptr(unsafe.Pointer(&y[len(y)-1])) && ++ uintptr(unsafe.Pointer(&y[0])) <= uintptr(unsafe.Pointer(&x[len(x)-1])) ++} ++ ++// InexactOverlap reports whether x and y share memory at any non-corresponding ++// index. The memory beyond the slice length is ignored. Note that x and y can ++// have different lengths and still not have any inexact overlap. ++// ++// InexactOverlap can be used to implement the requirements of the crypto/cipher ++// AEAD, Block, BlockMode and Stream interfaces. ++func InexactOverlap(x, y []byte) bool { ++ if len(x) == 0 || len(y) == 0 || &x[0] == &y[0] { ++ return false ++ } ++ return AnyOverlap(x, y) ++} +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/openssl.go b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/openssl.go +new file mode 100644 +index 00000000000000..2c354e1df0f818 +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/openssl.go +@@ -0,0 +1,260 @@ ++// Copyright (c) Microsoft Corporation. ++// Licensed under the MIT License. ++ ++//go:build linux && !android ++// +build linux,!android ++ ++// Package openssl provides access to OpenSSL cryptographic functions. ++package openssl ++ ++// #include "goopenssl.h" ++// #include ++// #cgo LDFLAGS: -ldl ++import "C" ++import ( ++ "errors" ++ "math/big" ++ "strconv" ++ "strings" ++ "sync" ++ "syscall" ++ "unsafe" ++) ++ ++var ( ++ providerNameFips = C.CString("fips") ++ providerNameDefault = C.CString("default") ++ propFipsYes = C.CString("fips=yes") ++ propFipsNo = C.CString("fips=no") ++ algProve = C.CString("SHA2-256") ++) ++ ++var ( ++ initOnce sync.Once ++ // errInit is set when first calling Init(). ++ errInit error ++ // vMajor and vMinor hold the major/minor OpenSSL version. ++ // It is only populated if Init has been called. ++ vMajor, vMinor int ++) ++ ++// knownVersions is a list of supported and well-known libcrypto.so suffixes in decreasing version order. ++// ++// FreeBSD library version numbering does not directly align to the version of OpenSSL. ++// Its preferred search order is 11 -> 111. ++// ++// Some distributions use 1.0.0 and others (such as Debian) 1.0.2 to refer to the same OpenSSL 1.0.2 version. ++// ++// Fedora derived distros use different naming for the version 1.0.x. ++var knownVersions = [...]string{"3", "1.1", "11", "111", "1.0.2", "1.0.0", "10"} ++ ++func errUnsuportedVersion() error { ++ return errors.New("openssl: OpenSSL version: " + strconv.Itoa(vMajor) + "." + strconv.Itoa(vMinor)) ++} ++ ++// Init loads and initializes OpenSSL. ++// It must be called before any other OpenSSL call. ++// ++// Only the first call to Init is effective, ++// subsequent calls will return the same error result as the one from the first call. ++// ++// If GO_OPENSSL_VERSION_OVERRIDE environment variable is not empty, its value will be appended to the OpenSSL shared library name ++// as a version suffix when calling dlopen. For example, "GO_OPENSSL_VERSION_OVERRIDE=1.1.1k-fips" ++// makes Init look for the shared library libcrypto.so.1.1.1k-fips. ++// If GO_OPENSSL_VERSION_OVERRIDE environment variable is empty, Init will try to load the OpenSSL shared library ++// using a list if supported and well-known version suffixes, going from higher to lower versions. ++func Init() error { ++ initOnce.Do(func() { ++ version, _ := syscall.Getenv("GO_OPENSSL_VERSION_OVERRIDE") ++ handle, err := loadLibrary(version) ++ if err != nil { ++ errInit = err ++ return ++ } ++ ++ vMajor = int(C.go_openssl_version_major(handle)) ++ vMinor = int(C.go_openssl_version_minor(handle)) ++ if vMajor == -1 || vMinor == -1 { ++ errInit = errors.New("openssl: can't retrieve OpenSSL version") ++ return ++ } ++ var supported bool ++ if vMajor == 1 { ++ supported = vMinor == 0 || vMinor == 1 ++ } else if vMajor == 3 { ++ // OpenSSL team guarantees API and ABI compatibility within the same major version since OpenSSL 3. ++ supported = true ++ } ++ if !supported { ++ errInit = errUnsuportedVersion() ++ return ++ } ++ ++ C.go_openssl_load_functions(handle, C.int(vMajor), C.int(vMinor)) ++ C.go_openssl_OPENSSL_init() ++ if vMajor == 1 && vMinor == 0 { ++ if C.go_openssl_thread_setup() != 1 { ++ errInit = newOpenSSLError("openssl: thread setup") ++ return ++ } ++ C.go_openssl_OPENSSL_add_all_algorithms_conf() ++ C.go_openssl_ERR_load_crypto_strings() ++ } else { ++ flags := C.uint64_t(C.GO_OPENSSL_INIT_ADD_ALL_CIPHERS | C.GO_OPENSSL_INIT_ADD_ALL_DIGESTS | C.GO_OPENSSL_INIT_LOAD_CONFIG | C.GO_OPENSSL_INIT_LOAD_CRYPTO_STRINGS) ++ if C.go_openssl_OPENSSL_init_crypto(flags, nil) != 1 { ++ errInit = newOpenSSLError("openssl: init crypto") ++ return ++ } ++ } ++ }) ++ return errInit ++} ++ ++func dlopen(version string) unsafe.Pointer { ++ cv := C.CString("libcrypto.so." + version) ++ defer C.free(unsafe.Pointer(cv)) ++ return C.dlopen(cv, C.RTLD_LAZY|C.RTLD_LOCAL) ++} ++ ++func loadLibrary(version string) (unsafe.Pointer, error) { ++ if version != "" { ++ // If version is specified try to load it or error out. ++ handle := dlopen(version) ++ if handle == nil { ++ errstr := C.GoString(C.dlerror()) ++ return nil, errors.New("openssl: can't load libcrypto.so." + version + ": " + errstr) ++ } ++ return handle, nil ++ } ++ for _, v := range knownVersions { ++ handle := dlopen(v) ++ if handle != nil { ++ return handle, nil ++ } ++ } ++ return nil, errors.New("openssl: can't load libcrypto.so using any known version suffix") ++} ++ ++// providerAvailable looks through provider's digests ++// checking if there is any that matches the props query. ++func providerAvailable(props *C.char) bool { ++ C.go_openssl_ERR_set_mark() ++ md := C.go_openssl_EVP_MD_fetch(nil, algProve, props) ++ C.go_openssl_ERR_pop_to_mark() ++ if md == nil { ++ return false ++ } ++ C.go_openssl_EVP_MD_free(md) ++ return true ++} ++ ++// FIPS returns true if OpenSSL is running in FIPS mode, else returns false. ++func FIPS() bool { ++ switch vMajor { ++ case 1: ++ return C.go_openssl_FIPS_mode() == 1 ++ case 3: ++ if C.go_openssl_EVP_default_properties_is_fips_enabled(nil) == 0 { ++ return false ++ } ++ // EVP_default_properties_is_fips_enabled can return true even if the FIPS provider isn't loaded, ++ // it is only based on the default properties. ++ return providerAvailable(propFipsYes) ++ default: ++ panic(errUnsuportedVersion()) ++ } ++} ++ ++// SetFIPS enables or disables FIPS mode. ++// ++// It implements the following provider fallback logic for OpenSSL 3: ++// - The "fips" provider is loaded if enabled=true and no loaded provider matches "fips=yes". ++// - The "default" provider is loaded if enabled=false and no loaded provider matches "fips=no". ++// This logic allows advanced users to define their own providers that match "fips=yes" and "fips=no" using the OpenSSL config file. ++func SetFIPS(enabled bool) error { ++ switch vMajor { ++ case 1: ++ var mode C.int ++ if enabled { ++ mode = C.int(1) ++ } else { ++ mode = C.int(0) ++ } ++ if C.go_openssl_FIPS_mode_set(mode) != 1 { ++ return newOpenSSLError("openssl: FIPS_mode_set") ++ } ++ return nil ++ case 3: ++ var props, provName *C.char ++ if enabled { ++ props = propFipsYes ++ provName = providerNameFips ++ } else { ++ props = propFipsNo ++ provName = providerNameDefault ++ } ++ // Check if there is any provider that matches props. ++ if !providerAvailable(props) { ++ // If not, fallback to provName provider. ++ if C.go_openssl_OSSL_PROVIDER_load(nil, provName) == nil { ++ return newOpenSSLError("openssl: OSSL_PROVIDER_try_load") ++ } ++ // Make sure we now have a provider available. ++ if !providerAvailable(props) { ++ return fail("SetFIPS(" + strconv.FormatBool(enabled) + ") not supported") ++ } ++ } ++ if C.go_openssl_EVP_set_default_properties(nil, props) != 1 { ++ return newOpenSSLError("openssl: EVP_set_default_properties") ++ } ++ return nil ++ default: ++ panic(errUnsuportedVersion()) ++ } ++} ++ ++// VersionText returns the version text of the OpenSSL currently loaded. ++func VersionText() string { ++ return C.GoString(C.go_openssl_OpenSSL_version(0)) ++} ++ ++func newOpenSSLError(msg string) error { ++ var b strings.Builder ++ var e C.ulong ++ ++ b.WriteString(msg) ++ b.WriteString("\nopenssl error(s):\n") ++ ++ for { ++ e = C.go_openssl_ERR_get_error() ++ if e == 0 { ++ break ++ } ++ var buf [256]byte ++ C.go_openssl_ERR_error_string_n(e, (*C.char)(unsafe.Pointer(&buf[0])), 256) ++ b.Write(buf[:]) ++ b.WriteByte('\n') ++ } ++ return errors.New(b.String()) ++} ++ ++type fail string ++ ++func (e fail) Error() string { return "openssl: " + string(e) + " failed" } ++ ++func bigToBN(x *big.Int) C.GO_BIGNUM_PTR { ++ if x == nil { ++ return nil ++ } ++ raw := x.Bytes() ++ return C.go_openssl_BN_bin2bn(base(raw), C.int(len(raw)), nil) ++} ++ ++func bnToBig(bn C.GO_BIGNUM_PTR) *big.Int { ++ if bn == nil { ++ return nil ++ } ++ raw := make([]byte, (C.go_openssl_BN_num_bits(bn)+7)/8) ++ n := C.go_openssl_BN_bn2bin(bn, base(raw)) ++ return new(big.Int).SetBytes(raw[:n]) ++} +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/openssl_funcs.h b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/openssl_funcs.h +new file mode 100644 +index 00000000000000..95024cf8ecb72c +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/openssl_funcs.h +@@ -0,0 +1,242 @@ ++// Copyright (c) Microsoft Corporation. ++// Licensed under the MIT License. ++ ++// To make this header standalone (so that building Go does not require ++// having a full set of OpenSSL headers), the struct details are not here. ++// Instead, while testing the openssl module, we generate and compile a C program ++// that checks that the function signatures match the OpenSSL equivalents. ++// The generation of the checking program depends on the declaration ++// forms used below, which includes commented directives (#include, #if and #endif) ++// and comments starting with `check:`. ++ ++#include // size_t ++#include // uint64_t ++ ++// #include ++enum { ++ GO_OPENSSL_INIT_LOAD_CRYPTO_STRINGS = 0x00000002L, ++ GO_OPENSSL_INIT_ADD_ALL_CIPHERS = 0x00000004L, ++ GO_OPENSSL_INIT_ADD_ALL_DIGESTS = 0x00000008L, ++ GO_OPENSSL_INIT_LOAD_CONFIG = 0x00000040L ++}; ++ ++// #include ++enum { ++ GO_AES_ENCRYPT = 1, ++ GO_AES_DECRYPT = 0 ++}; ++ ++// #include ++enum { ++ GO_EVP_CTRL_GCM_GET_TAG = 0x10, ++ GO_EVP_CTRL_GCM_SET_TAG = 0x11, ++ GO_EVP_PKEY_CTRL_MD = 1, ++ GO_EVP_PKEY_RSA = 6, ++ GO_EVP_PKEY_EC = 408, ++ GO_EVP_MAX_MD_SIZE = 64 ++}; ++ ++// #include ++enum { ++ GO_EVP_PKEY_CTRL_EC_PARAMGEN_CURVE_NID = 0x1001 ++}; ++ ++// #include ++enum { ++ GO_NID_X9_62_prime256v1 = 415, ++ GO_NID_secp224r1 = 713, ++ GO_NID_secp384r1 = 715, ++ GO_NID_secp521r1 = 716 ++}; ++ ++// #include ++enum { ++ GO_RSA_PKCS1_PADDING = 1, ++ GO_RSA_NO_PADDING = 3, ++ GO_RSA_PKCS1_OAEP_PADDING = 4, ++ GO_RSA_PKCS1_PSS_PADDING = 6, ++ GO_EVP_PKEY_CTRL_RSA_PADDING = 0x1001, ++ GO_EVP_PKEY_CTRL_RSA_PSS_SALTLEN = 0x1002, ++ GO_EVP_PKEY_CTRL_RSA_KEYGEN_BITS = 0x1003, ++ GO_EVP_PKEY_CTRL_RSA_OAEP_MD = 0x1009, ++ GO_EVP_PKEY_CTRL_RSA_OAEP_LABEL = 0x100A ++}; ++ ++typedef void* GO_EVP_CIPHER_PTR; ++typedef void* GO_EVP_CIPHER_CTX_PTR; ++typedef void* GO_EVP_PKEY_PTR; ++typedef void* GO_EVP_PKEY_CTX_PTR; ++typedef void* GO_EVP_MD_PTR; ++typedef void* GO_EVP_MD_CTX_PTR; ++typedef void* GO_HMAC_CTX_PTR; ++typedef void* GO_OPENSSL_INIT_SETTINGS_PTR; ++typedef void* GO_OSSL_LIB_CTX_PTR; ++typedef void* GO_OSSL_PROVIDER_PTR; ++typedef void* GO_ENGINE_PTR; ++typedef void* GO_BIGNUM_PTR; ++typedef void* GO_BN_CTX_PTR; ++typedef void* GO_EC_KEY_PTR; ++typedef void* GO_EC_POINT_PTR; ++typedef void* GO_EC_GROUP_PTR; ++typedef void* GO_RSA_PTR; ++ ++// List of all functions from the libcrypto that are used in this package. ++// Forgetting to add a function here results in build failure with message reporting the function ++// that needs to be added. ++// ++// The purpose of FOR_ALL_OPENSSL_FUNCTIONS is to define all libcrypto functions ++// without depending on the openssl headers so it is easier to use this package ++// with an openssl version different that the one used at build time. ++// ++// The following macros may not be defined at this point, ++// they are not resolved here but just accumulated in FOR_ALL_OPENSSL_FUNCTIONS. ++// ++// DEFINEFUNC defines and loads openssl functions that can be directly called from Go as their signatures match ++// the OpenSSL API and do not require special logic. ++// The process will be aborted if the function can't be loaded. ++// ++// DEFINEFUNC_LEGACY_1_0 acts like DEFINEFUNC but only aborts the process if the function can't be loaded ++// when using 1.0.x. This indicates the function is required when using 1.0.x, but is unused when using later versions. ++// It also might not exist in later versions. ++// ++// DEFINEFUNC_LEGACY_1 acts like DEFINEFUNC but only aborts the process if the function can't be loaded ++// when using 1.x. This indicates the function is required when using 1.x, but is unused when using later versions. ++// It also might not exist in later versions. ++// ++// DEFINEFUNC_1_1 acts like DEFINEFUNC but only aborts the process if function can't be loaded ++// when using 1.1.0 or higher. ++// ++// DEFINEFUNC_3_0 acts like DEFINEFUNC but only aborts the process if function can't be loaded ++// when using 3.0.0 or higher. ++// ++// DEFINEFUNC_RENAMED_1_1 acts like DEFINEFUNC but tries to load the function using the new name when using >= 1.1.x ++// and the old name when using 1.0.2. In both cases the function will have the new name. ++// ++// DEFINEFUNC_RENAMED_3_0 acts like DEFINEFUNC but tries to load the function using the new name when using >= 3.x ++// and the old name when using 1.x. In both cases the function will have the new name. ++// ++// #include ++// #include ++// #include ++// #include ++// #include ++// #include ++// #include ++// #if OPENSSL_VERSION_NUMBER >= 0x30000000L ++// #include ++// #endif ++#define FOR_ALL_OPENSSL_FUNCTIONS \ ++DEFINEFUNC(int, ERR_set_mark, (void), ()) \ ++DEFINEFUNC(int, ERR_pop_to_mark, (void), ()) \ ++DEFINEFUNC(unsigned long, ERR_get_error, (void), ()) \ ++DEFINEFUNC(void, ERR_error_string_n, (unsigned long e, char *buf, size_t len), (e, buf, len)) \ ++DEFINEFUNC_RENAMED_1_1(const char *, OpenSSL_version, SSLeay_version, (int type), (type)) \ ++DEFINEFUNC(void, OPENSSL_init, (void), ()) \ ++DEFINEFUNC_LEGACY_1_0(void, ERR_load_crypto_strings, (void), ()) \ ++DEFINEFUNC_LEGACY_1_0(int, CRYPTO_num_locks, (void), ()) \ ++DEFINEFUNC_LEGACY_1_0(void, CRYPTO_set_id_callback, (unsigned long (*id_function)(void)), (id_function)) \ ++DEFINEFUNC_LEGACY_1_0(void, CRYPTO_set_locking_callback, (void (*locking_function)(int mode, int n, const char *file, int line)), (locking_function)) \ ++DEFINEFUNC_LEGACY_1_0(void, OPENSSL_add_all_algorithms_conf, (void), ()) \ ++DEFINEFUNC_1_1(int, OPENSSL_init_crypto, (uint64_t ops, const GO_OPENSSL_INIT_SETTINGS_PTR settings), (ops, settings)) \ ++DEFINEFUNC_LEGACY_1(int, FIPS_mode, (void), ()) \ ++DEFINEFUNC_LEGACY_1(int, FIPS_mode_set, (int r), (r)) \ ++DEFINEFUNC_3_0(int, EVP_default_properties_is_fips_enabled, (GO_OSSL_LIB_CTX_PTR libctx), (libctx)) \ ++DEFINEFUNC_3_0(int, EVP_set_default_properties, (GO_OSSL_LIB_CTX_PTR libctx, const char *propq), (libctx, propq)) \ ++DEFINEFUNC_3_0(GO_OSSL_PROVIDER_PTR, OSSL_PROVIDER_load, (GO_OSSL_LIB_CTX_PTR libctx, const char *name), (libctx, name)) \ ++DEFINEFUNC(int, RAND_bytes, (unsigned char* arg0, int arg1), (arg0, arg1)) \ ++DEFINEFUNC(int, EVP_DigestInit_ex, (GO_EVP_MD_CTX_PTR ctx, const GO_EVP_MD_PTR type, GO_ENGINE_PTR impl), (ctx, type, impl)) \ ++DEFINEFUNC(int, EVP_DigestUpdate, (GO_EVP_MD_CTX_PTR ctx, const void *d, size_t cnt), (ctx, d, cnt)) \ ++DEFINEFUNC(int, EVP_DigestFinal_ex, (GO_EVP_MD_CTX_PTR ctx, unsigned char *md, unsigned int *s), (ctx, md, s)) \ ++DEFINEFUNC_RENAMED_1_1(GO_EVP_MD_CTX_PTR, EVP_MD_CTX_new, EVP_MD_CTX_create, (), ()) \ ++DEFINEFUNC_RENAMED_1_1(void, EVP_MD_CTX_free, EVP_MD_CTX_destroy, (GO_EVP_MD_CTX_PTR ctx), (ctx)) \ ++DEFINEFUNC(int, EVP_MD_CTX_copy_ex, (GO_EVP_MD_CTX_PTR out, const GO_EVP_MD_CTX_PTR in), (out, in)) \ ++DEFINEFUNC_RENAMED_1_1(int, EVP_MD_CTX_reset, EVP_MD_CTX_cleanup, (GO_EVP_MD_CTX_PTR ctx), (ctx)) \ ++DEFINEFUNC(const GO_EVP_MD_PTR, EVP_md5, (void), ()) \ ++DEFINEFUNC(const GO_EVP_MD_PTR, EVP_sha1, (void), ()) \ ++DEFINEFUNC(const GO_EVP_MD_PTR, EVP_sha224, (void), ()) \ ++DEFINEFUNC(const GO_EVP_MD_PTR, EVP_sha256, (void), ()) \ ++DEFINEFUNC(const GO_EVP_MD_PTR, EVP_sha384, (void), ()) \ ++DEFINEFUNC(const GO_EVP_MD_PTR, EVP_sha512, (void), ()) \ ++DEFINEFUNC_1_1(const GO_EVP_MD_PTR, EVP_md5_sha1, (void), ()) \ ++DEFINEFUNC_3_0(GO_EVP_MD_PTR, EVP_MD_fetch, (GO_OSSL_LIB_CTX_PTR ctx, const char *algorithm, const char *properties), (ctx, algorithm, properties)) \ ++DEFINEFUNC_3_0(void, EVP_MD_free, (GO_EVP_MD_PTR md), (md)) \ ++DEFINEFUNC_RENAMED_3_0(int, EVP_MD_get_size, EVP_MD_size, (const GO_EVP_MD_PTR arg0), (arg0)) \ ++DEFINEFUNC_LEGACY_1_0(void, HMAC_CTX_init, (GO_HMAC_CTX_PTR arg0), (arg0)) \ ++DEFINEFUNC_LEGACY_1_0(void, HMAC_CTX_cleanup, (GO_HMAC_CTX_PTR arg0), (arg0)) \ ++DEFINEFUNC(int, HMAC_Init_ex, (GO_HMAC_CTX_PTR arg0, const void *arg1, int arg2, const GO_EVP_MD_PTR arg3, GO_ENGINE_PTR arg4), (arg0, arg1, arg2, arg3, arg4)) \ ++DEFINEFUNC(int, HMAC_Update, (GO_HMAC_CTX_PTR arg0, const unsigned char *arg1, size_t arg2), (arg0, arg1, arg2)) \ ++DEFINEFUNC(int, HMAC_Final, (GO_HMAC_CTX_PTR arg0, unsigned char *arg1, unsigned int *arg2), (arg0, arg1, arg2)) \ ++DEFINEFUNC(int, HMAC_CTX_copy, (GO_HMAC_CTX_PTR dest, GO_HMAC_CTX_PTR src), (dest, src)) \ ++DEFINEFUNC_1_1(void, HMAC_CTX_free, (GO_HMAC_CTX_PTR arg0), (arg0)) \ ++DEFINEFUNC_1_1(GO_HMAC_CTX_PTR, HMAC_CTX_new, (void), ()) \ ++DEFINEFUNC_1_1(int, HMAC_CTX_reset, (GO_HMAC_CTX_PTR arg0), (arg0)) \ ++DEFINEFUNC(GO_EVP_CIPHER_CTX_PTR, EVP_CIPHER_CTX_new, (void), ()) \ ++DEFINEFUNC(int, EVP_CIPHER_CTX_set_padding, (GO_EVP_CIPHER_CTX_PTR x, int padding), (x, padding)) \ ++DEFINEFUNC(int, EVP_CipherInit_ex, (GO_EVP_CIPHER_CTX_PTR ctx, const GO_EVP_CIPHER_PTR type, GO_ENGINE_PTR impl, const unsigned char *key, const unsigned char *iv, int enc), (ctx, type, impl, key, iv, enc)) \ ++DEFINEFUNC(int, EVP_CipherUpdate, (GO_EVP_CIPHER_CTX_PTR ctx, unsigned char *out, int *outl, const unsigned char *in, int inl), (ctx, out, outl, in, inl)) \ ++DEFINEFUNC(GO_BIGNUM_PTR, BN_new, (void), ()) \ ++DEFINEFUNC(void, BN_free, (GO_BIGNUM_PTR arg0), (arg0)) \ ++DEFINEFUNC(void, BN_clear_free, (GO_BIGNUM_PTR arg0), (arg0)) \ ++DEFINEFUNC(int, BN_num_bits, (const GO_BIGNUM_PTR arg0), (arg0)) \ ++DEFINEFUNC(GO_BIGNUM_PTR, BN_bin2bn, (const unsigned char *arg0, int arg1, GO_BIGNUM_PTR arg2), (arg0, arg1, arg2)) \ ++DEFINEFUNC(int, BN_bn2bin, (const GO_BIGNUM_PTR arg0, unsigned char *arg1), (arg0, arg1)) \ ++DEFINEFUNC(void, EC_GROUP_free, (GO_EC_GROUP_PTR arg0), (arg0)) \ ++DEFINEFUNC(GO_EC_POINT_PTR, EC_POINT_new, (const GO_EC_GROUP_PTR arg0), (arg0)) \ ++DEFINEFUNC(void, EC_POINT_free, (GO_EC_POINT_PTR arg0), (arg0)) \ ++DEFINEFUNC(int, EC_POINT_get_affine_coordinates_GFp, (const GO_EC_GROUP_PTR arg0, const GO_EC_POINT_PTR arg1, GO_BIGNUM_PTR arg2, GO_BIGNUM_PTR arg3, GO_BN_CTX_PTR arg4), (arg0, arg1, arg2, arg3, arg4)) \ ++DEFINEFUNC(GO_EC_KEY_PTR, EC_KEY_new_by_curve_name, (int arg0), (arg0)) \ ++DEFINEFUNC(int, EC_KEY_set_public_key_affine_coordinates, (GO_EC_KEY_PTR key, GO_BIGNUM_PTR x, GO_BIGNUM_PTR y), (key, x, y)) \ ++DEFINEFUNC(void, EC_KEY_free, (GO_EC_KEY_PTR arg0), (arg0)) \ ++DEFINEFUNC(const GO_EC_GROUP_PTR, EC_KEY_get0_group, (const GO_EC_KEY_PTR arg0), (arg0)) \ ++DEFINEFUNC(int, EC_KEY_set_private_key, (GO_EC_KEY_PTR arg0, const GO_BIGNUM_PTR arg1), (arg0, arg1)) \ ++DEFINEFUNC(const GO_BIGNUM_PTR, EC_KEY_get0_private_key, (const GO_EC_KEY_PTR arg0), (arg0)) \ ++DEFINEFUNC(const GO_EC_POINT_PTR, EC_KEY_get0_public_key, (const GO_EC_KEY_PTR arg0), (arg0)) \ ++DEFINEFUNC(GO_RSA_PTR, RSA_new, (void), ()) \ ++DEFINEFUNC(void, RSA_free, (GO_RSA_PTR arg0), (arg0)) \ ++DEFINEFUNC_1_1(int, RSA_set0_factors, (GO_RSA_PTR rsa, GO_BIGNUM_PTR p, GO_BIGNUM_PTR q), (rsa, p, q)) \ ++DEFINEFUNC_1_1(int, RSA_set0_crt_params, (GO_RSA_PTR rsa, GO_BIGNUM_PTR dmp1, GO_BIGNUM_PTR dmp2,GO_BIGNUM_PTR iqmp), (rsa, dmp1, dmp2, iqmp)) \ ++DEFINEFUNC_1_1(void, RSA_get0_crt_params, (const GO_RSA_PTR r, const GO_BIGNUM_PTR *dmp1, const GO_BIGNUM_PTR *dmq1, const GO_BIGNUM_PTR *iqmp), (r, dmp1, dmq1, iqmp)) \ ++DEFINEFUNC_1_1(int, RSA_set0_key, (GO_RSA_PTR r, GO_BIGNUM_PTR n, GO_BIGNUM_PTR e, GO_BIGNUM_PTR d), (r, n, e, d)) \ ++DEFINEFUNC_1_1(void, RSA_get0_factors, (const GO_RSA_PTR rsa, const GO_BIGNUM_PTR *p, const GO_BIGNUM_PTR *q), (rsa, p, q)) \ ++DEFINEFUNC_1_1(void, RSA_get0_key, (const GO_RSA_PTR rsa, const GO_BIGNUM_PTR *n, const GO_BIGNUM_PTR *e, const GO_BIGNUM_PTR *d), (rsa, n, e, d)) \ ++DEFINEFUNC(int, EVP_EncryptInit_ex, (GO_EVP_CIPHER_CTX_PTR ctx, const GO_EVP_CIPHER_PTR type, GO_ENGINE_PTR impl, const unsigned char *key, const unsigned char *iv), (ctx, type, impl, key, iv)) \ ++DEFINEFUNC(int, EVP_EncryptUpdate, (GO_EVP_CIPHER_CTX_PTR ctx, unsigned char *out, int *outl, const unsigned char *in, int inl), (ctx, out, outl, in, inl)) \ ++DEFINEFUNC(int, EVP_EncryptFinal_ex, (GO_EVP_CIPHER_CTX_PTR ctx, unsigned char *out, int *outl), (ctx, out, outl)) \ ++DEFINEFUNC(int, EVP_DecryptUpdate, (GO_EVP_CIPHER_CTX_PTR ctx, unsigned char *out, int *outl, const unsigned char *in, int inl), (ctx, out, outl, in, inl)) \ ++DEFINEFUNC(int, EVP_DecryptFinal_ex, (GO_EVP_CIPHER_CTX_PTR ctx, unsigned char *outm, int *outl), (ctx, outm, outl)) \ ++DEFINEFUNC(const GO_EVP_CIPHER_PTR, EVP_aes_128_gcm, (void), ()) \ ++DEFINEFUNC(const GO_EVP_CIPHER_PTR, EVP_aes_128_cbc, (void), ()) \ ++DEFINEFUNC(const GO_EVP_CIPHER_PTR, EVP_aes_128_ctr, (void), ()) \ ++DEFINEFUNC(const GO_EVP_CIPHER_PTR, EVP_aes_128_ecb, (void), ()) \ ++DEFINEFUNC(const GO_EVP_CIPHER_PTR, EVP_aes_192_gcm, (void), ()) \ ++DEFINEFUNC(const GO_EVP_CIPHER_PTR, EVP_aes_192_cbc, (void), ()) \ ++DEFINEFUNC(const GO_EVP_CIPHER_PTR, EVP_aes_192_ctr, (void), ()) \ ++DEFINEFUNC(const GO_EVP_CIPHER_PTR, EVP_aes_192_ecb, (void), ()) \ ++DEFINEFUNC(const GO_EVP_CIPHER_PTR, EVP_aes_256_cbc, (void), ()) \ ++DEFINEFUNC(const GO_EVP_CIPHER_PTR, EVP_aes_256_ctr, (void), ()) \ ++DEFINEFUNC(const GO_EVP_CIPHER_PTR, EVP_aes_256_ecb, (void), ()) \ ++DEFINEFUNC(const GO_EVP_CIPHER_PTR, EVP_aes_256_gcm, (void), ()) \ ++DEFINEFUNC(void, EVP_CIPHER_CTX_free, (GO_EVP_CIPHER_CTX_PTR arg0), (arg0)) \ ++DEFINEFUNC(int, EVP_CIPHER_CTX_ctrl, (GO_EVP_CIPHER_CTX_PTR ctx, int type, int arg, void *ptr), (ctx, type, arg, ptr)) \ ++DEFINEFUNC(GO_EVP_PKEY_PTR, EVP_PKEY_new, (void), ()) \ ++/* EVP_PKEY_size pkey parameter is const since OpenSSL 1.1.1. */ \ ++/* Exclude it from headercheck tool when using previous OpenSSL versions. */ \ ++/*check:from=1.1.1*/ DEFINEFUNC_RENAMED_3_0(int, EVP_PKEY_get_size, EVP_PKEY_size, (const GO_EVP_PKEY_PTR pkey), (pkey)) \ ++DEFINEFUNC(void, EVP_PKEY_free, (GO_EVP_PKEY_PTR arg0), (arg0)) \ ++DEFINEFUNC(GO_EC_KEY_PTR, EVP_PKEY_get1_EC_KEY, (GO_EVP_PKEY_PTR pkey), (pkey)) \ ++DEFINEFUNC(GO_RSA_PTR, EVP_PKEY_get1_RSA, (GO_EVP_PKEY_PTR pkey), (pkey)) \ ++DEFINEFUNC(int, EVP_PKEY_assign, (GO_EVP_PKEY_PTR pkey, int type, void *key), (pkey, type, key)) \ ++DEFINEFUNC(int, EVP_PKEY_verify, (GO_EVP_PKEY_CTX_PTR ctx, const unsigned char *sig, size_t siglen, const unsigned char *tbs, size_t tbslen), (ctx, sig, siglen, tbs, tbslen)) \ ++DEFINEFUNC(GO_EVP_PKEY_CTX_PTR, EVP_PKEY_CTX_new, (GO_EVP_PKEY_PTR arg0, GO_ENGINE_PTR arg1), (arg0, arg1)) \ ++DEFINEFUNC(GO_EVP_PKEY_CTX_PTR, EVP_PKEY_CTX_new_id, (int id, GO_ENGINE_PTR e), (id, e)) \ ++DEFINEFUNC(int, EVP_PKEY_keygen_init, (GO_EVP_PKEY_CTX_PTR ctx), (ctx)) \ ++DEFINEFUNC(int, EVP_PKEY_keygen, (GO_EVP_PKEY_CTX_PTR ctx, GO_EVP_PKEY_PTR *ppkey), (ctx, ppkey)) \ ++DEFINEFUNC(void, EVP_PKEY_CTX_free, (GO_EVP_PKEY_CTX_PTR arg0), (arg0)) \ ++DEFINEFUNC(int, EVP_PKEY_CTX_ctrl, (GO_EVP_PKEY_CTX_PTR ctx, int keytype, int optype, int cmd, int p1, void *p2), (ctx, keytype, optype, cmd, p1, p2)) \ ++DEFINEFUNC(int, EVP_PKEY_decrypt, (GO_EVP_PKEY_CTX_PTR arg0, unsigned char *arg1, size_t *arg2, const unsigned char *arg3, size_t arg4), (arg0, arg1, arg2, arg3, arg4)) \ ++DEFINEFUNC(int, EVP_PKEY_encrypt, (GO_EVP_PKEY_CTX_PTR arg0, unsigned char *arg1, size_t *arg2, const unsigned char *arg3, size_t arg4), (arg0, arg1, arg2, arg3, arg4)) \ ++DEFINEFUNC(int, EVP_PKEY_decrypt_init, (GO_EVP_PKEY_CTX_PTR arg0), (arg0)) \ ++DEFINEFUNC(int, EVP_PKEY_encrypt_init, (GO_EVP_PKEY_CTX_PTR arg0), (arg0)) \ ++DEFINEFUNC(int, EVP_PKEY_sign_init, (GO_EVP_PKEY_CTX_PTR arg0), (arg0)) \ ++DEFINEFUNC(int, EVP_PKEY_verify_init, (GO_EVP_PKEY_CTX_PTR arg0), (arg0)) \ ++DEFINEFUNC(int, EVP_PKEY_sign, (GO_EVP_PKEY_CTX_PTR arg0, unsigned char *arg1, size_t *arg2, const unsigned char *arg3, size_t arg4), (arg0, arg1, arg2, arg3, arg4)) +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/openssl_lock_setup.c b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/openssl_lock_setup.c +new file mode 100644 +index 00000000000000..5cd7275f4075ed +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/openssl_lock_setup.c +@@ -0,0 +1,53 @@ ++// Copyright (c) Microsoft Corporation. ++// Licensed under the MIT License. ++ ++//go:build linux && !android ++// +build linux,!android ++ ++#include "goopenssl.h" ++ ++#include ++#include ++#include ++#include ++ ++#define _GNU_SOURCE ++#include ++ ++#define MUTEX_TYPE pthread_mutex_t ++#define MUTEX_SETUP(x) pthread_mutex_init(&(x), NULL) ++#define MUTEX_CLEANUP(x) pthread_mutex_destroy(&(x)) ++#define MUTEX_LOCK(x) pthread_mutex_lock(&(x)) ++#define MUTEX_UNLOCK(x) pthread_mutex_unlock(&(x)) ++#define THREAD_ID pthread_self() ++#define CRYPTO_LOCK 0x01 ++ ++/* This array will store all of the mutexes available to OpenSSL. */ ++static MUTEX_TYPE *mutex_buf = NULL; ++ ++static void locking_function(int mode, int n, const char *file, int line) ++{ ++ if(mode & CRYPTO_LOCK) ++ MUTEX_LOCK(mutex_buf[n]); ++ else ++ MUTEX_UNLOCK(mutex_buf[n]); ++} ++ ++static unsigned long id_function(void) ++{ ++ return ((unsigned long)syscall(__NR_gettid)); ++} ++ ++int go_openssl_thread_setup(void) ++{ ++ int i; ++ ++ mutex_buf = malloc(go_openssl_CRYPTO_num_locks() * sizeof(MUTEX_TYPE)); ++ if(!mutex_buf) ++ return 0; ++ for(i = 0; i < go_openssl_CRYPTO_num_locks(); i++) ++ MUTEX_SETUP(mutex_buf[i]); ++ go_openssl_CRYPTO_set_id_callback(id_function); ++ go_openssl_CRYPTO_set_locking_callback(locking_function); ++ return 1; ++} +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/rand.go b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/rand.go +new file mode 100644 +index 00000000000000..17f64a52ae5255 +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/rand.go +@@ -0,0 +1,24 @@ ++// Copyright (c) Microsoft Corporation. ++// Licensed under the MIT License. ++ ++//go:build linux && !android ++// +build linux,!android ++ ++package openssl ++ ++// #include "goopenssl.h" ++import "C" ++import "unsafe" ++ ++type randReader int ++ ++func (randReader) Read(b []byte) (int, error) { ++ // Note: RAND_bytes should never fail; the return value exists only for historical reasons. ++ // We check it even so. ++ if len(b) > 0 && C.go_openssl_RAND_bytes((*C.uchar)(unsafe.Pointer(&b[0])), C.int(len(b))) == 0 { ++ return 0, fail("RAND_bytes") ++ } ++ return len(b), nil ++} ++ ++const RandReader = randReader(0) +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/rsa.go b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/rsa.go +new file mode 100644 +index 00000000000000..05ff62cd7399af +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/rsa.go +@@ -0,0 +1,303 @@ ++// Copyright (c) Microsoft Corporation. ++// Licensed under the MIT License. ++ ++//go:build linux && !android ++// +build linux,!android ++ ++package openssl ++ ++// #include "goopenssl.h" ++import "C" ++import ( ++ "crypto" ++ "crypto/subtle" ++ "errors" ++ "hash" ++ "math/big" ++ "runtime" ++ "unsafe" ++) ++ ++func GenerateKeyRSA(bits int) (N, E, D, P, Q, Dp, Dq, Qinv *big.Int, err error) { ++ bad := func(e error) (N, E, D, P, Q, Dp, Dq, Qinv *big.Int, err error) { ++ return nil, nil, nil, nil, nil, nil, nil, nil, e ++ } ++ pkey, err := generateEVPPKey(C.GO_EVP_PKEY_RSA, bits, "") ++ if err != nil { ++ return bad(err) ++ } ++ defer C.go_openssl_EVP_PKEY_free(pkey) ++ key := C.go_openssl_EVP_PKEY_get1_RSA(pkey) ++ if key == nil { ++ return bad(newOpenSSLError("EVP_PKEY_get1_RSA failed")) ++ } ++ N, E, D = rsaGetKey(key) ++ P, Q = rsaGetFactors(key) ++ Dp, Dq, Qinv = rsaGetCRTParams(key) ++ return ++} ++ ++type PublicKeyRSA struct { ++ // _pkey MUST NOT be accessed directly. Instead, use the withKey method. ++ _pkey C.GO_EVP_PKEY_PTR ++} ++ ++func NewPublicKeyRSA(N, E *big.Int) (*PublicKeyRSA, error) { ++ key := C.go_openssl_RSA_new() ++ if key == nil { ++ return nil, newOpenSSLError("RSA_new failed") ++ } ++ if !rsaSetKey(key, N, E, nil) { ++ return nil, fail("RSA_set0_key") ++ } ++ pkey := C.go_openssl_EVP_PKEY_new() ++ if pkey == nil { ++ C.go_openssl_RSA_free(key) ++ return nil, newOpenSSLError("EVP_PKEY_new failed") ++ } ++ if C.go_openssl_EVP_PKEY_assign(pkey, C.GO_EVP_PKEY_RSA, (unsafe.Pointer)(key)) != 1 { ++ C.go_openssl_RSA_free(key) ++ C.go_openssl_EVP_PKEY_free(pkey) ++ return nil, newOpenSSLError("EVP_PKEY_assign failed") ++ } ++ k := &PublicKeyRSA{_pkey: pkey} ++ runtime.SetFinalizer(k, (*PublicKeyRSA).finalize) ++ return k, nil ++} ++ ++func (k *PublicKeyRSA) finalize() { ++ C.go_openssl_EVP_PKEY_free(k._pkey) ++} ++ ++func (k *PublicKeyRSA) withKey(f func(C.GO_EVP_PKEY_PTR) C.int) C.int { ++ // Because of the finalizer, any time _pkey is passed to cgo, that call must ++ // be followed by a call to runtime.KeepAlive, to make sure k is not ++ // collected (and finalized) before the cgo call returns. ++ defer runtime.KeepAlive(k) ++ return f(k._pkey) ++} ++ ++type PrivateKeyRSA struct { ++ // _pkey MUST NOT be accessed directly. Instead, use the withKey method. ++ _pkey C.GO_EVP_PKEY_PTR ++} ++ ++func NewPrivateKeyRSA(N, E, D, P, Q, Dp, Dq, Qinv *big.Int) (*PrivateKeyRSA, error) { ++ key := C.go_openssl_RSA_new() ++ if key == nil { ++ return nil, newOpenSSLError("RSA_new failed") ++ } ++ if !rsaSetKey(key, N, E, D) { ++ return nil, fail("RSA_set0_key") ++ } ++ if P != nil && Q != nil { ++ if !rsaSetFactors(key, P, Q) { ++ return nil, fail("RSA_set0_factors") ++ } ++ } ++ if Dp != nil && Dq != nil && Qinv != nil { ++ if !rsaSetCRTParams(key, Dp, Dq, Qinv) { ++ return nil, fail("RSA_set0_crt_params") ++ } ++ } ++ pkey := C.go_openssl_EVP_PKEY_new() ++ if pkey == nil { ++ C.go_openssl_RSA_free(key) ++ return nil, newOpenSSLError("EVP_PKEY_new failed") ++ } ++ if C.go_openssl_EVP_PKEY_assign(pkey, C.GO_EVP_PKEY_RSA, (unsafe.Pointer)(key)) != 1 { ++ C.go_openssl_RSA_free(key) ++ C.go_openssl_EVP_PKEY_free(pkey) ++ return nil, newOpenSSLError("EVP_PKEY_assign failed") ++ } ++ k := &PrivateKeyRSA{_pkey: pkey} ++ runtime.SetFinalizer(k, (*PrivateKeyRSA).finalize) ++ return k, nil ++} ++ ++func (k *PrivateKeyRSA) finalize() { ++ C.go_openssl_EVP_PKEY_free(k._pkey) ++} ++ ++func (k *PrivateKeyRSA) withKey(f func(C.GO_EVP_PKEY_PTR) C.int) C.int { ++ // Because of the finalizer, any time _pkey is passed to cgo, that call must ++ // be followed by a call to runtime.KeepAlive, to make sure k is not ++ // collected (and finalized) before the cgo call returns. ++ defer runtime.KeepAlive(k) ++ return f(k._pkey) ++} ++ ++func DecryptRSAOAEP(h hash.Hash, priv *PrivateKeyRSA, ciphertext, label []byte) ([]byte, error) { ++ return evpDecrypt(priv.withKey, C.GO_RSA_PKCS1_OAEP_PADDING, h, label, ciphertext) ++} ++ ++func EncryptRSAOAEP(h hash.Hash, pub *PublicKeyRSA, msg, label []byte) ([]byte, error) { ++ return evpEncrypt(pub.withKey, C.GO_RSA_PKCS1_OAEP_PADDING, h, label, msg) ++} ++ ++func DecryptRSAPKCS1(priv *PrivateKeyRSA, ciphertext []byte) ([]byte, error) { ++ return evpDecrypt(priv.withKey, C.GO_RSA_PKCS1_PADDING, nil, nil, ciphertext) ++} ++ ++func EncryptRSAPKCS1(pub *PublicKeyRSA, msg []byte) ([]byte, error) { ++ return evpEncrypt(pub.withKey, C.GO_RSA_PKCS1_PADDING, nil, nil, msg) ++} ++ ++func DecryptRSANoPadding(priv *PrivateKeyRSA, ciphertext []byte) ([]byte, error) { ++ ret, err := evpDecrypt(priv.withKey, C.GO_RSA_NO_PADDING, nil, nil, ciphertext) ++ if err != nil { ++ return nil, err ++ } ++ // We could return here, but the Go standard library test expects DecryptRSANoPadding to verify the result ++ // in order to defend against errors in the CRT computation. ++ // ++ // The following code tries to replicate the verification implemented in the upstream function decryptAndCheck, found at ++ // https://github.com/golang/go/blob/9de1ac6ac2cad3871760d0aa288f5ca713afd0a6/src/crypto/rsa/rsa.go#L569-L582. ++ pub := &PublicKeyRSA{_pkey: priv._pkey} ++ // A private EVP_PKEY can be used as a public key as it contains the public information. ++ enc, err := EncryptRSANoPadding(pub, ret) ++ if err != nil { ++ return nil, err ++ } ++ // Upstream does not do a constant time comparison because it works with math/big instead of byte slices, ++ // and math/big does not support constant-time arithmetic yet. See #20654 for more info. ++ if subtle.ConstantTimeCompare(ciphertext, enc) != 1 { ++ return nil, errors.New("rsa: internal error") ++ } ++ return ret, nil ++} ++ ++func EncryptRSANoPadding(pub *PublicKeyRSA, msg []byte) ([]byte, error) { ++ return evpEncrypt(pub.withKey, C.GO_RSA_NO_PADDING, nil, nil, msg) ++} ++ ++func SignRSAPSS(priv *PrivateKeyRSA, h crypto.Hash, hashed []byte, saltLen int) ([]byte, error) { ++ if saltLen == 0 { ++ saltLen = -1 // RSA_PSS_SALTLEN_DIGEST ++ } ++ return evpSign(priv.withKey, C.GO_RSA_PKCS1_PSS_PADDING, saltLen, h, hashed) ++} ++ ++func VerifyRSAPSS(pub *PublicKeyRSA, h crypto.Hash, hashed, sig []byte, saltLen int) error { ++ if saltLen == 0 { ++ saltLen = -2 // RSA_PSS_SALTLEN_AUTO ++ } ++ return evpVerify(pub.withKey, C.GO_RSA_PKCS1_PSS_PADDING, saltLen, h, sig, hashed) ++} ++ ++func SignRSAPKCS1v15(priv *PrivateKeyRSA, h crypto.Hash, hashed []byte) ([]byte, error) { ++ return evpSign(priv.withKey, C.GO_RSA_PKCS1_PADDING, 0, h, hashed) ++} ++ ++func VerifyRSAPKCS1v15(pub *PublicKeyRSA, h crypto.Hash, hashed, sig []byte) error { ++ if pub.withKey(func(pkey C.GO_EVP_PKEY_PTR) C.int { ++ size := C.go_openssl_EVP_PKEY_get_size(pkey) ++ if len(sig) < int(size) { ++ return 0 ++ } ++ return 1 ++ }) == 0 { ++ return errors.New("crypto/rsa: verification error") ++ } ++ return evpVerify(pub.withKey, C.GO_RSA_PKCS1_PADDING, 0, h, sig, hashed) ++} ++ ++// rsa_st_1_0_2 is rsa_st memory layout in OpenSSL 1.0.2. ++type rsa_st_1_0_2 struct { ++ _ C.int ++ _ C.long ++ _ [2]unsafe.Pointer ++ n, e, d C.GO_BIGNUM_PTR ++ p, q C.GO_BIGNUM_PTR ++ dmp1, dmq1, iqmp C.GO_BIGNUM_PTR ++ // It contains more fields, but we are not interesed on them. ++} ++ ++func bnSet(b1 *C.GO_BIGNUM_PTR, b2 *big.Int) { ++ if b2 == nil { ++ return ++ } ++ if *b1 != nil { ++ C.go_openssl_BN_clear_free(*b1) ++ } ++ *b1 = bigToBN(b2) ++} ++ ++func rsaSetKey(key C.GO_RSA_PTR, n, e, d *big.Int) bool { ++ if vMajor == 1 && vMinor == 0 { ++ r := (*rsa_st_1_0_2)(unsafe.Pointer(key)) ++ //r.d and d will be nil for public keys. ++ if (r.n == nil && n == nil) || ++ (r.e == nil && e == nil) { ++ return false ++ } ++ bnSet(&r.n, n) ++ bnSet(&r.e, e) ++ bnSet(&r.d, d) ++ return true ++ } ++ return C.go_openssl_RSA_set0_key(key, bigToBN(n), bigToBN(e), bigToBN(d)) == 1 ++} ++ ++func rsaSetFactors(key C.GO_RSA_PTR, p, q *big.Int) bool { ++ if vMajor == 1 && vMinor == 0 { ++ r := (*rsa_st_1_0_2)(unsafe.Pointer(key)) ++ if (r.p == nil && p == nil) || ++ (r.q == nil && q == nil) { ++ return false ++ } ++ bnSet(&r.p, p) ++ bnSet(&r.q, q) ++ return true ++ } ++ return C.go_openssl_RSA_set0_factors(key, bigToBN(p), bigToBN(q)) == 1 ++} ++ ++func rsaSetCRTParams(key C.GO_RSA_PTR, dmp1, dmq1, iqmp *big.Int) bool { ++ if vMajor == 1 && vMinor == 0 { ++ r := (*rsa_st_1_0_2)(unsafe.Pointer(key)) ++ if (r.dmp1 == nil && dmp1 == nil) || ++ (r.dmq1 == nil && dmq1 == nil) || ++ (r.iqmp == nil && iqmp == nil) { ++ return false ++ } ++ bnSet(&r.dmp1, dmp1) ++ bnSet(&r.dmq1, dmq1) ++ bnSet(&r.iqmp, iqmp) ++ return true ++ } ++ return C.go_openssl_RSA_set0_crt_params(key, bigToBN(dmp1), bigToBN(dmq1), bigToBN(iqmp)) == 1 ++} ++ ++func rsaGetKey(key C.GO_RSA_PTR) (*big.Int, *big.Int, *big.Int) { ++ var n, e, d C.GO_BIGNUM_PTR ++ if vMajor == 1 && vMinor == 0 { ++ r := (*rsa_st_1_0_2)(unsafe.Pointer(key)) ++ n, e, d = r.n, r.e, r.d ++ } else { ++ C.go_openssl_RSA_get0_key(key, &n, &e, &d) ++ } ++ return bnToBig(n), bnToBig(e), bnToBig(d) ++} ++ ++func rsaGetFactors(key C.GO_RSA_PTR) (*big.Int, *big.Int) { ++ var p, q C.GO_BIGNUM_PTR ++ if vMajor == 1 && vMinor == 0 { ++ r := (*rsa_st_1_0_2)(unsafe.Pointer(key)) ++ p, q = r.p, r.q ++ } else { ++ C.go_openssl_RSA_get0_factors(key, &p, &q) ++ } ++ return bnToBig(p), bnToBig(q) ++} ++ ++func rsaGetCRTParams(key C.GO_RSA_PTR) (*big.Int, *big.Int, *big.Int) { ++ var dmp1, dmq1, iqmp C.GO_BIGNUM_PTR ++ if vMajor == 1 && vMinor == 0 { ++ r := (*rsa_st_1_0_2)(unsafe.Pointer(key)) ++ dmp1, dmq1, iqmp = r.dmp1, r.dmq1, r.iqmp ++ } else { ++ C.go_openssl_RSA_get0_crt_params(key, &dmp1, &dmq1, &iqmp) ++ } ++ return bnToBig(dmp1), bnToBig(dmq1), bnToBig(iqmp) ++} +diff --git a/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/sha.go b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/sha.go +new file mode 100644 +index 00000000000000..c5b0189efc8132 +--- /dev/null ++++ b/src/vendor/github.com/microsoft/go-crypto-openssl/openssl/sha.go +@@ -0,0 +1,534 @@ ++// Copyright (c) Microsoft Corporation. ++// Licensed under the MIT License. ++ ++//go:build linux && !android ++// +build linux,!android ++ ++package openssl ++ ++// #include "goopenssl.h" ++import "C" ++import ( ++ "crypto" ++ "errors" ++ "hash" ++ "runtime" ++ "strconv" ++ "unsafe" ++) ++ ++type evpHash struct { ++ md C.GO_EVP_MD_PTR ++ ctx C.GO_EVP_MD_CTX_PTR ++ // ctx2 is used in evpHash.sum to avoid changing ++ // the state of ctx. Having it here allows reusing the ++ // same allocated object multiple times. ++ ctx2 C.GO_EVP_MD_CTX_PTR ++ size int ++ blockSize int ++} ++ ++func newEvpHash(ch crypto.Hash, size, blockSize int) *evpHash { ++ md := cryptoHashToMD(ch) ++ if md == nil { ++ panic("openssl: unsupported hash function: " + strconv.Itoa(int(ch))) ++ } ++ ctx := C.go_openssl_EVP_MD_CTX_new() ++ ctx2 := C.go_openssl_EVP_MD_CTX_new() ++ h := &evpHash{ ++ md: md, ++ ctx: ctx, ++ ctx2: ctx2, ++ size: size, ++ blockSize: blockSize, ++ } ++ runtime.SetFinalizer(h, (*evpHash).finalize) ++ h.Reset() ++ return h ++} ++ ++func (h *evpHash) finalize() { ++ C.go_openssl_EVP_MD_CTX_free(h.ctx) ++ C.go_openssl_EVP_MD_CTX_free(h.ctx2) ++} ++ ++func (h *evpHash) Reset() { ++ // There is no need to reset h.ctx2 because it is always reset after ++ // use in evpHash.sum. ++ C.go_openssl_EVP_MD_CTX_reset(h.ctx) ++ ++ if C.go_openssl_EVP_DigestInit_ex(h.ctx, h.md, nil) != 1 { ++ panic("openssl: EVP_DigestInit_ex failed") ++ } ++ runtime.KeepAlive(h) ++} ++ ++func (h *evpHash) Write(p []byte) (int, error) { ++ if len(p) > 0 && C.go_openssl_EVP_DigestUpdate(h.ctx, unsafe.Pointer(&p[0]), C.size_t(len(p))) != 1 { ++ panic("openssl: EVP_DigestUpdate failed") ++ } ++ runtime.KeepAlive(h) ++ return len(p), nil ++} ++ ++func (h *evpHash) Size() int { ++ return h.size ++} ++ ++func (h *evpHash) BlockSize() int { ++ return h.blockSize ++} ++ ++func (h *evpHash) sum(out []byte) { ++ // Make copy of context because Go hash.Hash mandates ++ // that Sum has no effect on the underlying stream. ++ // In particular it is OK to Sum, then Write more, then Sum again, ++ // and the second Sum acts as if the first didn't happen. ++ C.go_openssl_EVP_DigestInit_ex(h.ctx2, h.md, nil) ++ if C.go_openssl_EVP_MD_CTX_copy_ex(h.ctx2, h.ctx) != 1 { ++ panic("openssl: EVP_MD_CTX_copy_ex failed") ++ } ++ if C.go_openssl_EVP_DigestFinal_ex(h.ctx2, base(out), nil) != 1 { ++ panic("openssl: EVP_DigestFinal_ex failed") ++ } ++ C.go_openssl_EVP_MD_CTX_reset(h.ctx2) ++ runtime.KeepAlive(h) ++} ++ ++// shaState returns a pointer to the internal sha structure. ++// ++// The EVP_MD_CTX memory layout has changed in OpenSSL 3 ++// and the property holding the internal structure is no longer md_data but algctx. ++func (h *evpHash) shaState() unsafe.Pointer { ++ switch vMajor { ++ case 1: ++ // https://github.com/openssl/openssl/blob/0418e993c717a6863f206feaa40673a261de7395/crypto/evp/evp_local.h#L12. ++ type mdCtx struct { ++ _ [2]unsafe.Pointer ++ _ C.ulong ++ md_data unsafe.Pointer ++ } ++ return (*mdCtx)(unsafe.Pointer(h.ctx)).md_data ++ case 3: ++ // https://github.com/openssl/openssl/blob/5675a5aaf6a2e489022bcfc18330dae9263e598e/crypto/evp/evp_local.h#L16. ++ type mdCtx struct { ++ _ [3]unsafe.Pointer ++ _ C.ulong ++ _ [3]unsafe.Pointer ++ algctx unsafe.Pointer ++ } ++ return (*mdCtx)(unsafe.Pointer(h.ctx)).algctx ++ default: ++ panic(errUnsuportedVersion()) ++ } ++} ++ ++// NewSHA1 returns a new SHA1 hash. ++func NewSHA1() hash.Hash { ++ return &sha1Hash{ ++ evpHash: newEvpHash(crypto.SHA1, 20, 64), ++ } ++} ++ ++type sha1Hash struct { ++ *evpHash ++ out [20]byte ++} ++ ++func (h *sha1Hash) Sum(in []byte) []byte { ++ h.sum(h.out[:]) ++ return append(in, h.out[:]...) ++} ++ ++// sha1State layout is taken from ++// https://github.com/openssl/openssl/blob/0418e993c717a6863f206feaa40673a261de7395/include/openssl/sha.h#L34. ++type sha1State struct { ++ h [5]uint32 ++ nl, nh uint32 ++ x [64]byte ++ nx uint32 ++} ++ ++const ( ++ sha1Magic = "sha\x01" ++ sha1MarshaledSize = len(sha1Magic) + 5*4 + 64 + 8 ++) ++ ++func (h *sha1Hash) MarshalBinary() ([]byte, error) { ++ d := (*sha1State)(h.shaState()) ++ if d == nil { ++ return nil, errors.New("crypto/sha1: can't retrieve hash state") ++ } ++ b := make([]byte, 0, sha1MarshaledSize) ++ b = append(b, sha1Magic...) ++ b = appendUint32(b, d.h[0]) ++ b = appendUint32(b, d.h[1]) ++ b = appendUint32(b, d.h[2]) ++ b = appendUint32(b, d.h[3]) ++ b = appendUint32(b, d.h[4]) ++ b = append(b, d.x[:d.nx]...) ++ b = b[:len(b)+len(d.x)-int(d.nx)] // already zero ++ b = appendUint64(b, uint64(d.nl)>>3|uint64(d.nh)<<29) ++ return b, nil ++} ++ ++func (h *sha1Hash) UnmarshalBinary(b []byte) error { ++ if len(b) < len(sha1Magic) || string(b[:len(sha1Magic)]) != sha1Magic { ++ return errors.New("crypto/sha1: invalid hash state identifier") ++ } ++ if len(b) != sha1MarshaledSize { ++ return errors.New("crypto/sha1: invalid hash state size") ++ } ++ d := (*sha1State)(h.shaState()) ++ if d == nil { ++ return errors.New("crypto/sha1: can't retrieve hash state") ++ } ++ b = b[len(sha1Magic):] ++ b, d.h[0] = consumeUint32(b) ++ b, d.h[1] = consumeUint32(b) ++ b, d.h[2] = consumeUint32(b) ++ b, d.h[3] = consumeUint32(b) ++ b, d.h[4] = consumeUint32(b) ++ b = b[copy(d.x[:], b):] ++ _, n := consumeUint64(b) ++ d.nl = uint32(n << 3) ++ d.nh = uint32(n >> 29) ++ d.nx = uint32(n) % 64 ++ return nil ++} ++ ++// NewSHA224 returns a new SHA224 hash. ++func NewSHA224() hash.Hash { ++ return &sha224Hash{ ++ evpHash: newEvpHash(crypto.SHA224, 224/8, 64), ++ } ++} ++ ++type sha224Hash struct { ++ *evpHash ++ out [224 / 8]byte ++} ++ ++func (h *sha224Hash) Sum(in []byte) []byte { ++ h.sum(h.out[:]) ++ return append(in, h.out[:]...) ++} ++ ++// NewSHA256 returns a new SHA256 hash. ++func NewSHA256() hash.Hash { ++ return &sha256Hash{ ++ evpHash: newEvpHash(crypto.SHA256, 256/8, 64), ++ } ++} ++ ++type sha256Hash struct { ++ *evpHash ++ out [256 / 8]byte ++} ++ ++func (h *sha256Hash) Sum(in []byte) []byte { ++ h.sum(h.out[:]) ++ return append(in, h.out[:]...) ++} ++ ++const ( ++ magic224 = "sha\x02" ++ magic256 = "sha\x03" ++ marshaledSize256 = len(magic256) + 8*4 + 64 + 8 ++) ++ ++// sha256State layout is taken from ++// https://github.com/openssl/openssl/blob/0418e993c717a6863f206feaa40673a261de7395/include/openssl/sha.h#L51. ++type sha256State struct { ++ h [8]uint32 ++ nl, nh uint32 ++ x [64]byte ++ nx uint32 ++} ++ ++func (h *sha224Hash) MarshalBinary() ([]byte, error) { ++ d := (*sha256State)(h.shaState()) ++ if d == nil { ++ return nil, errors.New("crypto/sha256: can't retrieve hash state") ++ } ++ b := make([]byte, 0, marshaledSize256) ++ b = append(b, magic224...) ++ b = appendUint32(b, d.h[0]) ++ b = appendUint32(b, d.h[1]) ++ b = appendUint32(b, d.h[2]) ++ b = appendUint32(b, d.h[3]) ++ b = appendUint32(b, d.h[4]) ++ b = appendUint32(b, d.h[5]) ++ b = appendUint32(b, d.h[6]) ++ b = appendUint32(b, d.h[7]) ++ b = append(b, d.x[:d.nx]...) ++ b = b[:len(b)+len(d.x)-int(d.nx)] // already zero ++ b = appendUint64(b, uint64(d.nl)>>3|uint64(d.nh)<<29) ++ return b, nil ++} ++ ++func (h *sha256Hash) MarshalBinary() ([]byte, error) { ++ d := (*sha256State)(h.shaState()) ++ if d == nil { ++ return nil, errors.New("crypto/sha256: can't retrieve hash state") ++ } ++ b := make([]byte, 0, marshaledSize256) ++ b = append(b, magic256...) ++ b = appendUint32(b, d.h[0]) ++ b = appendUint32(b, d.h[1]) ++ b = appendUint32(b, d.h[2]) ++ b = appendUint32(b, d.h[3]) ++ b = appendUint32(b, d.h[4]) ++ b = appendUint32(b, d.h[5]) ++ b = appendUint32(b, d.h[6]) ++ b = appendUint32(b, d.h[7]) ++ b = append(b, d.x[:d.nx]...) ++ b = b[:len(b)+len(d.x)-int(d.nx)] // already zero ++ b = appendUint64(b, uint64(d.nl)>>3|uint64(d.nh)<<29) ++ return b, nil ++} ++ ++func (h *sha224Hash) UnmarshalBinary(b []byte) error { ++ if len(b) < len(magic224) || string(b[:len(magic224)]) != magic224 { ++ return errors.New("crypto/sha256: invalid hash state identifier") ++ } ++ if len(b) != marshaledSize256 { ++ return errors.New("crypto/sha256: invalid hash state size") ++ } ++ d := (*sha256State)(h.shaState()) ++ if d == nil { ++ return errors.New("crypto/sha256: can't retrieve hash state") ++ } ++ b = b[len(magic224):] ++ b, d.h[0] = consumeUint32(b) ++ b, d.h[1] = consumeUint32(b) ++ b, d.h[2] = consumeUint32(b) ++ b, d.h[3] = consumeUint32(b) ++ b, d.h[4] = consumeUint32(b) ++ b, d.h[5] = consumeUint32(b) ++ b, d.h[6] = consumeUint32(b) ++ b, d.h[7] = consumeUint32(b) ++ b = b[copy(d.x[:], b):] ++ _, n := consumeUint64(b) ++ d.nl = uint32(n << 3) ++ d.nh = uint32(n >> 29) ++ d.nx = uint32(n) % 64 ++ return nil ++} ++ ++func (h *sha256Hash) UnmarshalBinary(b []byte) error { ++ if len(b) < len(magic256) || string(b[:len(magic256)]) != magic256 { ++ return errors.New("crypto/sha256: invalid hash state identifier") ++ } ++ if len(b) != marshaledSize256 { ++ return errors.New("crypto/sha256: invalid hash state size") ++ } ++ d := (*sha256State)(h.shaState()) ++ if d == nil { ++ return errors.New("crypto/sha256: can't retrieve hash state") ++ } ++ b = b[len(magic256):] ++ b, d.h[0] = consumeUint32(b) ++ b, d.h[1] = consumeUint32(b) ++ b, d.h[2] = consumeUint32(b) ++ b, d.h[3] = consumeUint32(b) ++ b, d.h[4] = consumeUint32(b) ++ b, d.h[5] = consumeUint32(b) ++ b, d.h[6] = consumeUint32(b) ++ b, d.h[7] = consumeUint32(b) ++ b = b[copy(d.x[:], b):] ++ _, n := consumeUint64(b) ++ d.nl = uint32(n << 3) ++ d.nh = uint32(n >> 29) ++ d.nx = uint32(n) % 64 ++ return nil ++} ++ ++// NewSHA384 returns a new SHA384 hash. ++func NewSHA384() hash.Hash { ++ return &sha384Hash{ ++ evpHash: newEvpHash(crypto.SHA384, 384/8, 128), ++ } ++} ++ ++type sha384Hash struct { ++ *evpHash ++ out [384 / 8]byte ++} ++ ++func (h *sha384Hash) Sum(in []byte) []byte { ++ h.sum(h.out[:]) ++ return append(in, h.out[:]...) ++} ++ ++// NewSHA512 returns a new SHA512 hash. ++func NewSHA512() hash.Hash { ++ return &sha512Hash{ ++ evpHash: newEvpHash(crypto.SHA512, 512/8, 128), ++ } ++} ++ ++type sha512Hash struct { ++ *evpHash ++ out [512 / 8]byte ++} ++ ++func (h *sha512Hash) Sum(in []byte) []byte { ++ h.sum(h.out[:]) ++ return append(in, h.out[:]...) ++} ++ ++// sha256State layout is taken from ++// https://github.com/openssl/openssl/blob/0418e993c717a6863f206feaa40673a261de7395/include/openssl/sha.h#L95. ++type sha512State struct { ++ h [8]uint64 ++ nl, nh uint64 ++ x [128]byte ++ nx uint32 ++} ++ ++const ( ++ magic384 = "sha\x04" ++ magic512_224 = "sha\x05" ++ magic512_256 = "sha\x06" ++ magic512 = "sha\x07" ++ marshaledSize512 = len(magic512) + 8*8 + 128 + 8 ++) ++ ++func (h *sha384Hash) MarshalBinary() ([]byte, error) { ++ d := (*sha512State)(h.shaState()) ++ if d == nil { ++ return nil, errors.New("crypto/sha512: can't retrieve hash state") ++ } ++ b := make([]byte, 0, marshaledSize512) ++ b = append(b, magic384...) ++ b = appendUint64(b, d.h[0]) ++ b = appendUint64(b, d.h[1]) ++ b = appendUint64(b, d.h[2]) ++ b = appendUint64(b, d.h[3]) ++ b = appendUint64(b, d.h[4]) ++ b = appendUint64(b, d.h[5]) ++ b = appendUint64(b, d.h[6]) ++ b = appendUint64(b, d.h[7]) ++ b = append(b, d.x[:d.nx]...) ++ b = b[:len(b)+len(d.x)-int(d.nx)] // already zero ++ b = appendUint64(b, d.nl>>3|d.nh<<61) ++ return b, nil ++} ++ ++func (h *sha512Hash) MarshalBinary() ([]byte, error) { ++ d := (*sha512State)(h.shaState()) ++ if d == nil { ++ return nil, errors.New("crypto/sha512: can't retrieve hash state") ++ } ++ b := make([]byte, 0, marshaledSize512) ++ b = append(b, magic512...) ++ b = appendUint64(b, d.h[0]) ++ b = appendUint64(b, d.h[1]) ++ b = appendUint64(b, d.h[2]) ++ b = appendUint64(b, d.h[3]) ++ b = appendUint64(b, d.h[4]) ++ b = appendUint64(b, d.h[5]) ++ b = appendUint64(b, d.h[6]) ++ b = appendUint64(b, d.h[7]) ++ b = append(b, d.x[:d.nx]...) ++ b = b[:len(b)+len(d.x)-int(d.nx)] // already zero ++ b = appendUint64(b, d.nl>>3|d.nh<<61) ++ return b, nil ++} ++ ++func (h *sha384Hash) UnmarshalBinary(b []byte) error { ++ if len(b) < len(magic512) { ++ return errors.New("crypto/sha512: invalid hash state identifier") ++ } ++ if string(b[:len(magic384)]) != magic384 { ++ return errors.New("crypto/sha512: invalid hash state identifier") ++ } ++ if len(b) != marshaledSize512 { ++ return errors.New("crypto/sha512: invalid hash state size") ++ } ++ d := (*sha512State)(h.shaState()) ++ if d == nil { ++ return errors.New("crypto/sha512: can't retrieve hash state") ++ } ++ b = b[len(magic512):] ++ b, d.h[0] = consumeUint64(b) ++ b, d.h[1] = consumeUint64(b) ++ b, d.h[2] = consumeUint64(b) ++ b, d.h[3] = consumeUint64(b) ++ b, d.h[4] = consumeUint64(b) ++ b, d.h[5] = consumeUint64(b) ++ b, d.h[6] = consumeUint64(b) ++ b, d.h[7] = consumeUint64(b) ++ b = b[copy(d.x[:], b):] ++ _, n := consumeUint64(b) ++ d.nl = n << 3 ++ d.nh = n >> 61 ++ d.nx = uint32(n) % 128 ++ return nil ++} ++ ++func (h *sha512Hash) UnmarshalBinary(b []byte) error { ++ if len(b) < len(magic512) { ++ return errors.New("crypto/sha512: invalid hash state identifier") ++ } ++ if string(b[:len(magic512)]) != magic512 { ++ return errors.New("crypto/sha512: invalid hash state identifier") ++ } ++ if len(b) != marshaledSize512 { ++ return errors.New("crypto/sha512: invalid hash state size") ++ } ++ d := (*sha512State)(h.shaState()) ++ if d == nil { ++ return errors.New("crypto/sha512: can't retrieve hash state") ++ } ++ b = b[len(magic512):] ++ b, d.h[0] = consumeUint64(b) ++ b, d.h[1] = consumeUint64(b) ++ b, d.h[2] = consumeUint64(b) ++ b, d.h[3] = consumeUint64(b) ++ b, d.h[4] = consumeUint64(b) ++ b, d.h[5] = consumeUint64(b) ++ b, d.h[6] = consumeUint64(b) ++ b, d.h[7] = consumeUint64(b) ++ b = b[copy(d.x[:], b):] ++ _, n := consumeUint64(b) ++ d.nl = n << 3 ++ d.nh = n >> 61 ++ d.nx = uint32(n) % 128 ++ return nil ++} ++ ++// appendUint64 appends x into b as a big endian byte sequence. ++func appendUint64(b []byte, x uint64) []byte { ++ return append(b, ++ byte(x>>56), ++ byte(x>>48), ++ byte(x>>40), ++ byte(x>>32), ++ byte(x>>24), ++ byte(x>>16), ++ byte(x>>8), ++ byte(x), ++ ) ++} ++ ++// appendUint32 appends x into b as a big endian byte sequence. ++func appendUint32(b []byte, x uint32) []byte { ++ return append(b, byte(x>>24), byte(x>>16), byte(x>>8), byte(x)) ++} ++ ++// consumeUint64 reads a big endian uint64 number from b. ++func consumeUint64(b []byte) ([]byte, uint64) { ++ _ = b[7] ++ x := uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | ++ uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 ++ return b[8:], x ++} ++ ++// consumeUint32 reads a big endian uint32 number from b. ++func consumeUint32(b []byte) ([]byte, uint32) { ++ _ = b[3] ++ x := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 ++ return b[4:], x ++} +diff --git a/src/vendor/modules.txt b/src/vendor/modules.txt +index 4714b1d38605dc..64b77392bad61a 100644 +--- a/src/vendor/modules.txt ++++ b/src/vendor/modules.txt +@@ -1,3 +1,7 @@ ++# github.com/microsoft/go-crypto-openssl v0.1.2 ++## explicit; go 1.16 ++github.com/microsoft/go-crypto-openssl/openssl ++github.com/microsoft/go-crypto-openssl/openssl/internal/subtle + # golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 + ## explicit; go 1.17 + golang.org/x/crypto/chacha20 diff --git a/patches/0103-Adjust-Go-tests-to-work-with-crypto-module.patch b/patches/0103-Adjust-Go-tests-to-work-with-crypto-module.patch new file mode 100644 index 00000000000..e758a320d32 --- /dev/null +++ b/patches/0103-Adjust-Go-tests-to-work-with-crypto-module.patch @@ -0,0 +1,119 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: microsoft-golang-bot +Date: Thu, 31 Mar 2022 13:11:06 -0500 +Subject: [PATCH] Adjust Go tests to work with crypto module + +use gocrypt tag for cgo static tests +--- + misc/cgo/test/pkg_test.go | 4 +++- + src/cmd/dist/test.go | 6 ++++-- + src/cmd/go/testdata/script/gopath_std_vendor.txt | 9 ++++++++- + src/crypto/boring/boring_test.go | 1 + + src/go/build/deps_test.go | 13 ++++++++++--- + 5 files changed, 26 insertions(+), 7 deletions(-) + +diff --git a/misc/cgo/test/pkg_test.go b/misc/cgo/test/pkg_test.go +index 14013a4cd9..62c2e95b09 100644 +--- a/misc/cgo/test/pkg_test.go ++++ b/misc/cgo/test/pkg_test.go +@@ -50,7 +50,9 @@ func TestCrossPackageTests(t *testing.T) { + t.Fatal(err) + } + +- cmd := exec.Command("go", "test") ++ // Use build tag 'gocrypt' to avoid importing go-crypto-openssl, which makes TestSetgid hang. ++ // See go.dev/issues/52141. ++ cmd := exec.Command("go", "test", "-tags=gocrypt") + if testing.Verbose() { + cmd.Args = append(cmd.Args, "-v") + } +diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go +index f65d9bf980..d66f15d6fc 100644 +--- a/src/cmd/dist/test.go ++++ b/src/cmd/dist/test.go +@@ -1195,11 +1195,13 @@ func (t *tester) cgoTest(dt *distTest) error { + t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-ldflags", `-linkmode=external`) + if goos != "android" { + t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-ldflags", `-linkmode=external -extldflags "-static -pthread"`) +- t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`) ++ // Use build tag 'gocrypt' to avoid importing go-crypto-openssl, which makes TestSetgid hang. ++ // See go.dev/issues/52141. ++ t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static,gocrypt", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`) + // -static in CGO_LDFLAGS triggers a different code path + // than -static in -extldflags, so test both. + // See issue #16651. +- cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static") ++ cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static,gocrypt") + setEnv(cmd, "CGO_LDFLAGS", "-static -pthread") + } + } +diff --git a/src/cmd/go/testdata/script/gopath_std_vendor.txt b/src/cmd/go/testdata/script/gopath_std_vendor.txt +index a0a41a50de..2d28bd2cf9 100644 +--- a/src/cmd/go/testdata/script/gopath_std_vendor.txt ++++ b/src/cmd/go/testdata/script/gopath_std_vendor.txt +@@ -23,7 +23,14 @@ go list -deps -f '{{.ImportPath}} {{.Dir}}' . + stdout $GOPATH[/\\]src[/\\]vendor[/\\]golang.org[/\\]x[/\\]net[/\\]http2[/\\]hpack + ! stdout $GOROOT[/\\]src[/\\]vendor + +-go list -test -deps -f '{{.ImportPath}} {{.Dir}}' . ++# Use build tag 'gocrypt' while evaluating test dependencies to avoid importing ++# vendored crypto module dependencies like go-crypto-openssl. This test script ++# is not set up to handle any vendored libraries being imported other than ++# golang.org/x/net/http2/hpack, so we must make sure it is the only one. ++# ++# See https://github.com/microsoft/go/issues/481 for more details, such as the ++# dependency chain that would cause the failure if the gocrypt tag isn't used. ++go list -tags=gocrypt -test -deps -f '{{.ImportPath}} {{.Dir}}' . + stdout $GOPATH[/\\]src[/\\]vendor[/\\]golang.org[/\\]x[/\\]net[/\\]http2[/\\]hpack + ! stdout $GOROOT[/\\]src[/\\]vendor + +diff --git a/src/crypto/boring/boring_test.go b/src/crypto/boring/boring_test.go +index ace50de0c2..83ef05d872 100644 +--- a/src/crypto/boring/boring_test.go ++++ b/src/crypto/boring/boring_test.go +@@ -11,6 +11,7 @@ import ( + ) + + func TestEnabled(t *testing.T) { ++ t.Skip("upstream assumes boring is enabled at build time, we don't") + supportedPlatform := runtime.GOOS == "linux" && runtime.GOARCH == "amd64" + if supportedPlatform && !boring.Enabled() { + t.Error("Enabled returned false on a supported platform") +diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go +index ed8ddcb307..0b5dbd3727 100644 +--- a/src/go/build/deps_test.go ++++ b/src/go/build/deps_test.go +@@ -416,7 +416,14 @@ var depsRules = ` + < crypto/ed25519/internal/edwards25519 + < crypto/cipher + < encoding/asn1 +- < crypto/internal/boring ++ < CRYPTO; ++ ++ CRYPTO < crypto/internal/boring; ++ ++ CRYPTO ++ < github.com/microsoft/go-crypto-openssl/openssl/internal/subtle ++ < github.com/microsoft/go-crypto-openssl/openssl ++ < crypto/internal/backend + < crypto/aes, crypto/des, crypto/hmac, crypto/md5, crypto/rc4, + crypto/sha1, crypto/sha256, crypto/sha512 + < crypto/rand +@@ -446,7 +453,7 @@ var depsRules = ` + crypto/internal/boring/sig, crypto/internal/boring/fipstls + < crypto/tls/fipsonly; + +- crypto/internal/boring ++ crypto/internal/backend + < crypto/boring; + + # crypto-aware packages +@@ -646,7 +653,7 @@ var buildIgnore = []byte("\n//go:build ignore") + + func findImports(pkg string) ([]string, error) { + vpkg := pkg +- if strings.HasPrefix(pkg, "golang.org") { ++ if strings.HasPrefix(pkg, "golang.org") || strings.HasPrefix(pkg, "github.com") { + vpkg = "vendor/" + pkg + } + dir := filepath.Join(Default.GOROOT, "src", vpkg)