diff --git a/.azuredevops/dependabot.yml b/.azuredevops/dependabot.yml new file mode 100644 index 00000000000..a47d2fdb43b --- /dev/null +++ b/.azuredevops/dependabot.yml @@ -0,0 +1,5 @@ +version: 2 + +# Disabling dependabot on Azure DevOps as this is a mirrored repo. Updates should go through github. +enable-campaigned-updates: false +enable-security-updates: false diff --git a/.azuredevops/policies/branchClassification.yml b/.azuredevops/policies/branchClassification.yml new file mode 100644 index 00000000000..ac2bb61d472 --- /dev/null +++ b/.azuredevops/policies/branchClassification.yml @@ -0,0 +1,15 @@ +# Schema taken from https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/product-catalog/branch-classification/branch-classification#optional-update-branch-classification-at-the-repo. +name: branch_classification +description: Branch classification configuration for repository +resource: repository +disabled: false +where: +configuration: + branchClassificationSettings: + defaultClassification: nonproduction + ruleset: + - name: prod-branches + branchNames: + - microsoft/main + - microsoft/release-branch.go* + classification: production diff --git a/.config/guardian/.gdnsuppress b/.config/guardian/.gdnsuppress index 528d4b92fa3..46a67e1548a 100644 --- a/.config/guardian/.gdnsuppress +++ b/.config/guardian/.gdnsuppress @@ -1,8 +1,8 @@ { - "hydrated": false, + "hydrated": true, "properties": { "helpUri": "https://eng.ms/docs/microsoft-security/security/azure-security/cloudai-security-fundamentals-engineering/security-integration/guardian-wiki/microsoft-guardian/general/suppressions", - "hydrationStatus": "This file does not contain identifying data. It is safe to check into your repo. To hydrate this file with identifying data, run `guardian hydrate --help` and follow the guidance." + "hydrationStatus": "This file is hydrated. This is ok, it doesn't contain sensitive information." }, "version": "1.0.0", "suppressionSets": { @@ -13,13 +13,149 @@ } }, "results": { + "dd76b3defecd301787000102e3ce76506d45147b98fc4accb410b87097b2f0dd": { + "signature": "dd76b3defecd301787000102e3ce76506d45147b98fc4accb410b87097b2f0dd", + "alternativeSignatures": [], + "target": "go/src/crypto/ecdh/ecdh_test.go", + "line": 128, + "memberOf": [ + "default" + ], + "tool": "credscan", + "ruleId": "CSCAN-GENERAL0060", + "createdDate": "2024-12-16 10:13:25Z" + }, + "6b250bd6def22ac84e477b4d22ec0b12cb69721002a7fe0fccf23ff5a7dfa688": { + "signature": "6b250bd6def22ac84e477b4d22ec0b12cb69721002a7fe0fccf23ff5a7dfa688", + "alternativeSignatures": [], + "target": "go/src/crypto/ecdh/ecdh_test.go", + "line": 136, + "memberOf": [ + "default" + ], + "tool": "credscan", + "ruleId": "CSCAN-GENERAL0060", + "createdDate": "2024-12-16 10:13:25Z" + }, + "877868030fa2e6114057d685aea3f4ec90d3190dcfe0e0ae1d86a3a4094fad87": { + "signature": "877868030fa2e6114057d685aea3f4ec90d3190dcfe0e0ae1d86a3a4094fad87", + "alternativeSignatures": [], + "target": "go/src/crypto/ecdh/ecdh_test.go", + "line": 147, + "memberOf": [ + "default" + ], + "tool": "credscan", + "ruleId": "CSCAN-GENERAL0060", + "createdDate": "2024-12-16 10:13:25Z" + }, + "3f72337007e55d003a9f252112dbaead6ddf0f89bac847d528b44c263ddce0e1": { + "signature": "3f72337007e55d003a9f252112dbaead6ddf0f89bac847d528b44c263ddce0e1", + "alternativeSignatures": [], + "target": "go/src/crypto/ecdh/ecdh_test.go", + "line": 154, + "memberOf": [ + "default" + ], + "tool": "credscan", + "ruleId": "CSCAN-GENERAL0060", + "createdDate": "2024-12-16 10:13:25Z" + }, + "1ebc50c2b5fecaf320705d05137b2d29a3851823251d6224fc6223ca653b7c02": { + "signature": "1ebc50c2b5fecaf320705d05137b2d29a3851823251d6224fc6223ca653b7c02", + "alternativeSignatures": [], + "target": "go/src/crypto/tls/example_test.go", + "line": 165, + "memberOf": [ + "default" + ], + "tool": "credscan", + "ruleId": "CSCAN-GENERAL0020", + "createdDate": "2024-12-16 10:13:25Z" + }, + "3d4c2bb9f6a10eec92970320f48c0ee107981491a38c0869e054c54f156aafa1": { + "signature": "3d4c2bb9f6a10eec92970320f48c0ee107981491a38c0869e054c54f156aafa1", + "alternativeSignatures": [], + "target": "go/src/crypto/x509/platform_root_key.pem", + "line": 1, + "memberOf": [ + "default" + ], + "tool": "credscan", + "ruleId": "CSCAN-GENERAL0020", + "createdDate": "2024-12-16 10:13:25Z" + }, + "ebf235ebc38c8301d92931d07d3ba98a286fc46d53f24467f7d804c7d907a88b": { + "signature": "ebf235ebc38c8301d92931d07d3ba98a286fc46d53f24467f7d804c7d907a88b", + "alternativeSignatures": [], + "target": "go/src/crypto/tls/testdata/example-key.pem", + "line": 1, + "memberOf": [ + "default" + ], + "tool": "credscan", + "ruleId": "CSCAN-GENERAL0020", + "createdDate": "2024-12-16 10:13:25Z" + }, + "d00714f4abdecfa0f2b96d616a8631088ace81abf5f0688c05937dcf9cc4bb5e": { + "signature": "d00714f4abdecfa0f2b96d616a8631088ace81abf5f0688c05937dcf9cc4bb5e", + "alternativeSignatures": [], + "target": "go/src/cmd/vendor/rsc.io/markdown/emoji.go", + "line": 1432, + "memberOf": [ + "default" + ], + "tool": "credscan", + "ruleId": "CSCAN-GENERAL0060", + "createdDate": "2024-12-16 10:13:25Z" + }, "d5ce1218657b3f7da8e9a6ac55d72833387257e76e39d837edfd5c62781b9b97": { "signature": "d5ce1218657b3f7da8e9a6ac55d72833387257e76e39d837edfd5c62781b9b97", "alternativeSignatures": [], + "target": "go/src/crypto/internal/hpke/testdata/rfc9180-vectors.json", + "line": 1, + "memberOf": [ + "default" + ], + "tool": "credscan", + "ruleId": "CSCAN-GENERAL0060", + "createdDate": "2024-12-20 00:07:58Z" + }, + "e194d8614310e9f30653b5fb8ae34c07968131fce828c224c30536e1cb217e9e": { + "signature": "e194d8614310e9f30653b5fb8ae34c07968131fce828c224c30536e1cb217e9e", + "alternativeSignatures": [], + "target": "go/src/cmd/go/internal/auth/gitauth_test.go", + "line": 51, + "memberOf": [ + "default" + ], + "tool": "credscan", + "ruleId": "CSCAN-GENERAL0060", + "createdDate": "2024-12-20 19:29:53Z" + }, + "dd34d1fbdcab2b03eccc9e9544d25b6830f28ef5f72e815035df834a6b3e57e4": { + "signature": "dd34d1fbdcab2b03eccc9e9544d25b6830f28ef5f72e815035df834a6b3e57e4", + "alternativeSignatures": [], + "target": "go/src/cmd/go/testdata/script/goauth_git.txt", + "line": 72, + "memberOf": [ + "default" + ], + "tool": "credscan", + "ruleId": "CSCAN-GENERAL0060", + "createdDate": "2024-12-20 19:29:53Z" + }, + "2d264e899faa2c2dcf9c3c63acad2f0b0b7b9464fe1657d0313d8aa477103a3c": { + "signature": "2d264e899faa2c2dcf9c3c63acad2f0b0b7b9464fe1657d0313d8aa477103a3c", + "alternativeSignatures": [], + "target": "go/src/cmd/go/testdata/script/goauth_userauth.txt", + "line": 126, "memberOf": [ "default" ], - "createdDate": "2024-06-10 09:31:52Z" + "tool": "credscan", + "ruleId": "CSCAN-GENERAL0120", + "createdDate": "2024-12-20 19:29:53Z" } } } \ No newline at end of file diff --git a/.config/tsa/tsaoptions.json b/.config/tsa/tsaoptions.json index c67efa26b34..8a0660b7893 100644 --- a/.config/tsa/tsaoptions.json +++ b/.config/tsa/tsaoptions.json @@ -8,7 +8,7 @@ ], "instanceUrl": "https://devdiv.visualstudio.com/", "projectName": "DEVDIV", - "areaPath": "DevDiv\\NET Compilers\\GoLang", + "areaPath": "DevDiv\\GoLang", "iterationPath": "DevDiv", "allTools": true } diff --git a/.git-go-patch b/.git-go-patch new file mode 100644 index 00000000000..74a8f15c009 --- /dev/null +++ b/.git-go-patch @@ -0,0 +1,7 @@ +{ + "MinimumToolVersion": "v1.0.1", + "SubmoduleDir": "go", + "PatchesDir": "patches", + "StatusFileDir": "eng/artifacts/go-patch", + "ExtractAsAuthor": "bot-for-go[bot] <199222863+bot-for-go[bot]@users.noreply.github.com>" +} diff --git a/.gitattributes b/.gitattributes index d8984ea762b..5a14080823e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,9 @@ +# Default: normalize text files to LF in the repo and on checkout. +* text=auto eol=lf + # On Windows, running "git apply" with CRLF patch files causes an error for binary files: # "git diff header lacks filename information when removing 1 leading pathname component". # To fix this, always check out patch files as LF. *.patch -text + +.github/workflows/*.lock.yml linguist-generated=true merge=ours diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bf3c89b27bf..b5dd7187f37 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,8 +1,9 @@ -# Require review from golang-compiler team for changes in any file. This keeps us in the loop on -# auto-merge PRs. The review bot is also an owner so that it can still trigger auto-merge for sync -# PRs on its own. We may remove this rule once auto-merges are routine. -* @microsoft/golang-compiler @microsoft-golang-review-bot +# Require review from golang-compiler team for changes in all files. -# Automatically request review from golang-compiler team for changes in the Microsoft-specific -# files. This takes precedence over earlier rules in the file. -/eng/ @microsoft/golang-compiler +* @microsoft/golang-compiler + +# Don't assign a code owner to automatically updated files to allow GitHub apps to use the +# auto-merge flow without human intervention. +/go +/MICROSOFT_REVISION +/VERSION diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000000..c42793e13e3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,103 @@ +# https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/configuring-issue-templates-for-your-repository#creating-issue-forms +# https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-githubs-form-schema +name: Bugs +description: Report a bug with the Microsoft build of Go +title: "issue title" + +body: + - type: markdown + attributes: + value: | + Thanks for helping us improve! πŸ™ Please answer these questions and provide as much information as possible about your problem. + + - type: input + id: go-version + attributes: + label: Microsoft build of Go version + description: | + What version of Go are you using (`go version`)? + + Note: we only [support](https://github.com/microsoft/go/blob/microsoft/main/eng/doc/Downloads.md) the two most recent major releases. + placeholder: ex. go version go1.24.1 darwin/arm64 + validations: + required: true + + - type: textarea + id: platform + attributes: + label: What is your operating system and platform? + description: Please add as much information here as possible. For example, if you have a problem related to platform-provided crypto (`systemcrypto`), your Linux distro+version or specific version of Windows may be necessary to reproduce it. + placeholder: e.g. "Ubuntu 24.04 on ARM64" or "Windows 10 2004 on x86-64" + validations: + required: true + + - type: textarea + id: go-env + attributes: + label: "Output of `go env` in your module/workspace:" + placeholder: | + GO111MODULE="" + GOARCH="arm64" + GOBIN="/Users/gopher/go/bin" + GOCACHE="/Users/gopher/go/cache" + GOENV="/Users/gopher/Library/Application Support/go/env" + GOEXE="" + GOEXPERIMENT="" + GOFLAGS="" + GOHOSTARCH="arm64" + GOHOSTOS="darwin" + GOINSECURE="" + GOMODCACHE="/Users/gopher/go/pkg/mod" + GONOPROXY="" + GONOSUMDB="" + GOOS="darwin" + GOPATH="/Users/gopher/go" + GOPRIVATE="" + GOPROXY="https://proxy.golang.org,direct" + GOROOT="/usr/local/go" + GOSUMDB="sum.golang.org" + GOTMPDIR="" + GOTOOLDIR="/usr/local/go/pkg/tool/darwin_arm64" + GOVCS="" + GOVERSION="go1.20.7" + GCCGO="gccgo" + AR="ar" + CC="clang" + CXX="clang++" + CGO_ENABLED="1" + GOMOD="/dev/null" + GOWORK="" + CGO_CFLAGS="-O2 -g" + CGO_CPPFLAGS="" + CGO_CXXFLAGS="-O2 -g" + CGO_FFLAGS="-O2 -g" + CGO_LDFLAGS="-O2 -g" + PKG_CONFIG="pkg-config" + GOGCCFLAGS="-fPIC -arch arm64 -pthread -fno-caret-diagnostics -Qunused-arguments -fmessage-length=0 -fdebug-prefix-map=/var/folders/44/nbbyll_10jd0z8rj_qxm43740000gn/T/go-build2331607515=/tmp/go-build -gno-record-gcc-switches -fno-common" + render: shell + validations: + required: true + + - type: textarea + id: what-did-you-do + attributes: + label: "What did you do?" + description: "If possible, provide a recipe for reproducing the error. A complete runnable program is good." + validations: + required: true + + - type: textarea + id: actual-behavior + attributes: + label: "What did you see happen?" + description: Command invocations and their associated output, functions with their arguments and return results, full stacktraces for panics (upload a file if it is very long), etc. Prefer copying text output over using screenshots. + validations: + required: true + + - type: textarea + id: expected-behavior + attributes: + label: "What did you expect to see?" + description: Why is the current output incorrect, and any additional context we may need to understand the issue. + validations: + required: true diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json new file mode 100644 index 00000000000..f4f8324579e --- /dev/null +++ b/.github/aw/actions-lock.json @@ -0,0 +1,14 @@ +{ + "entries": { + "actions/github-script@v8": { + "repo": "actions/github-script", + "version": "v8", + "sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd" + }, + "github/gh-aw/actions/setup@v0.50.2": { + "repo": "github/gh-aw/actions/setup", + "version": "v0.50.2", + "sha": "e32435511ac2c5aa0e08b19284a25dc98fadf1e1" + } + } +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3979801675d..40eee4603e3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,11 +9,6 @@ updates: schedule: interval: daily open-pull-requests-limit: 10 -- package-ecosystem: gomod - directory: "eng/_core" - schedule: - interval: daily - open-pull-requests-limit: 10 - package-ecosystem: gomod directory: "eng/_util" schedule: diff --git a/.github/instructions/patch-consistency.instructions.md b/.github/instructions/patch-consistency.instructions.md new file mode 100644 index 00000000000..11b2f94a69b --- /dev/null +++ b/.github/instructions/patch-consistency.instructions.md @@ -0,0 +1,55 @@ +--- +description: "Use when reviewing, editing, or creating Git patch files in the patches/ directory. Covers patch consistency rules, vendor constraints, naming conventions, and Go file build tag guidelines. Apply during code review and pull request review of patch changes." +applyTo: "patches/*.patch" +--- + +# Patch consistency review + +## Project context + +The `patches/` directory contains `git format-patch` files that are applied on top of the upstream Go source tree, which is in the `go/` submodule. Each patch is prefixed with a sequence number and a short description of its purpose, e.g. `0001-Vendor-external-dependencies.patch`, `0004-Use-crypto-backends.patch`. The patches are applied in order, and each one should be self-contained to a single logical concern and describe that concern in the commit message. + +## Rules to check + +### Vendor changes must be in the vendor patch + +If the diff adds, modifies, or removes any file under a `vendor/` directory path, verify: +- The change is in `0001-Vendor-external-dependencies.patch` only. No other patch should touch vendor files +- The corresponding `go.mod`, `go.sum`, and `modules.txt` changes are included in the same patch +- The vendor patch doesn't include changes other than `vendor/` files, related module files, and `src/crypto/internal/backend/deps_ignore.go` + +### Patch naming convention + +New or renamed patch files must follow the `NNNN-Short-Description.patch` pattern (zero-padded four-digit prefix, hyphen-separated description) consistent with existing patches in the directory. + +### No redundant or misplaced changes + +- A new patch should not duplicate changes already covered by an existing patch +- Changes should be in the patch whose description matches the concern. For example, crypto backend changes belong in the crypto backend patch, not in the vendor patch or a new unrelated patch +- If an existing patch already covers the area being changed, the change should amend that patch rather than create a new one + +### Go file OS/arch constraints + +When a patch adds or modifies a Go file that targets a specific OS or architecture, prefer the filename suffix convention (`_linux.go`, `_windows.go`, `_darwin.go`) over adding OS/arch constraints in `//go:build` tags. If the filename suffix already implies the OS/arch, the build tag should not repeat it. + +- Preferred: `foo_linux.go` with `//go:build goexperiment.systemcrypto` +- Avoid: `foo.go` with `//go:build goexperiment.systemcrypto && linux` + +### No extraneous blank lines + +Patches should not introduce unnecessary blank lines between functions, at the end of files, or between import groups. Whitespace should be consistent with the surrounding code in the Go source tree. + +## How to review + +1. **Identify which patch(es) changed**: Check the PR's modified files for `patches/*.patch` +2. **Analyze the intent**: Read the patch diff to understand what feature or fix is being implemented +3. **Cross-reference other patches**: Check if other patches in the directory touch the same source files or implement related functionality. Compare method signatures, behavior, and documentation for consistency +4. **Flag inconsistencies**: Be specific about which patches need updates and what changes would bring them into alignment +5. **If no issues**: Include the message "Patches are happy!" in the review conclusion if no consistency issues are found. + +## Review tone + +- Frame feedback as suggestions for maintaining consistency, not demands +- Focus on consistency across the patch set, not general code quality +- Skip trivial differences like comment styles or variable naming +- Only comment when there are actual consistency issues to flag diff --git a/.github/workflows/check-line-endings.yml b/.github/workflows/check-line-endings.yml new file mode 100644 index 00000000000..faa5f00a596 --- /dev/null +++ b/.github/workflows/check-line-endings.yml @@ -0,0 +1,16 @@ +name: Check Line Endings + +on: + push: + branches: + - "microsoft/*" + pull_request: + branches: + - "microsoft/*" + +permissions: + contents: read + +jobs: + check-line-endings: + uses: microsoft/go-infra/.github/workflows/check-line-endings.yml@eae52708dd530100eb695850bfc7f994dd9340d3 # v0.0.14 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 408662547a2..334cc5409fa 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -18,27 +18,21 @@ jobs: permissions: security-events: write - strategy: - fail-fast: false - matrix: - language: - - 'cpp' - - 'go' - steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0 with: - languages: ${{ matrix.language }} - build-mode: manual + languages: go - - run: pwsh eng/run.ps1 submodule-refresh -shallow - - run: pwsh eng/run.ps1 build + - name: Autobuild + uses: github/codeql-action/autobuild@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0 + with: + working-directory: eng/_util - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0 with: - category: /language:${{matrix.language}} \ No newline at end of file + category: /language:go diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 00000000000..3a5bbf562b9 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,22 @@ +name: golangci-lint +on: + push: + branches: + - "microsoft/*" + pull_request: + branches: + - "microsoft/*" + +permissions: + contents: read + pull-requests: read + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Run golangci-lint + working-directory: eng/_util + run: docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v2.10 golangci-lint run -v diff --git a/.github/workflows/patch-build.yml b/.github/workflows/patch-build.yml new file mode 100644 index 00000000000..4c947c0a949 --- /dev/null +++ b/.github/workflows/patch-build.yml @@ -0,0 +1,63 @@ +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This job tests that each patch file is buildable (in numerical order) + +name: "Patch Build" + +on: + pull_request: + branches: [ microsoft/* ] + +# Cancel existing runs if user makes another push. +concurrency: + group: "${{ github.ref }}-${{ github.workflow}}" + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + list_patches: + name: Generate patch build matrix + runs-on: ubuntu-latest + outputs: + patches: ${{ steps.list.outputs.patches }} + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Write patch matrix + id: list + run: pwsh eng/run.ps1 write-patch-matrix -github-actions + + build_patches: + name: "Build up to ${{ matrix.patch.name }}" + needs: list_patches + runs-on: ubuntu-latest + strategy: + matrix: + patch: ${{ fromJson(needs.list_patches.outputs.patches) }} + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: true + + - name: Set mock git config name/email + run: | + git config --global user.email "joe@blogs.com" + git config --global user.name "Joe Blogs" + + - name: Apply patches 1-${{ matrix.patch.number }} + run: pwsh eng/run.ps1 submodule-refresh -commits -take ${{ matrix.patch.number }} + + - name: Build + run: | + set -x + pwsh eng/run.ps1 build + cd ${{ github.workspace }}/go/src + ${{ github.workspace }}/go/bin/go mod vendor + cd ${{ github.workspace }}/go/src/cmd + ${{ github.workspace }}/go/bin/go mod vendor + cd ${{ github.workspace }}/go/src + # Check if the vendor directory is clean + git diff --exit-code vendor cmd/vendor || (echo "Vendor directories are not clean. Please run 'go mod vendor' in the appropriate directories and commit the changes." && exit 1) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 922fae84320..7682d7f3018 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,18 +11,25 @@ # time to hit the patch failure N times. However, the actual tests run in AzDO, so we can't # reasonably cancel them from here (GitHub Actions). -name: "Test" +# The "0:" prefix makes this workflow's checks sort above the alphabetical AzDO stage names on the +# PR checks list, so patch failures are immediately visible without scrolling. +name: "0: Test" on: pull_request: branches: [ microsoft/* ] +# Cancel existing runs if user makes another push. +concurrency: + group: "${{ github.ref }}-${{ github.workflow}}" + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: check_patches: name: Patches Apply Cleanly runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: pwsh eng/run.ps1 submodule-refresh -shallow diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..505c905fa0a --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +microsoft-go.sln diff --git a/Custom b/Custom new file mode 100644 index 00000000000..cb5c0e26129 --- /dev/null +++ b/Custom @@ -0,0 +1,32 @@ + + +/ +.metadata/ +.recommenders/ + +/.idea/ +/.project/ +/.settings/ +/compile_commands.json +/.cache +/.gdbinit +/.lldbinit + + +/.vscode/ +/nbproject/ +nbproject/private/ +/webrev +/.src-rev +/.jib/ + + + + + + +/src/hotspot/CMakeLists.txt/. +/src/hotspot/compile_commands.json/. +/src/hotspot/cmake-build-debug/. +/src/hotspot/.cache/. +/src/hotspot/.idea/. diff --git a/README.md b/README.md index 2e827154062..a8226310f2e 100644 --- a/README.md +++ b/README.md @@ -1,126 +1,84 @@ -# The Go Programming Language +# The Microsoft build of Go -Go is an open source programming language that makes it easy to build simple, -reliable, and efficient software. +Go is an open source programming language that makes it easy to build simple, reliable, and efficient software. +The Microsoft build of Go, maintained in the [microsoft/go repository](https://github.com/microsoft/go), contains the infrastructure Microsoft uses to build a modified version of the Go toolset. -This repository, [microsoft/go](https://github.com/microsoft/go), contains the -infrastructure Microsoft uses to build Go. The submodule named `go` contains the -Go source code. By default, the submodule's remote URL is the official GitHub -mirror of Go, [golang/go](https://github.com/golang/go). The canonical Git -repository for Go source code is located at https://go.googlesource.com/go. +Unless otherwise noted, the Go source files are distributed under the BSD-style license found in the LICENSE file. -This project is not involved in producing the [official binary distributions -of Go](https://go.dev/dl/). - -Unless otherwise noted, the Go source files are distributed under the -BSD-style license found in the LICENSE file. - -If you are using this fork and have a Microsoft corporate account, consider -[clicking here to instantly join the Microsoft Go Toolset Announcements email distribution list πŸ“§](https://idwebelements.microsoft.com/GroupManagement.aspx?Group=golang-announce&Operation=join) -and receive notifications about Microsoft releases of Go and breaking changes. -We also maintain an [internal doc page](https://eng.ms/docs/more/languages-at-microsoft/go/articles/overview). +This project is not involved in producing the [official binary distributions of Go](https://go.dev/dl/). ## Why does this fork exist? -This repository produces a modified version of Go that can be used to build FIPS -140-2 compliant applications. Our goal is to share this implementation with -others in the Go community who have the same requirement, and to merge this -capability into upstream Go as soon as possible. See -[eng/doc/fips](eng/doc/fips) for more information about this feature and the -history of FIPS 140-2 compliance in Go. - -The binaries produced by this repository are also intended for general use -within Microsoft instead of the official binary distribution of Go. - -We call this repository a fork even though it isn't a traditional Git fork. Its -branches do not share Git ancestry with the Go repository. However, the -repository serves the same purpose as a Git fork: maintaining a modified version -of the Go source code over time. - -## Support - -This project follows the upstream Go -[Release Policy](https://go.dev/doc/devel/release#policy). -This means we support each major release (1.X) until there are two newer major -releases. A new Go major version is -[released every six months](https://github.com/golang/go/wiki/Go-Release-Cycle), -so each Go major version is supported for about one year. - -When upstream Go releases a new minor version (1.X.Y), we release a -corresponding microsoft/go version that may also include fork-specific changes. -This normally happens once a month. At any time, we may release a new revision -(1.X.Y-Z) to fix an issue without waiting for the next upstream minor release. -Revision releases are uncommon. - -Each microsoft/go release is announced in -[a Microsoft-internal email distribution list πŸ“§](https://idwebelements.microsoft.com/GroupManagement.aspx?Group=golang-announce&Operation=join) and the [Microsoft for Go Developers](https://devblogs.microsoft.com/go/) blog. - -## Download and install +This repository produces a modified version of Go that: -We build the forked Go toolset with this list of OS/Arch combinations. To use a -prebuilt copy of Go while targeting a platform that is not on this list, -cross-compilation may be necessary. +* Builds programs that are compliant with internal Microsoft policies by default. +* Can be used to build FIPS 140 compliant applications. + * See [eng/doc/fips](eng/doc/fips) for more information about this feature and the history of FIPS 140 compliance in Go. -* `linux_amd64` -* `linux_armv6l` -* `linux_arm64` -* `windows_amd64` +For a complete summary of the changes we make, see [the "What's different?" section of the Migration Guide](eng/doc/MigrationGuide.md#whats-different). -The following sections list the ways to get a build of the Microsoft fork of Go. +We submit changes to the upstream Go project rather than patching it, when +possible. Our goals are to avoid breaking compatibility and to minimize the +number of changes we maintain in this fork. -> [!NOTE] -> Don't see an option that works for you? Let us know! -> File a GitHub issue, or comment on an existing issue in this tag: - [![](https://img.shields.io/github/labels/microsoft/go/Area-Acquisition)](https://github.com/microsoft/go/labels/Area-Acquisition) +We call this project a fork even though it isn't a traditional Git fork: the Git branches don't share ancestry with the upstream Git repository. +However, the repository serves the same purpose as a Git fork: to maintain a modified version of the Go source code over time. +The submodule named `go` contains the Go source code, and the `patches` directory contains our changes. +The submodule is updated regularly to the latest commit available in both the upstream repository, , and the GitHub mirror, . -### Docker Container Images +## Support -**[microsoft/go-images](https://github.com/microsoft/go-images)** maintains and -documents container images that are available on Microsoft Artifact Registry. +See [SUPPORT.md](SUPPORT.md) for more information about reporting bugs, requesting features, and asking questions. -### Azure Linux +There are a few additional support resources internal to Microsoft: -The **[Azure Linux](https://github.com/microsoft/azurelinux)** distribution -includes builds of this Go fork. +* [(Microsoft-internal) Languages at Microsoft: Introduction to Go](https://eng.ms/docs/more/languages-at-microsoft/go/articles/overview). +* [(Microsoft-internal) Languages at Microsoft: Get Help with Go](https://eng.ms/docs/more/languages-at-microsoft/go/articles/support). + * Includes internal Microsoft support channels such as an email contact for our team and a community Teams group. -* In Azure Linux 2.0, the package `msft-golang` installs this fork. -* In Azure Linux 3.0, the `golang` package installs this fork. +## Release cycle and policy -### Binary archive +This project follows the upstream Go [Release Policy](https://go.dev/doc/devel/release#policy). +This means we support each major release (1.X) until there are two newer major releases. +A new Go major version is [released every six months](https://github.com/golang/go/wiki/Go-Release-Cycle), so each Go major version is supported for about one year. -[Completed builds of Go](https://github.com/microsoft/go/blob/microsoft/main/eng/doc/Downloads.md) -for several platforms are available as `zip` and `tar.gz` files. +When upstream Go releases a new minor version (1.X.Y), we release a corresponding microsoft/go version that may also include fork-specific changes. +This normally happens once a month. -### Build from source +At any time, we may release a new revision (1.X.Y-Z) to fix an issue without waiting for the next upstream minor release. +Revision releases are uncommon. -#### Pre-patched source tarball +We announce each Microsoft build of Go release through the following channels: -[The microsoft/go GitHub releases](https://github.com/microsoft/go/releases) -include a source tarball file ending in `.src.tar.gz`. After downloading and -extracting the tar.gz file, build it using the -[upstream instructions](https://go.dev/doc/install/source). +* [Microsoft for Go Developers blog](https://devblogs.microsoft.com/go/). +* [Microsoft-internal email distribution list πŸ“§ (instant join link)](https://idwebelements.microsoft.com/GroupManagement.aspx?Group=golang-announce&Operation=join). -#### Clone and build +The Go team announces upstream releases on the [golang-announce mailing list](https://groups.google.com/g/golang-announce). +These announcement emails include a carefully written summary of the changes that may not be found elsewhere. -This repository wraps the upstream Go repository and includes build scripts that -automate some aspects of the build process. See [eng/README.md](eng/README.md) -for more details about the infrastructure. +## Download and install -Prerequisites: +We build the Microsoft build of Go toolset with the following OS/Arch combinations: -* [PowerShell 6+](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell) -* [Go install from source prerequisites](https://go.dev/doc/install/source) - * Exception: this repository's build script automatically downloads a - bootstrap version of Go. +| OS | `amd64` | `arm64` | `armv6l` | +| --- | :---: | :---: | :---: | +| `linux` | βœ“ | βœ“ | βœ“ | +| `windows` | βœ“ | βœ“ | | +| `darwin` (macOS) | βœ“ | βœ“ | | -After cloning the repository and checking out the desired tag or commit, use the -following build command: +Visit the [Migration Guide](eng/doc/MigrationGuide.md) for guidance about how we recommend migrating existing Go projects to use the Microsoft build of Go. +This guide also helps resolve commonly encountered issues. -``` -pwsh eng/run.ps1 build -refresh -``` +The [Installation](eng/doc/Installation.md) documentation contains sections describing each of the following installation methods: -The resulting Go binary can then be found at `go/bin/go`. +* [Docker Container Images](eng/doc/Installation.md#docker-container-images) +* [Azure Linux](eng/doc/Installation.md#azure-linux) +* [Ubuntu](eng/doc/Installation.md#ubuntu) +* [Azure Pipelines `GoTool@0` task](eng/doc/Installation.md#azure-pipelines-gotool0-task) +* [GitHub Actions `setup-go` action](eng/doc/Installation.md#github-actions-setup-go-action) +* [The `go-install.ps1` script](eng/doc/Installation.md#the-go-installps1-script) +* [Binary archive](eng/doc/Installation.md#binary-archive) (`tar.gz` and `zip`) +* [Build from source](eng/doc/Installation.md#build-from-source) ## Contributing @@ -136,6 +94,8 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. +Please read the [Developer Guide](eng/doc/DeveloperGuide.md) for more information about contributing to this project. + ## Trademarks This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft @@ -143,3 +103,18 @@ trademarks or logos is subject to and must follow [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. Any use of third-party trademarks or logos are subject to those third-party's policies. + +## Data Collection + +The software may collect information about you and your use of the software and +send it to Microsoft. Microsoft may use this information to provide services +and improve our products and services. You may turn off the telemetry by +setting the `MS_GOTOOLCHAIN_TELEMETRY_ENABLED` environment variable to `0`. +There are also some features in the software that may enable you and Microsoft +to collect data from users of your applications. If you use these features, +you must comply with applicable law, including providing appropriate notices to +users of your applications together with a copy of Microsoft’s privacy +statement. Our privacy statement is located at https://go.microsoft.com/fwlink/?LinkID=824704. +You can learn more about data collection and use in the help documentation and +our privacy statement. Your use of the software operates as your consent to +these practices. diff --git a/SUPPORT.md b/SUPPORT.md index 4a664139f0d..a5c8aaa4b58 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -1,17 +1,24 @@ -# Support - -For help and questions about the Go programming language and tools, visit the official website: . - -## How to file issues and get help - -This project uses GitHub Issues to track bugs and feature requests. Please search the existing -issues before filing new issues to avoid duplicates. For new issues, [file your bug or -feature request as a new Issue][file]. - -For help and questions about the microsoft/go project, please [file an issue][file]. - -## Microsoft Support Policy - -Support for this project is limited to the resources listed above. - -[file]: https://github.com/microsoft/go/issues/new/choose +# Support + +For help and questions about the Go programming language and tools, visit the official website: . + +Take a look at the [Migration Guide](eng/doc/MigrationGuide.md) for more information about migrating from the official build of Go to the Microsoft build of Go, specifically. + +> [!TIP] +> Additional support options internal to Microsoft are listed in the ["Support" section of the README file](README.md#support). + +## How to file issues and get help + +This project uses GitHub Issues to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, [file your bug or +feature request as a new Issue][file]. + +For help and questions about the microsoft/go project, please [file an issue][file]. + +If you believe you have found a security vulnerability, please report it privately following the instructions in [SECURITY.md](SECURITY.md). + +## Microsoft Support Policy + +Support for this project is limited to the resources listed above. + +[file]: https://github.com/microsoft/go/issues/new/choose diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000000..d57f9a61fa7 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,7 @@ +# Release Notes + +This directory contains release notes for the current branch's major version of the Microsoft build of Go. +Each branch contains the final release notes for that major version and notes for minor releases of that major version, if any have been written. + +> [!IMPORTANT] +> General docs for the Microsoft build of Go are in the [root README](/README.md), and additional docs are [in `eng/doc`](/eng/doc). diff --git a/docs/go1.23.md b/docs/go1.23.md deleted file mode 100644 index d9b9f56fa98..00000000000 --- a/docs/go1.23.md +++ /dev/null @@ -1,3 +0,0 @@ -# Microsoft Go 1.23 release notes - -After the release of 1.23, 1.21 is no longer supported, per the [Go release policy](https://go.dev/doc/devel/release). diff --git a/docs/go1.27.md b/docs/go1.27.md new file mode 100644 index 00000000000..03f2881c71c --- /dev/null +++ b/docs/go1.27.md @@ -0,0 +1,3 @@ +# Microsoft build of Go 1.27 release notes + +After the release of 1.27, 1.25 is no longer supported, per the [Go release policy](https://go.dev/doc/devel/release). diff --git a/eng/README.md b/eng/README.md index b9982ebc896..9a8786749f2 100644 --- a/eng/README.md +++ b/eng/README.md @@ -23,6 +23,14 @@ submodule to new Go commits. ## Building Go +Prerequisites: + +* [Git](https://git-scm.com/downloads) +* [PowerShell 6+](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell) +* [Go install from source prerequisites](https://go.dev/doc/install/source) + * Exception: this repository's build script automatically downloads a + bootstrap version of Go. + In the root of the repository, run this command: ```pwsh @@ -33,8 +41,10 @@ pwsh eng/run.ps1 build -refresh patches) before the command builds the repository. Remove `-refresh` if you've made changes in the submodule (`go`) that you want to keep. * Add `-test` to run tests after the build completes. -* Add `-pack` to create an archive file containing the Go build in +* Add `-packbuild` to create an archive file containing the Go build in `eng/artifacts/bin`. (A `.tar.gz` or `.zip` file, depending on GOOS) +* Add `-packsource` to create a `.tar.gz` file containing the Go sources in + `eng/artifacts/bin`. Run this command for more information: @@ -54,14 +64,31 @@ complete, to match the content of the official binary releases of Go. ## Patch files -The Microsoft Go repository uses patch files to apply changes to the `go` -submodule. The patch files are found in [`/patches`](/patches). The `-refresh` -argument to the `build` tool applies patches. Or, try: +The Microsoft build of Go repository uses patch files to store changes to the `go` +submodule. The patch files are found in [`/patches`](/patches). -``` -pwsh eng/run.ps1 submodule-refresh -h -``` +We created [the `git-go-patch` tool][git-go-patch] to develop and maintain the +patch files. We wrote this tool specifically for the Microsoft for Go developers project. +It's a Go program that can be invoked as `git go-patch` after it's installed. See +[the `git-go-patch` readme][git-go-patch] for more information. + +We also have some utilities in this repository to apply patches without +installing `git-go-patch`: + +* `pwsh eng/run.ps1 submodule-refresh` updates the submodule and applies the + patches. + * Pass `-commits` to apply each patch as a separate commit. +* `pwsh eng/run.ps1 build -refresh` refreshes the submodule and applies patches + and then goes on to build the Microsoft build of Go. + +The patch files are ordinary Git patches and can also be applied manually +without any custom tooling. Git commands like [`git +am`](https://git-scm.com/docs/git-am) and [`git +apply`](https://git-scm.com/docs/git-apply) work directly. [`git +format-patch`](https://git-scm.com/docs/git-format-patch) produces the same +patch format as `git-go-patch`. + +Editing the patch files by hand is not recommended. Use `git-go-patch` or manual +`git` patching commands to let Git handle the formatting and fine details. -These patch files contain all the changes made to the upstream Go source code. -To explore them with Git, run `pwsh eng/run.ps1 submodule-refresh -commits` and -look at Git history inside the `go` submodule. +[git-go-patch]: https://github.com/microsoft/go-infra/tree/main/cmd/git-go-patch diff --git a/eng/_core/README.md b/eng/_core/README.md deleted file mode 100644 index 00e5a681990..00000000000 --- a/eng/_core/README.md +++ /dev/null @@ -1,30 +0,0 @@ -## `github.com/microsoft/go/_core` - -This module is a set of utilities Microsoft uses to build Go in Azure DevOps and -maintain this repository. Run `eng/run.ps1 build -h` to list available build -options, or `eng/run.ps1` to list all commands in this module. - -Unlike `_util`, the `_core` module should have zero external dependencies and -only requires a stage 0 Go toolset to build. The commands in this module are -used to produce the signed Microsoft binaries. - -### Support for gotestsum wrapping -The `_util` module implements a gotestsum wrapper around `_core`'s `build` -command. This requires some features in `_core` that accommodate gotestsum but -don't make sense as standalone features a dev would use. For example, JSON test -output and stderr redirection to stdout. - -The high-level execution flow looks roughly like this when running in CI: - -* `eng/pipeline/jobs/run-stage.yml` - runs: -* `eng/run.ps1 run-builder -test -builder linux-amd64-test -junitfile [...]` - which runs the Go function: -* `gotestsum.Run(... eng/run.ps1 build -test -json ...)` - which runs and captures the output of: -* `eng/run.ps1 build -test -json` - which runs [`cmd/build/build.go`](cmd/build/build.go) in this module. - -This is not currently used in our CI because this process seems to cut off -some test output: -[microsoft/go#1114](https://github.com/microsoft/go/issues/1114). diff --git a/eng/_core/cmd/submodule-refresh/submodule-refresh.go b/eng/_core/cmd/submodule-refresh/submodule-refresh.go deleted file mode 100644 index 9566fb11511..00000000000 --- a/eng/_core/cmd/submodule-refresh/submodule-refresh.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "flag" - "fmt" - "os" - - "github.com/microsoft/go/_core/patch" - "github.com/microsoft/go/_core/submodule" -) - -const description = ` -This command refreshes the Go submodule: initializes it, resets the content, and -applies patches to the stage by default, or optionally as commits. -` - -var commits = flag.Bool("commits", false, "Apply the patches as commits.") -var skipPatch = flag.Bool("skip-patch", false, "Skip applying patches.") -var origin = flag.String("origin", "", "Use this origin instead of the default defined in '.gitmodules' to fetch the repository.") -var shallow = flag.Bool("shallow", false, "Clone the submodule with depth 1.") -var fetchBearerToken = flag.String("fetch-bearer-token", "", "Use this bearer token to fetch the submodule repository.") - -func main() { - repoRootDir, err := os.Getwd() - if err != nil { - panic(err) - } - - var help = flag.Bool("h", false, "Print this help message.") - - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n") - flag.PrintDefaults() - fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) - } - - flag.Parse() - if *help { - flag.Usage() - return - } - - if err := refresh(repoRootDir); err != nil { - panic(err) - } -} - -func refresh(rootDir string) error { - if err := submodule.Init(rootDir, *origin, *fetchBearerToken, *shallow); err != nil { - return err - } - - if err := submodule.Reset(rootDir); err != nil { - return err - } - - if *skipPatch { - return nil - } - - mode := patch.ApplyModeIndex - if *commits { - mode = patch.ApplyModeCommits - } - - if err := patch.Apply(rootDir, mode); err != nil { - return err - } - return nil -} diff --git a/eng/_core/cmd/write-checksum/write-checksum.go b/eng/_core/cmd/write-checksum/write-checksum.go deleted file mode 100644 index 191ebd0eae6..00000000000 --- a/eng/_core/cmd/write-checksum/write-checksum.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "crypto/sha256" - "encoding/hex" - "flag" - "fmt" - "io" - "log" - "os" - "path/filepath" -) - -const description = ` -This command creates a SHA256 checksum file for the given files, in the same -location and with the same name as each given file but with ".sha256" added to -the end. Pass files as non-flag arguments. - -Generated files are compatible with "sha256sum -c". -` - -func main() { - help := flag.Bool("h", false, "Print this help message.") - - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n") - flag.PrintDefaults() - fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) - } - - flag.Parse() - if *help { - flag.Usage() - return - } - if flag.NArg() == 0 { - flag.Usage() - log.Fatal("No files specified.") - } - for _, m := range flag.Args() { - if err := writeSHA256ChecksumFile(m); err != nil { - log.Fatal(err) - } - } -} - -func writeSHA256ChecksumFile(path string) error { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - checksum := sha256.New() - if _, err = io.Copy(checksum, file); err != nil { - return err - } - // Write the checksum in a format that "sha256sum -c" can work with. Use the base path of the - // tarball (not full path, not relative path) because then "sha256sum -c" automatically works - // when the file and the checksum file are downloaded to the same directory. - content := fmt.Sprintf("%v %v\n", hex.EncodeToString(checksum.Sum(nil)), filepath.Base(path)) - outputPath := path + ".sha256" - if err := os.WriteFile(outputPath, []byte(content), 0o666); err != nil { - return err - } - fmt.Printf("Wrote checksum file %q with content: %v", outputPath, content) - return nil -} diff --git a/eng/_core/patch/patch.go b/eng/_core/patch/patch.go deleted file mode 100644 index 88e25d8e4db..00000000000 --- a/eng/_core/patch/patch.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package patch - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" -) - -type ApplyMode int - -const ( - // ApplyModeCommits applies patches as commits. This is useful for developing changes to the - // patches, because the commits can be automatically extracted back into patch files. - ApplyModeCommits ApplyMode = iota - // ApplyModeIndex applies patches as changes to the Git index and working tree. This means - // further changes to the Go source code will show up as unstaged changes, so if any intentional - // changes are performed in this state, they can be differentiated from the patch changes. - ApplyModeIndex -) - -// Apply runs a Git command to apply the patches in the repository onto the submodule. The exact Git -// command used ("am" or "apply") depends on the patch mode. -func Apply(rootDir string, mode ApplyMode) error { - goDir := filepath.Join(rootDir, "go") - patchDir := filepath.Join(rootDir, "patches") - - cmd := exec.Command("git") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Dir = goDir - - switch mode { - case ApplyModeCommits: - cmd.Args = append(cmd.Args, "am") - case ApplyModeIndex: - cmd.Args = append(cmd.Args, "apply", "--index") - default: - return fmt.Errorf("invalid patch mode '%v'", mode) - } - - // Trailing whitespace may be present in the patch files. Don't emit warnings for it here. These - // warnings should be avoided when authoring each patch file. If we made it to this point, it's - // too late to cause noisy warnings because of them. - cmd.Args = append(cmd.Args, "--whitespace=nowarn") - - // ReadDir returns alphabetical order for patches: we depend on it for the patch apply order. - entries, err := os.ReadDir(patchDir) - if err != nil { - return err - } - - for _, entry := range entries { - if entry.IsDir() { - continue - } - if filepath.Ext(entry.Name()) != ".patch" { - continue - } - cmd.Args = append(cmd.Args, filepath.Join(patchDir, entry.Name())) - } - - if err := runCmd(cmd); err != nil { - return err - } - return nil -} - -func runCmd(cmd *exec.Cmd) error { - fmt.Printf("---- Running command: %v\n", cmd.Args) - return cmd.Run() -} diff --git a/eng/_core/submodule/submodule.go b/eng/_core/submodule/submodule.go deleted file mode 100644 index 6fb4d54bb03..00000000000 --- a/eng/_core/submodule/submodule.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package submodule - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" -) - -// Init initializes and updates the submodule, but does not clean it. This func offers more options -// for initialization than Reset. If origin is defined, fetch the submodule from there instead of -// the default defined in '.gitmodules'. If fetchBearerToken is nonempty, use it as a bearer token -// during the fetch. If shallow is true, clone the submodule with depth 1. -func Init(rootDir, origin, fetchBearerToken string, shallow bool) error { - // Update the submodule commit, and initialize if it hasn't been done already. - command := []string{"git"} - if origin != "" { - command = append(command, "-c", "submodule.go.url="+origin) - } - if fetchBearerToken != "" { - command = append(command, "-c", "http.extraheader=AUTHORIZATION: bearer "+fetchBearerToken) - } - command = append(command, "submodule", "update", "--init") - if shallow { - command = append(command, "--depth", "1") - } - - if err := run(rootDir, command...); err != nil { - return err - } - return nil -} - -// Reset updates the submodule (with '--init'), resets all changes, and cleans all untracked files. -func Reset(rootDir string) error { - goDir := filepath.Join(rootDir, "go") - - // Update the submodule commit, and initialize if it hasn't been done already. - if err := run(rootDir, "git", "submodule", "update", "--init"); err != nil { - return err - } - - // Find toplevel directories (Git working tree roots) for the outer repo and what we expect to - // be the Go submodule. If the toplevel directory is the same for both, make sure not to clean! - // The submodule likely wasn't set up properly, and cleaning could result in unexpectedly losing - // work in the outer repo when the command spills over. - rootToplevel, err := getToplevel(rootDir) - if err != nil { - return err - } - goToplevel, err := getToplevel(goDir) - if err != nil { - return err - } - - if rootToplevel == goToplevel { - return fmt.Errorf("go submodule (%v) toplevel is the same as root (%v) toplevel: %v", goDir, rootDir, goToplevel) - } - - // Reset the index and working directory. This doesn't clean up new untracked files. - if err := run(goDir, "git", "reset", "--hard"); err != nil { - return err - } - // Delete untracked files detected by Git. Deliberately leave files that are ignored in - // '.gitignore': these files shouldn't interfere with the build process and could be used for - // incremental builds. - if err := run(goDir, "git", "clean", "-df"); err != nil { - return err - } - return nil -} - -func getToplevel(dir string) (string, error) { - c := exec.Command("git", "rev-parse", "--show-toplevel") - c.Dir = dir - out, err := c.CombinedOutput() - if err != nil { - return "", err - } - return string(out), nil -} - -func run(dir string, args ...string) error { - c := exec.Command(args[0], args[1:]...) - c.Stdout = os.Stdout - c.Stderr = os.Stderr - c.Dir = dir - return runCmd(c) -} - -func runCmd(cmd *exec.Cmd) error { - fmt.Printf("---- Running command: %v\n", cmd.Args) - return cmd.Run() -} diff --git a/eng/_util/.golangci.yml b/eng/_util/.golangci.yml new file mode 100644 index 00000000000..e7be9d9583c --- /dev/null +++ b/eng/_util/.golangci.yml @@ -0,0 +1,18 @@ +version: "2" +linters: + enable: + - bodyclose + - godox + - nakedret + - predeclared + - unconvert + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling +formatters: + enable: + - gofumpt diff --git a/eng/_util/README.md b/eng/_util/README.md index 870e9b96e02..da664d9c3a9 100644 --- a/eng/_util/README.md +++ b/eng/_util/README.md @@ -1,9 +1,18 @@ ## `github.com/microsoft/go/_util` This module is a set of utilities Microsoft uses to build Go in Azure DevOps and -maintain this repository. Run `eng/run.ps1` to list the available commands and -see instructions on how to use them. +maintain this repository. Run `eng/run.ps1 build -h` to list available build +options, or `eng/run.ps1` to list all commands in this module. -The `_util` module requires the `gotestsum` library and doesn't vendor it. -`_util` is not strictly necessary to build Go, so it's ok if its dependencies -are downloaded when needed. CI avoids uses the `_util` module when possible. +### Minimal dependencies +Some commands in this module use minimal external dependencies. This reduces the +dependencies used to produce the signed Microsoft binaries. + +Commands that use more than the minimal external dependencies will panic upon +init if `MS_GO_UTIL_ALLOW_ONLY_MINIMAL_DEPS` is set to `1`. This makes it +possible to test our pipelines to make sure they only use the expected commands. + +The minimal dependencies are themselves tested by +`TestMinimalCommandDependencies` in `testutil`. It uses `go list` to ensure that +all commands that use more than the minimal set of dependencies include the +conditional panic upon init. diff --git a/eng/_core/buildutil/buildutil.go b/eng/_util/buildutil/buildutil.go similarity index 63% rename from eng/_core/buildutil/buildutil.go rename to eng/_util/buildutil/buildutil.go index 4de7511bf5c..3699591662e 100644 --- a/eng/_core/buildutil/buildutil.go +++ b/eng/_util/buildutil/buildutil.go @@ -6,15 +6,16 @@ package buildutil import ( "fmt" + "io" "log" "os" + "os/exec" "strconv" - "strings" ) // Retry runs f until it succeeds or the attempt limit is reached. func Retry(attempts int, f func() error) error { - var i = 0 + i := 0 for ; i < attempts; i++ { if attempts > 1 { fmt.Printf("---- Running attempt %v of %v...\n", i+1, attempts) @@ -83,23 +84,52 @@ func GetEnvOrDefault(varName, defaultValue string) (string, error) { return v, nil } +// SetEnv sets an env var and logs it. Panics if it doesn't succeed. +func SetEnv(key, value string) { + fmt.Printf("Setting env '%s' to '%s'\n", key, value) + if err := os.Setenv(key, value); err != nil { + panic(err) + } +} + +// AppendEnv sets the given environment variable to the given value, or if the +// environment variable is already set, appends sep and then the given value. +func AppendEnv(key, value, sep string) { + if v, ok := os.LookupEnv(key); ok { + value = v + sep + value + } + SetEnv(key, value) +} + // AppendExperimentEnv sets the GOEXPERIMENT env var to the given value, or if GOEXPERIMENT is // already set, appends a comma separator and then the given value. func AppendExperimentEnv(experiment string) { - // If the experiment enables a crypto backend, allow fallback to Go crypto. Go turns off cgo - // and/or cross-builds in various situations during the build/tests, so we need to allow for it. - if strings.Contains(experiment, "opensslcrypto") || - strings.Contains(experiment, "cngcrypto") || - strings.Contains(experiment, "boringcrypto") || - strings.Contains(experiment, "systemcrypto") { + AppendEnv("GOEXPERIMENT", experiment, ",") +} - experiment += ",allowcryptofallback" - } - if v, ok := os.LookupEnv("GOEXPERIMENT"); ok { - experiment = v + "," + experiment - } - fmt.Printf("Setting GOEXPERIMENT: %v\n", experiment) - if err := os.Setenv("GOEXPERIMENT", experiment); err != nil { - panic(err) +// UnassignGOROOT unsets the GOROOT env var if it is set. +// +// Setting GOROOT explicitly in the environment has not been necessary since Go +// 1.9 (https://go.dev/doc/go1.9#goroot), but a dev or build machine may still +// have it set. It interferes with attempts to run the built Go (such as when +// building the race runtime), so remove the explicit GOROOT if set. +func UnassignGOROOT() error { + if explicitRoot, ok := os.LookupEnv("GOROOT"); ok { + fmt.Printf("---- Removing explicit GOROOT from environment: %v\n", explicitRoot) + if err := os.Unsetenv("GOROOT"); err != nil { + return err + } } + return nil +} + +// RunCmdMultiWriter runs a command and outputs the stdout to multiple [io.Writer]. +// The writers are closed after the command completes. +func RunCmdMultiWriter(cmdline []string, stdout ...io.Writer) error { + c := exec.Command(cmdline[0], cmdline[1:]...) + c.Stdout = io.MultiWriter(stdout...) + c.Stderr = os.Stderr + + fmt.Printf("---- Running command: %v\n", c.Args) + return c.Run() } diff --git a/eng/_util/buildutil/testjson.go b/eng/_util/buildutil/testjson.go new file mode 100644 index 00000000000..303a3ae6b28 --- /dev/null +++ b/eng/_util/buildutil/testjson.go @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package buildutil + +import ( + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/microsoft/go-infra/json2junit" +) + +type TestJSONFlags struct { + JUnitOutFile string + RawTestOutFile string +} + +func BindTestJSONFlags() *TestJSONFlags { + var f TestJSONFlags + flag.StringVar( + &f.JUnitOutFile, "junitout", "", + "Write the test output to a new file at this path as a JUnit file if running tests.") + flag.StringVar( + &f.RawTestOutFile, "rawtestout", "", + "Write raw test output to a new file at this path and summarize any test JSON before it reaches stdout.") + return &f +} + +func (f *TestJSONFlags) AppendToCmdline(cmdline []string) []string { + if f != nil { + if f.JUnitOutFile != "" { + cmdline = append(cmdline, "-junitout", f.JUnitOutFile) + } + if f.RawTestOutFile != "" { + cmdline = append(cmdline, "-rawtestout", f.RawTestOutFile) + } + } + return cmdline +} + +func (f *TestJSONFlags) RunTestCmd(cmdline []string) (err error) { + var writers []io.Writer + var needJSON bool + var rawTestOutFile string + var removeRawTestOutFile bool + var rawTestOut *os.File + + if f != nil { + if f.RawTestOutFile != "" || f.JUnitOutFile != "" { + rawTestOutFile = f.RawTestOutFile + if rawTestOutFile != "" { + if err := os.MkdirAll(filepath.Dir(rawTestOutFile), 0o755); err != nil { + return fmt.Errorf("failed to create directory for raw test output: %v", err) + } + rawTestOut, err = os.Create(rawTestOutFile) + } else { + rawTestOut, err = os.CreateTemp("", "go-test-json-*.txt") + } + if err != nil { + return fmt.Errorf("failed to create raw test output: %v", err) + } + if rawTestOutFile == "" { + rawTestOutFile = rawTestOut.Name() + removeRawTestOutFile = true + } + if removeRawTestOutFile { + defer func() { + err = errors.Join(err, os.Remove(rawTestOutFile)) + }() + } + if f.RawTestOutFile != "" { + writers = append(writers, &testJSONSummaryConverter{w: os.Stdout}) + } + writers = append(writers, rawTestOut) + if f.RawTestOutFile == "" { + writers = append(writers, os.Stdout) + } + needJSON = true + } else { + // If we don't summarize, we need to write directly to stdout. + writers = append(writers, os.Stdout) + } + } + if needJSON { + cmdline = append(cmdline, "-json") + } + + runErr := RunCmdMultiWriter(cmdline, writers...) + if rawTestOut != nil { + runErr = errors.Join(runErr, rawTestOut.Close()) + } + + if f != nil && f.JUnitOutFile != "" { + // Convert after the command exits so a converter error can't close the + // test process' stdout pipe and truncate the raw output. + runErr = errors.Join(runErr, json2junit.ConvertFileWithOptions( + f.JUnitOutFile, + rawTestOutFile, + &json2junit.Options{IncludePackageInTestName: true}, + )) + } + return runErr +} + +// testJSONSummaryConverter reads Go JSON test output and writes a summary that +// is sufficient for human readability in CI while filtering out excessive +// details that would otherwise result in hard-to-load CI logs. +// +// It is very similar to default "go test" output, but only good enough. It +// doesn't have the same behavior in some situations. +type testJSONSummaryConverter struct { + b bytes.Buffer + w io.Writer +} + +func (c *testJSONSummaryConverter) Write(b []byte) (int, error) { + c.b.Write(b) + lines := bytes.Split(c.b.Bytes(), []byte("\n")) + if len(lines) < 2 { + // We don't have a complete line yet. + return len(b), nil + } + completeLines := lines[:len(lines)-1] + for _, line := range completeLines { + var entry jsonEntry + if err := json.Unmarshal(line, &entry); err != nil { + // An error means it either isn't a JSON line, or it's an invalid one. + // In both cases, simply write it for the summary. + fmt.Fprintf(c.w, "%q %v\n", line, err) + continue + } + if entry.Action != "output" || entry.Test != "" { + // Too much info for a summary. + continue + } + if entry.Output == "PASS\n" || + entry.Output == "FAIL\n" { + continue + } + _, _ = c.w.Write([]byte(entry.Output)) + } + // Keep the last, incomplete line in the buffer for the next call, if any. + c.b.Reset() + _, _ = c.b.Write(lines[len(lines)-1]) + return len(b), nil +} + +// jsonEntry is the parts of a single entry in the JSON file relevant to testJSONSummaryConverter. +type jsonEntry struct { + Time time.Time + Action string + Package string + Test string + Output string + Elapsed float64 +} diff --git a/eng/_util/buildutil/testjson_test.go b/eng/_util/buildutil/testjson_test.go new file mode 100644 index 00000000000..191e5c54100 --- /dev/null +++ b/eng/_util/buildutil/testjson_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package buildutil + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunTestCmdConvertsRawOutputAfterCommand(t *testing.T) { + t.Setenv("GO_WANT_TESTJSON_HELPER_PROCESS", "good-json") + + tmpDir := t.TempDir() + rawOut := filepath.Join(tmpDir, "raw.jsonl") + junitOut := filepath.Join(tmpDir, "test-results.xml") + flags := &TestJSONFlags{ + JUnitOutFile: junitOut, + RawTestOutFile: rawOut, + } + + if err := flags.RunTestCmd([]string{os.Args[0], "-test.run=TestRunTestCmdHelperProcess", "--"}); err != nil { + t.Fatal(err) + } + + raw, err := os.ReadFile(rawOut) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), "TestDescribeConflict") { + t.Fatalf("raw output does not contain test event:\n%s", raw) + } + + junit, err := os.ReadFile(junitOut) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(junit), `name="net/http.TestDescribeConflict"`) { + t.Fatalf("JUnit output does not contain package-prefixed test name:\n%s", junit) + } +} + +func TestRunTestCmdPreservesRawOutputWhenJUnitConversionFails(t *testing.T) { + t.Setenv("GO_WANT_TESTJSON_HELPER_PROCESS", "bad-json") + + tmpDir := t.TempDir() + rawOut := filepath.Join(tmpDir, "raw.jsonl") + junitOut := filepath.Join(tmpDir, "test-results.xml") + flags := &TestJSONFlags{ + JUnitOutFile: junitOut, + RawTestOutFile: rawOut, + } + + err := flags.RunTestCmd([]string{os.Args[0], "-test.run=TestRunTestCmdHelperProcess", "--"}) + if err == nil { + t.Fatal("expected JUnit conversion to fail") + } + if !strings.Contains(err.Error(), "no run entry for TestMissing") { + t.Fatalf("expected missing run entry error, got %v", err) + } + + raw, err := os.ReadFile(rawOut) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), "TestMissing") { + t.Fatalf("raw output does not contain failing converter line:\n%s", raw) + } +} + +func TestRunTestCmdHelperProcess(t *testing.T) { + switch os.Getenv("GO_WANT_TESTJSON_HELPER_PROCESS") { + case "good-json": + fmt.Print(`{"Time":"2026-05-19T07:24:14Z","Action":"start","Package":"net/http"} +{"Time":"2026-05-19T07:24:14Z","Action":"run","Package":"net/http","Test":"TestDescribeConflict"} +{"Time":"2026-05-19T07:24:14Z","Action":"output","Package":"net/http","Test":"TestDescribeConflict","Output":"=== RUN TestDescribeConflict\n"} +{"Time":"2026-05-19T07:24:14Z","Action":"pass","Package":"net/http","Test":"TestDescribeConflict","Elapsed":0} +{"Time":"2026-05-19T07:24:14Z","Action":"pass","Package":"net/http","Elapsed":0} +`) + case "bad-json": + fmt.Print(`{"Time":"2026-05-19T07:24:14Z","Action":"start","Package":"net/http"} +{"Time":"2026-05-19T07:24:14Z","Action":"run","Package":"net/http","Test":"TestDescribeConflict"} +{"Time":"2026-05-19T07:24:14Z","Action":"pass","Package":"net/http","Test":"TestDescribeConflict","Elapsed":0} +{"Time":"2026-05-19T07:24:14Z","Action":"pass","Package":"net/http","Test":"TestMissing","Elapsed":0} +`) + default: + return + } + os.Exit(0) +} diff --git a/eng/_core/cmd/build/build.go b/eng/_util/cmd/build/build.go similarity index 71% rename from eng/_core/cmd/build/build.go rename to eng/_util/cmd/build/build.go index a316f6dd04d..923d89e04ce 100644 --- a/eng/_core/cmd/build/build.go +++ b/eng/_util/cmd/build/build.go @@ -15,9 +15,9 @@ import ( "runtime" "strings" - "github.com/microsoft/go/_core/buildutil" - "github.com/microsoft/go/_core/patch" - "github.com/microsoft/go/_core/submodule" + "github.com/microsoft/go-infra/patch" + "github.com/microsoft/go-infra/submodule" + "github.com/microsoft/go/_util/buildutil" ) const description = ` @@ -33,16 +33,16 @@ in 'src' such as 'src/run.bash' instead of this script. Example: Build Go, run tests, and produce an archive file: - eng/run.ps1 build -test -pack + eng/run.ps1 build -test -packbuild ` func main() { - var help = flag.Bool("h", false, "Print this help message.") + help := flag.Bool("h", false, "Print this help message.") o := &options{} flag.BoolVar(&o.SkipBuild, "skipbuild", false, "Disable building Go.") + flag.BoolVar(&o.SkipBuildRace, "skipbuildrace", false, "Disable building Go with race detector.") flag.BoolVar(&o.Test, "test", false, "Enable running tests.") - flag.BoolVar(&o.JSON, "json", false, "Runs tests with -json flag to emit verbose results in JSON format. For use in CI.") flag.BoolVar(&o.PackBuild, "packbuild", false, "Enable creating an archive of this build using upstream 'distpack' and placing it in eng/artifacts/bin.") flag.BoolVar(&o.PackSource, "packsource", false, "Enable creating a source archive using upstream 'distpack' and placing it in eng/artifacts/bin.") flag.BoolVar(&o.CreatePDB, "pdb", false, "Create PDB files for all the PE binaries in the bin and tool directories. The PE files are modified in place and PDBs are placed in eng/artifacts/symbols.") @@ -54,6 +54,8 @@ func main() { flag.StringVar(&o.Experiment, "experiment", "", "Include this string in GOEXPERIMENT.") + o.TestJSONFlags = buildutil.BindTestJSONFlags() + o.MaxMakeAttempts = buildutil.MaxMakeRetryAttemptsOrExit() flag.Usage = func() { @@ -70,28 +72,28 @@ func main() { // If build returns an error, handle it here with panic. Having build return an error makes it // easier to adapt build in the future to somewhere else in the module to use it as an API. (For - // example, "build" could be changed to "Build" and run-builder could use it. The reason this - // hasn't been done yet is that gotestsum can only run a command line, not a Go function.) + // example, "build" could be changed to "Build" and run-builder could use it.) if err := build(o); err != nil { panic(err) } } type options struct { - SkipBuild bool - Test bool - JSON bool - PackBuild bool - PackSource bool - CreatePDB bool - Refresh bool - Experiment string + SkipBuild bool + SkipBuildRace bool + Test bool + PackBuild bool + PackSource bool + CreatePDB bool + Refresh bool + Experiment string + + TestJSONFlags *buildutil.TestJSONFlags MaxMakeAttempts int } -func build(o *options) error { - +func build(o *options) (err error) { scriptExtension := ".bash" executableExtension := "" archiveExtension := ".tar.gz" @@ -110,12 +112,17 @@ func build(o *options) error { if err != nil { return err } + goRootDir := filepath.Join(rootDir, "go") if o.Refresh { - if err := submodule.Reset(rootDir); err != nil { + config, err := patch.FindAncestorConfig(rootDir) + if err != nil { return err } - if err := patch.Apply(rootDir, patch.ApplyModeIndex); err != nil { + if err := submodule.Reset(rootDir, filepath.Join(config.RootDir, config.SubmoduleDir), true); err != nil { + return err + } + if err := patch.Apply(config, patch.ApplyModeIndex); err != nil { return err } } @@ -134,6 +141,25 @@ func build(o *options) error { } fmt.Printf("---- Target platform: %v_%v\n", targetOS, targetArch) + if err := buildutil.UnassignGOROOT(); err != nil { + return err + } + + // Make sure the MICROSOFT_REVISION file from the outer repository is in the Go root. We need to + // copy it in before running the build (not just before running distpack) because it's used + // during the build to generate the "src/internal/buildcfg/zbootstrap.go" file. + microsoftRevisionDst := filepath.Join(goRootDir, "MICROSOFT_REVISION") + if err := copyFile(microsoftRevisionDst, filepath.Join(rootDir, "MICROSOFT_REVISION")); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("unable to read MICROSOFT_REVISION file for unexpected reason: %v", err) + } + // Ok: a main branch or pre-stable release branch has no MICROSOFT_REVISION file. + } else { + // Best effort: clean up the MICROSOFT_REVISION file when we're done. Clean up for + // tidier dev workflows: the temp MICROSOFT_REVISION file should never be checked in. + defer os.Remove(microsoftRevisionDst) + } + // The upstream build scripts in {repo-root}/src require your working directory to be src, or // they instantly fail. Change the current process dir so that we can run them. if err := os.Chdir("go/src"); err != nil { @@ -178,7 +204,12 @@ func build(o *options) error { // The race runtime requires cgo. // It isn't supported on arm or 386. // It's supported on arm64, but the official linux-arm64 distribution doesn't include it. - if os.Getenv("CGO_ENABLED") != "0" && targetArch != "arm" && targetArch != "arm64" && targetArch != "386" { + if !o.SkipBuildRace && + os.Getenv("CGO_ENABLED") != "0" && + targetArch != "arm" && + targetArch != "arm64" && + targetArch != "386" { + fmt.Println("---- Building race runtime...") err := runCommandLine( filepath.Join("..", "bin", "go"+executableExtension), @@ -200,46 +231,11 @@ func build(o *options) error { }..., ) - // "src/run.bat" doesn't pass arguments through to "dist test" like "src/run.bash" does. - // This prevents "-json" from working properly: in "src/run.bat -json", "-json" is a no-op. - // So, use "dist test" directly, here. Some environment variables may be subtly different, - // but it appears to work fine for dev scenarios. https://github.com/microsoft/go/issues/109 - if runtime.GOOS == "windows" { - testCommandLine = []string{ - filepath.Join("..", "bin", "go"+executableExtension), - "tool", "dist", "test", - } - } - - // "-json": Get test results as lines of JSON. - if o.JSON { - testCommandLine = append(testCommandLine, "-json") - } - - testCmd := exec.Command(testCommandLine[0], testCommandLine[1:]...) - testCmd.Stdout = os.Stdout - // Redirect stderr to stdout. We expect some lines of stderr to always show up during the - // test run, but "build"'s caller might not understand that. - // - // For example, if we're running in CI, gotestsum may be capturing our output to report in a - // JUnit file. If gotestsum detects output in stderr, it prints it in an error message. This - // error message stands out, and could mislead someone trying to diagnose a failed test run. - // Redirecting all stderr output avoids this scenario. (See /eng/_core/README.md for more - // info on why we may be wrapped by gotestsum.) - // - // An example of benign stderr output is when the tests check for machine capabilities. A - // Cgo static linking test emits "/usr/bin/ld: cannot find -lc" when it checks the - // capabilities of "ld" on the current system. - // - // The stderr output isn't used to determine whether the tests succeeded or not. (The - // redirect doesn't cause an issue where tests succeed that should have failed.) - testCmd.Stderr = os.Stdout - if err := runCmd(testCmd); err != nil { + if err := o.TestJSONFlags.RunTestCmd(testCommandLine); err != nil { return err } } - goRootDir := filepath.Join(rootDir, "go") if o.CreatePDB { if _, err := exec.LookPath("gopdb"); err != nil { return fmt.Errorf("gopdb not found in PATH: %v", err) @@ -288,26 +284,28 @@ func build(o *options) error { } if o.PackBuild || o.PackSource { - // Find the host version of distpack. (Not the target version, which might not run.) - toolsDir := filepath.Join(goRootDir, "pkg", "tool", runtime.GOOS+"_"+runtime.GOARCH) - // distpack needs a VERSION file to run. If we're on the main branch, we don't have one, so - // use dist's version calculation to create a temp dev version and put it in VERSION. + // We need to figure out the version string so we can identify the distpack output files and + // copy them to our artifacts dir with the right names. var version string if data, err := os.ReadFile(filepath.Join(goRootDir, "VERSION")); err != nil { if errors.Is(err, os.ErrNotExist) { - if version, err = writeDevelVersionFile(goRootDir, toolsDir); err != nil { - return fmt.Errorf("unable to pack: failed writing development VERSION file: %v", err) + // The distpack tool needs a VERSION file to exist or it'll fail. If we're on the + // main branch, there won't be a VERSION file, so use dist's version calculation to + // create a temp dev version and put it in VERSION. + if version, err = writeDevelVersionFile(goRootDir, executableExtension); err != nil { + return fmt.Errorf("unable to write development VERSION file: %v", err) } - // Best effort: clean up the VERSION file when we're done. This is just for dev - // workflows: the temp VERSION file should never be checked in. + // Best effort: clean up the VERSION file when we're done with the build. Clean up + // for tidier dev workflows: the temp VERSION file should never be checked in. defer os.Remove(filepath.Join(goRootDir, "VERSION")) } else { - return fmt.Errorf("unable to pack: VERSION file in unexpected state: %v", err) + return fmt.Errorf("unable to read VERSION file for unexpected reason: %v", err) } } else { version, _, _ = strings.Cut(string(data), "\n") } - cmd := exec.Command(filepath.Join(toolsDir, "distpack"+executableExtension)) + + cmd := exec.Command(filepath.Join(goRootDir, "bin", "go"+executableExtension), "tool", "distpack") cmd.Env = append(os.Environ(), "GOROOT="+goRootDir) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -355,27 +353,27 @@ func build(o *options) error { return nil } -func writeDevelVersionFile(goRootDir, toolsDir string) (string, error) { - cmd := exec.Command(filepath.Join(toolsDir, "dist"), "version") +func writeDevelVersionFile(goRootDir, executableExtension string) (string, error) { + // Use "go tool dist version" to directly get the version the toolset would call itself. + cmd := exec.Command(filepath.Join(goRootDir, "bin", "go"+executableExtension), "tool", "dist", "version") cmd.Env = append(os.Environ(), "GOROOT="+goRootDir) vBytes, err := cmd.CombinedOutput() if err != nil { return "", fmt.Errorf("unable to get dist version: %v (%v)", err, string(vBytes)) } - fields := strings.Fields(string(vBytes)) - if len(fields) < 2 { - return "", fmt.Errorf("expected at least 2 fields in dist version output, got %q in %q", len(fields), string(vBytes)) - } - if fields[0] != "devel" { - return "", fmt.Errorf("expected first field 'devel' in dist version, got %q", fields[0]) + ver, _, _ := strings.Cut(string(vBytes), " ") + // The version string is something like "go1.21-devel_abcde1234 Mon Oct 16 12:34:56 2023" + // Just using the first field removing the devel_ substring: the full VERSION file string + // is placed into the archive filename, so this keeps it simple and avoids special characters. + const devel = "devel_" + if !strings.Contains(ver, devel) { + return "", fmt.Errorf("expected substring %q in dist version, got %q", devel, ver) } - // The second field should be something like "go1.21-abcde1234", and the remaining fields are a - // timestamp. Just using the second field as is: the full VERSION file string is placed into the - // archive filename, so this keeps it simple and avoids special characters. - if err := os.WriteFile(filepath.Join(goRootDir, "VERSION"), []byte(fields[1]), 0o666); err != nil { + ver = strings.Replace(ver, devel, "", 1) + if err := os.WriteFile(filepath.Join(goRootDir, "VERSION"), []byte(ver), 0o666); err != nil { return "", err } - return fields[1], nil + return ver, nil } // copyFile copies src to dst, creating dst's directory if necessary. Handles errors robustly, diff --git a/eng/_core/cmd/cmdscan/cmdscan.go b/eng/_util/cmd/cmdscan/cmdscan.go similarity index 99% rename from eng/_core/cmd/cmdscan/cmdscan.go rename to eng/_util/cmd/cmdscan/cmdscan.go index 3c4c8a3e06d..08853261f8e 100644 --- a/eng/_core/cmd/cmdscan/cmdscan.go +++ b/eng/_util/cmd/cmdscan/cmdscan.go @@ -186,7 +186,7 @@ func scan(r io.Reader, commands, echo *os.File) error { for _, f := range filters { if f.regexp.MatchString(s.Text()) { fmt.Fprintf(echo, "Found pattern '%v'\n", f.regexp) - fmt.Fprintf(commands, warn(&f, s.Text())) + fmt.Fprint(commands, warn(&f, s.Text())) } } } diff --git a/eng/_util/cmd/createbuildassetjson/nonminimaldeps.go b/eng/_util/cmd/createbuildassetjson/nonminimaldeps.go new file mode 100644 index 00000000000..dd9d230013f --- /dev/null +++ b/eng/_util/cmd/createbuildassetjson/nonminimaldeps.go @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// This command uses non-minimal dependencies, so ensure it can't be used while in minimal mode. + +import _ "github.com/microsoft/go/_util/internal/depsinitpanic" diff --git a/eng/_util/cmd/go/go.go b/eng/_util/cmd/go/go.go new file mode 100644 index 00000000000..26f98f72fa4 --- /dev/null +++ b/eng/_util/cmd/go/go.go @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/microsoft/go-infra/executil" +) + +// This command runs the given go command using _util's version of Go with the +// working directory set to the root of the _util module. +// +// All args pass through to Go. Unlike other commands, "-h"/"-help" are not +// handled to give a detailed description of this command. It doesn't seem worth +// the effort to handle help args in a way that doesn't introduce further edge +// cases or usability complications. + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} + +func run() error { + stage0Goroot := os.Getenv("STAGE_0_GOROOT") + if stage0Goroot == "" { + return fmt.Errorf("STAGE_0_GOROOT not set") + } + + return executil.Run(executil.Dir( + filepath.Join("eng", "_util"), + filepath.Join(stage0Goroot, "bin", "go"), + os.Args[1:]..., + )) +} diff --git a/eng/_util/cmd/patchcheck/patchcheck.go b/eng/_util/cmd/patchcheck/patchcheck.go new file mode 100644 index 00000000000..206e40cd519 --- /dev/null +++ b/eng/_util/cmd/patchcheck/patchcheck.go @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + "log" + + "github.com/microsoft/go/_util/internal/patchcheck" +) + +var description = ` +This command checks the patch files in the patches/ directory for common issues. +` + +func main() { + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n") + flag.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) + } + + flag.Parse() + + if err := run(); err != nil { + log.Fatalln(err) + } +} + +func run() error { + issues, err := patchcheck.FindPatchIssues() + if err != nil { + return err + } + if len(issues) == 0 { + fmt.Println("Patches are happy!") + return nil + } + for _, issue := range issues { + fmt.Printf("%s: %s\n", issue.PatchFile, issue.Message) + } + return fmt.Errorf("found %d patch issue(s)", len(issues)) +} diff --git a/eng/_util/cmd/pipelineymlgen/gen.go b/eng/_util/cmd/pipelineymlgen/gen.go new file mode 100644 index 00000000000..78e52ff0066 --- /dev/null +++ b/eng/_util/cmd/pipelineymlgen/gen.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// Generate the Azure Pipelines YAML from .gen.yml files for this repo. Normally +// this would be done from eng/ or eng/pipeline, but in this repo we don't have +// a Go module containing that directory. + +//go:generate go run github.com/microsoft/go-infra/cmd/pipelineymlgen -r ../../../../eng/pipeline diff --git a/eng/_util/cmd/pipelineymlgen/pipelineymlgen.go b/eng/_util/cmd/pipelineymlgen/pipelineymlgen.go new file mode 100644 index 00000000000..8bbaf01ffc9 --- /dev/null +++ b/eng/_util/cmd/pipelineymlgen/pipelineymlgen.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" +) + +var description = ` +This command regenerates pipeline yml files in eng/pipeline using the +pipelineymlgen tool. +` + +// This command exists only because it's awkward to use the "eng/_util" module +// from the root of the repository with go commands alone. + +func main() { + help := flag.Bool("h", false, "Print this help message.") + + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n") + flag.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) + } + + flag.Parse() + if *help { + flag.Usage() + return + } + + if err := run(); err != nil { + log.Fatalln(err) + } +} + +func run() error { + cmd := exec.Command("go", "generate", ".") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Dir = "eng/_util/cmd/pipelineymlgen" + + fmt.Printf("Running %q in %q...\n", cmd, cmd.Dir) + if err := cmd.Run(); err != nil { + return fmt.Errorf("running %q in %q: %v", cmd, cmd.Dir, err) + } + + wd, err := os.Getwd() + if err != nil { + return fmt.Errorf("getting working directory: %v", err) + } + + fmt.Printf("Done. See %s\n", filepath.Join(wd, "eng", "pipeline")) + return nil +} diff --git a/eng/_util/cmd/pipelineymlgen/pipelineymlgen_test.go b/eng/_util/cmd/pipelineymlgen/pipelineymlgen_test.go new file mode 100644 index 00000000000..92686da5632 --- /dev/null +++ b/eng/_util/cmd/pipelineymlgen/pipelineymlgen_test.go @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "os/exec" + "testing" +) + +func TestGoInfraPipelineGenReproducible(t *testing.T) { + cmd := exec.Command("go", "run", "github.com/microsoft/go-infra/cmd/pipelineymlgen", "-exit-code", "-r", "../../../../eng/pipeline") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("pipelineymlgen reproduciblility check failed: %v\nOutput:\n%s\nRun 'pwsh eng/run.ps1 pipelineymlgen' if differences are expected.", err, out) + } +} diff --git a/eng/_util/cmd/run-builder/nonminimaldeps.go b/eng/_util/cmd/run-builder/nonminimaldeps.go new file mode 100644 index 00000000000..dd9d230013f --- /dev/null +++ b/eng/_util/cmd/run-builder/nonminimaldeps.go @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// This command uses non-minimal dependencies, so ensure it can't be used while in minimal mode. + +import _ "github.com/microsoft/go/_util/internal/depsinitpanic" diff --git a/eng/_util/cmd/run-builder/run-builder.go b/eng/_util/cmd/run-builder/run-builder.go index b3187737242..caec910f4c1 100644 --- a/eng/_util/cmd/run-builder/run-builder.go +++ b/eng/_util/cmd/run-builder/run-builder.go @@ -5,6 +5,7 @@ package main import ( + "errors" "flag" "fmt" "log" @@ -13,8 +14,7 @@ import ( "strconv" "strings" - "github.com/microsoft/go/_core/buildutil" - gotestsumcmd "gotest.tools/gotestsum/cmd" + "github.com/microsoft/go/_util/buildutil" ) const description = ` @@ -37,14 +37,15 @@ in your repository to read-only. var dryRun = flag.Bool("n", false, "Enable dry run: print the commands that would be run, but do not run them.") func main() { - var builder = flag.String("builder", "", "[Required] Specify a builder to run. Note, this may be destructive!") - var experiment = flag.String("experiment", "", "Include this string in GOEXPERIMENT.") - var fipsMode = flag.Bool("fipsmode", false, "Run the Go tests in FIPS mode.") - var jUnitFile = flag.String("junitfile", "", "Write a JUnit XML file to this path if this builder runs tests.") - var build = flag.Bool("build", false, "Run the build.") - var test = flag.Bool("test", false, "Run the tests.") + builder := flag.String("builder", "", "[Required] Specify a builder to run. Note, this may be destructive!") + experiment := flag.String("experiment", "", "Include this string in GOEXPERIMENT.") + fipsMode := flag.Bool("fipsmode", false, "Run the Go tests in FIPS mode.") + build := flag.Bool("build", false, "Run the build.") + test := flag.Bool("test", false, "Run the tests.") - var help = flag.Bool("h", false, "Print this help message.") + testJSONFlags := buildutil.BindTestJSONFlags() + + help := flag.Bool("h", false, "Print this help message.") flag.Usage = func() { fmt.Fprintf(flag.CommandLine.Output(), "Usage of run-builder.go:\n") @@ -79,18 +80,18 @@ func main() { // running tests: switch config { case "clang": - env("CC", "/usr/bin/clang-3.9") + buildutil.SetEnv("CC", "/usr/bin/clang-3.9") case "longtest": - env("GO_TEST_SHORT", "false") + buildutil.SetEnv("GO_TEST_SHORT", "false") timeoutScale *= 5 case "nocgo": - env("CGO_ENABLED", "0") + buildutil.SetEnv("CGO_ENABLED", "0") case "noopt": - env("GO_GCFLAGS", "-N -l") + buildutil.SetEnv("GO_GCFLAGS", "-N -l") case "regabi": buildutil.AppendExperimentEnv("regabi") case "ssacheck": - env("GO_GCFLAGS", "-d=ssa/check/on") + buildutil.SetEnv("GO_GCFLAGS", "-d=ssa/check/on") case "staticlockranking": buildutil.AppendExperimentEnv("staticlockranking") } @@ -102,7 +103,11 @@ func main() { } if timeoutScale != 1 { - env("GO_TEST_TIMEOUT_SCALE", strconv.Itoa(timeoutScale)) + buildutil.SetEnv("GO_TEST_TIMEOUT_SCALE", strconv.Itoa(timeoutScale)) + } + + if err := buildutil.UnassignGOROOT(); err != nil { + log.Fatal(err) } buildCmdline := []string{"pwsh", "eng/run.ps1", "build"} @@ -130,7 +135,8 @@ func main() { // validate the run.ps1 script with "build" tool works to build and test Go. It runs a // subset of the "test" builder's tests, but it uses the dev workflow. testCmdline := append(buildCmdline, "-skipbuild", "-test") - if err := runTest(testCmdline, *jUnitFile); err != nil { + testCmdline = testJSONFlags.AppendToCmdline(testCmdline) + if err := run(testCmdline...); err != nil { log.Fatal(err) } @@ -143,7 +149,7 @@ func main() { } if *fipsMode { - env("GOFIPS", "1") + buildutil.AppendEnv("GODEBUG", "fips140=on", ",") // Enable system-wide FIPS if supported by the host platform. restore, err := enableSystemWideFIPS() if err != nil { @@ -156,14 +162,14 @@ func main() { // The tests read GO_BUILDER_NAME and make decisions based on it. For some configurations, // we only need to set this env var. - env("GO_BUILDER_NAME", *builder) + buildutil.SetEnv("GO_BUILDER_NAME", *builder) // The "fake" config "test" is a sentinel value that means we should omit the config part of // the builder name. This lets us have a stable "{os}-{arch}-{config}" API (particularly // useful when dealing with AzDO YAML limitations) while still being able to test e.g. the // "linux-amd64" builder from upstream. if config == "test" { - env("GO_BUILDER_NAME", goos+"-"+goarch) + buildutil.SetEnv("GO_BUILDER_NAME", goos+"-"+goarch) } cmdline := []string{ @@ -190,27 +196,23 @@ func main() { ) } - err := runTest(cmdline, *jUnitFile) - // If we got an ExitError, the error message was already printed by the command. We just - // need to exit with the same exit code. - if exitErr, ok := err.(*exec.ExitError); ok { - os.Exit(exitErr.ExitCode()) - } - if err != nil { - // Something else happened: alert the user. - log.Fatal(err) + if *dryRun { + fmt.Printf("---- Dry run. Would have run test command: %v\n", cmdline) + } else { + if err := testJSONFlags.RunTestCmd(cmdline); err != nil { + // If we got an ExitError, the error message was already printed by the command. We just + // need to exit with the same exit code. + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + os.Exit(exitErr.ExitCode()) + } + // Something else happened: alert the user. + log.Fatal(err) + } } } } -// env sets an env var and logs it. Panics if it doesn't succeed. -func env(key, value string) { - fmt.Printf("Setting env '%s' to '%s'\n", key, value) - if err := os.Setenv(key, value); err != nil { - panic(err) - } -} - func run(cmdline ...string) error { c := exec.Command(cmdline[0], cmdline[1:]...) c.Stdout = os.Stdout @@ -231,67 +233,3 @@ func runOrPanic(cmdline ...string) { panic(err) } } - -// runTest runs a testing command. If given a JUnit XML file path, runs the test command inside a -// gotestsum command that converts the JSON output into JUnit XML and writes it to a file at this -// path. -func runTest(cmdline []string, jUnitFile string) error { - if jUnitFile != "" { - // Emit verbose JSON results in stdout for conversion. - cmdline = append(cmdline, "-json") - } - - if *dryRun { - fmt.Printf("---- Dry run. Would have run test command: %v\n", cmdline) - return nil - } - - if jUnitFile != "" { - // Set up gotestsum args. We rely on gotestsum to run the command, capture its output, and - // convert it to JUnit test result XML. - gotestsumArgs := append( - []string{ - "--junitfile", jUnitFile, - "--hide-summary", "skipped,output", - "--format", "standard-quiet", - // When a builder runs tests, some JSON lines are mixed in with standard output - // lines. Normally gotestsum treats this as an error, but we need to allow it. - "--ignore-non-json-output-lines", - // We don't use 'go test', we pass our own raw command. ("cmdline" args.) - "--raw-command", - }, - cmdline..., - ) - - // gotestsum embeds the current version of Go into the JUnit file. This causes some - // problems, so use GOVERSION to override the behavior and use a simple placeholder. - // - // To find the Go version, gotestsum first looks up GOVERSION in env. If it doesn't exist, - // then it looks for "go" in PATH and uses the output of "go version". If Go doesn't exist - // in PATH, then gotestsum emits a warning. - // - // There are two problems. First, in CI, we don't have Go in PATH, so the warning shows up. - // It's shown as the last line of output in CI, so it seems more important than it really - // is. Second, even if gotestsum does find Go in PATH, it's the wrong version. We're running - // tests using the Go we just built, which is never in PATH. Both of these problems could - // end up being red herrings in the future, but we prevent them by setting GOVERSION. - // - // We could run "go version", parse the output, and use that as GOVERSION. However, this - // doesn't seem useful, because we know that we ran tests using the Go we just built. - env("GOVERSION", "gotestsum_go_version_placeholder") - - fmt.Printf("---- Running gotestsum command: %v\n", gotestsumArgs) - - // Use "ARG_0_PLACEHOLDER" as an arbitrary placeholder name. This is because here, we're - // essentially directly calling gotestsum's main method. The 0th arg to a main method is - // usually the program's path. This is used in the program's help text to give example - // commands that the user can copy-paste no matter where the executable lives or if it's - // been renamed. However, run-builder uses gotestsum as a library, so it's compiled into our - // binary and there is no actual 'gotestsum' program. We could pass run-builder's path, but - // that would be misleading if it ever shows up in gotestsum's output unexpectedly. Instead, - // pass an obvious placeholder. - return gotestsumcmd.Run("ARG_0_PLACEHOLDER", gotestsumArgs) - } - // If we don't have a jUnitFile target, run the command normally. - return run(cmdline...) -} diff --git a/eng/_util/cmd/run-builder/systemfips_fallback.go b/eng/_util/cmd/run-builder/systemfips_fallback.go index 8f6c89e557b..e66b19aada0 100644 --- a/eng/_util/cmd/run-builder/systemfips_fallback.go +++ b/eng/_util/cmd/run-builder/systemfips_fallback.go @@ -1,8 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//go:build !windows -// +build !windows +//go:build !linux package main diff --git a/eng/_util/cmd/run-builder/systemfips_linux.go b/eng/_util/cmd/run-builder/systemfips_linux.go new file mode 100644 index 00000000000..b7afe7d3ada --- /dev/null +++ b/eng/_util/cmd/run-builder/systemfips_linux.go @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "log" + "os" + + "github.com/microsoft/go/_util/buildutil" +) + +// enableSystemWideFIPS enables Mariner and Azure Linux 3 process-wide FIPS mode +// for any process that inherits the current process' environment variables. +func enableSystemWideFIPS() (restore func(), err error) { + // FIPS mode is enabled if OPENSSL_FORCE_FIPS_MODE is set, regardless of the value. + _, ok := os.LookupEnv("OPENSSL_FORCE_FIPS_MODE") + if ok { + log.Println("Mariner and Azure Linux 3 forced FIPS mode (OPENSSL_FORCE_FIPS_MODE) already enabled.") + return nil, nil + } + + buildutil.SetEnv("OPENSSL_FORCE_FIPS_MODE", "1") + log.Println("Enabled Mariner and Azure Linux 3 FIPS mode (OPENSSL_FORCE_FIPS_MODE).") + + return func() { + err := os.Unsetenv("OPENSSL_FORCE_FIPS_MODE") + if err != nil { + log.Printf("Unable to unset OPENSSL_FORCE_FIPS_MODE: %v\n", err) + return + } + log.Println("Successfully unset OPENSSL_FORCE_FIPS_MODE.") + }, nil +} diff --git a/eng/_util/cmd/run-builder/systemfips_windows.go b/eng/_util/cmd/run-builder/systemfips_windows.go deleted file mode 100644 index 64d03bd981c..00000000000 --- a/eng/_util/cmd/run-builder/systemfips_windows.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -package main - -import ( - "fmt" - "log" - - "golang.org/x/sys/windows/registry" -) - -// enableSystemWideFIPS enables Windows system-wide FIPS and returns a state-restoring func in case -// the host will be used later by another process. If the host is simultaneously shared, enabling -// system-wide FIPS may interfere because this policy is a machine setting. -func enableSystemWideFIPS() (restore func(), err error) { - key, err := registry.OpenKey( - registry.LOCAL_MACHINE, - `SYSTEM\CurrentControlSet\Control\Lsa\FipsAlgorithmPolicy`, - registry.QUERY_VALUE|registry.SET_VALUE) - if err != nil { - return nil, err - } - - enabled, enabledType, err := key.GetIntegerValue("Enabled") - if err != nil { - return nil, err - } - - if enabledType != registry.DWORD { - return nil, fmt.Errorf("unexpected FIPS algorithm policy Enabled key type: %v", enabledType) - } - - if enabled == 1 { - log.Println("FIPS algorithm policy already enabled.") - return nil, nil - } - - log.Printf("Found FIPS algorithm policy Enabled value: %v\n", enabled) - if err := key.SetDWordValue("Enabled", 1); err != nil { - return nil, err - } - - log.Println("Enabled FIPS algorithm policy.") - - return func() { - defer key.Close() - err := key.SetDWordValue("Enabled", uint32(enabled)) - if err != nil { - log.Printf("Unable to set FIPS algorithm policy to original value %v: %v\n", enabled, err) - return - } - log.Printf("Successfully reset FIPS algorithm policy back to original value %v: %v\n", enabled, err) - }, nil -} diff --git a/eng/_util/cmd/selftest/selftest.go b/eng/_util/cmd/selftest/selftest.go new file mode 100644 index 00000000000..3fa3bd8fec5 --- /dev/null +++ b/eng/_util/cmd/selftest/selftest.go @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/microsoft/go-infra/executil" +) + +const description = ` +This command runs the _util self-tests using the stage 0 Go toolchain. +` + +var count = flag.Int("count", -1, "Pass '[...] -count={count}' to test runner. Use '1' to force re-run. Does nothing if negative.") + +func main() { + help := flag.Bool("h", false, "Print this help message.") + + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage of selftest:\n") + flag.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) + } + + flag.Parse() + if *help { + flag.Usage() + return + } + + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} + +func run() error { + stage0Goroot := os.Getenv("STAGE_0_GOROOT") + if stage0Goroot == "" { + return fmt.Errorf("STAGE_0_GOROOT not set") + } + + args := []string{"test", "./..."} + if *count >= 0 { + args = append(args, fmt.Sprintf("-count=%d", *count)) + } + + return executil.Run(executil.Dir( + filepath.Join("eng", "_util"), + filepath.Join(stage0Goroot, "bin", "go"), + args..., + )) +} diff --git a/eng/_util/cmd/sign/README.md b/eng/_util/cmd/sign/README.md new file mode 100644 index 00000000000..56f2794154c --- /dev/null +++ b/eng/_util/cmd/sign/README.md @@ -0,0 +1,50 @@ +# `sign` and the Microsoft build of Go signing infrastructure + +Most of the logic for signing (extracting files, repackaging, creating checksums) is implemented by this `sign` command. + +The [`/eng/signing`](/eng/signing) directory contains the MSBuild project that `sign` invokes to run real signing. +The MSBuild project uses [MicroBuild Signing](https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/650/MicroBuild-Signing) (internal Microsoft wiki link). + +To see signing in action, go to [`/eng/pipeline/README.md`](/eng/pipeline/README.md) and follow the link for `microsoft-go`. + +## Dry run + +1. Create the directory `/eng/signing/tosign` and add the `.tar.gz` and `.zip` artifacts to sign. + * Download artifacts from the `microsoft-go` pipeline, for example. + * It's ok to skip downloading some artifacts. The signing process doesn't require all platforms to be present. + * If you specify `-files`, you can use your own directory. +1. From the root of the repository, run `pwsh eng/run.ps1 sign -n` + +The `-n` argument makes it a dry run: it extracts/repacks files in the same way it would if it were signing them, but no signing is done. +This doesn't involve .NET/MSBuild, so this is a good way for a developer to test changes to the signing logic. + +See `pwsh eng/run.ps1 sign -h` for more options. + +## Test signing + +> [!NOTE] +> Test signing has not been observed to work. +> It has been documented for completeness, in case someone wants to try. + +### Prerequisites + +* Windows +* .NET Core SDK 8.0 or later. + * [Download](https://dot.net/download) +* The signing plugin. + 1. Download the latest NuGet Package: https://devdiv.visualstudio.com/DevDiv/_artifacts/feed/MicroBuildToolset/NuGet/MicroBuild.Plugins.Signing + 1. Extract its contents (the file is a zip) to `%userprofile%\.nuget\packages\microbuild.plugins.signing\1.1.900`. + * Optionally make the versioned dir's name match the version of the package you downloaded. It will be discovered dynamically, as a plugin, whether or not the version matches. + +### Test signing run + +1. Set up `tosign` as described in the dry run section. +1. From the root of the repository, run `pwsh eng/run.ps1 sign` + +## Real signing + +This can't be done from a dev machine. +It occurs in the `microsoft-go` pipeline, on a Windows machine. +See [`/eng/pipeline/README.md`](/eng/pipeline/README.md). + +The invocation of `sign` can be found in [`/eng/pipeline/stages/sign-stage.yml`](/eng/pipeline/stages/sign-stage.yml). diff --git a/eng/_util/cmd/sign/archive.go b/eng/_util/cmd/sign/archive.go new file mode 100644 index 00000000000..b76451d3c6b --- /dev/null +++ b/eng/_util/cmd/sign/archive.go @@ -0,0 +1,535 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "archive/tar" + "archive/zip" + "cmp" + "context" + "errors" + "fmt" + "io" + "io/fs" + "log" + "os" + "path/filepath" + "strings" +) + +type archiveType int + +const ( + // zipArchive is a Windows zip archive. + zipArchive archiveType = iota + // tarGzArchive is a macOS or Linux tar.gz archive. + tarGzArchive +) + +type archive struct { + path string + name string + + archiveType archiveType + archiveMacOS bool + + // workDir is a work dir absolute path that is only used for processing this archive. + workDir string + + // repackedPath is a repackaged archive with signed content. Assigned upon completion. + // Windows and macOS archives get repacked. + repackedPath string + // notarizedPath is a repacked archive that has also had the notarization ticket attached. + // Assigned upon completion. + notarizedPath string +} + +const signingWorkingDirPrefix = "sign-work-" + +func newArchive(p string) (*archive, error) { + name := filepath.Base(p) + a := archive{ + path: p, + name: name, + } + if matchOrPanic("go*.zip", name) { + a.archiveType = zipArchive + } else if matchOrPanic("go*.tar.gz", name) { + a.archiveType = tarGzArchive + } else { + return nil, fmt.Errorf("unknown archive type: %s", p) + } + + if matchOrPanic("go*darwin*.tar.gz", name) { + a.archiveMacOS = true + } + + if err := os.MkdirAll(*tempDir, 0o777); err != nil { + return nil, err + } + workDir, err := os.MkdirTemp(*tempDir, signingWorkingDirPrefix+name) + if err != nil { + return nil, fmt.Errorf("failed to create work directory: %v", err) + } + workDir, err = filepath.Abs(workDir) + if err != nil { + return nil, err + } + a.workDir = workDir + + return &a, nil +} + +// latestPath returns the path of the file that has the most signing steps applied to it. This +// allows for some generalization across platforms in later steps. +func (a *archive) latestPath() string { + if a.notarizedPath != "" { + return a.notarizedPath + } + if a.repackedPath != "" { + return a.repackedPath + } + return a.path +} + +func (a *archive) sigPath() string { + return filepath.Join(a.workDir, a.name+".sig") +} + +func (a *archive) macHardenPackPath() string { + return filepath.Join(a.workDir, a.name+".ToHardenBundle.zip") +} + +func (a *archive) macIndividualNotarizePackPath() string { + return filepath.Join(a.workDir, a.name+".FilesToNotarize.zip") +} + +// entrySignInfo returns signing details for a given file in the Go archive, or nil if the given +// file entry doesn't need to be signed. +func (a *archive) entrySignInfo(name string) *fileToSign { + if a.archiveType == zipArchive { + if strings.HasSuffix(name, ".exe") { + return &fileToSign{ + originalPath: a.path, + fullPath: filepath.Join(a.workDir, "extract", name), + authenticode: "Microsoft400", + } + } + } else if a.archiveMacOS { + if matchOrPanic("go/bin/*", name) || + matchOrPanic("go/pkg/tool/*/*", name) { + + return &fileToSign{ + originalPath: a.path, + zip: true, + } + } + } + return nil +} + +// prepareEntriesToSign extracts files from the archive that need to be signed and returns a list +// of their extracted locations and details about how they should be signed. +func (a *archive) prepareEntriesToSign(ctx context.Context) ([]*fileToSign, error) { + fail := func(err error) ([]*fileToSign, error) { + return nil, fmt.Errorf("failed to extract file from %q: %v", a.path, err) + } + + var results []*fileToSign + + if a.archiveType == zipArchive { + log.Printf("Extracting files to sign from %q", a.path) + zr, err := zip.OpenReader(a.path) + if err != nil { + return fail(err) + } + defer zr.Close() + + if err := eachZipEntry(zr, func(f *zip.File) error { + if err := ctx.Err(); err != nil { + return err + } + if f.FileInfo().IsDir() { + return nil + } + if info := a.entrySignInfo(f.Name); info != nil { + if err := withFileCreate(info.fullPath, func(fWriter *os.File) error { + fReader, err := f.Open() + if err != nil { + return err + } + _, err = io.Copy(fWriter, fReader) + return cmp.Or(err, fReader.Close()) + }); err != nil { + return err + } + results = append(results, info) + } + return nil + }); err != nil { + return fail(err) + } + } else if a.archiveMacOS { + // Store macOS files to sign in a zip. Zipping is needed for this platform specifically, + // and the "Zip=true" feature mentioned in the doc only works when signing on a macOS + // runtime, so we need to do it ourselves. + // https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/19841/Additional-Requirements-for-Signing-or-Notarizing-Mac-Files + fts := &fileToSign{ + originalPath: a.path, + fullPath: a.macHardenPackPath(), + authenticode: "MacDeveloperHarden", + } + log.Printf("Creating macOS file hardening bundle at %q", fts.fullPath) + if err := withZipCreate(fts.fullPath, func(zw *zip.Writer) error { + return a.extractMacOSEntriesToZip(ctx, zw) + }); err != nil { + return fail(err) + } + results = append(results, fts) + } + + return results, nil +} + +func (a *archive) prepareIndividualNotarize(ctx context.Context) ([]*fileToSign, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + if !a.archiveMacOS { + return nil, nil + } + + // Simply send the hardened zip back to the signing service for notarization. + // Copy it first so that we can still access the hardened, pre-notarized files for diagnosis. + if err := copyFile(a.macIndividualNotarizePackPath(), a.macHardenPackPath()); err != nil { + return nil, err + } + + return []*fileToSign{ + { + originalPath: a.path, + fullPath: a.macIndividualNotarizePackPath(), + authenticode: "8020", // Can't specify MacNotarize or MacAppName is not detected. + macAppName: "MicrosoftGo", + }, + }, nil +} + +func (a *archive) extractMacOSEntriesToZip(ctx context.Context, zw *zip.Writer) error { + // Open tar.gz macOS archive to put files into the zip. + writtenNames := make(map[string]struct{}) + return withTarGzOpen(a.path, func(tr *tar.Reader) error { + return eachTarEntry(tr, func(header *tar.Header, r io.Reader) error { + if err := ctx.Err(); err != nil { + return err + } + if header.Typeflag != tar.TypeReg { + return nil + } + if info := a.entrySignInfo(header.Name); info != nil { + if !info.zip { + return fmt.Errorf("unexpected file to sign directly rather than include in the zip batch: %q", header.Name) + } + + base := filepath.Base(header.Name) + if _, ok := writtenNames[base]; ok { + return fmt.Errorf("duplicate file name in archive: %q", base) + } + writtenNames[base] = struct{}{} + + w, err := zw.CreateHeader(&zip.FileHeader{ + Name: base, + }) + if err != nil { + return err + } + _, err = io.Copy(w, r) + return err + } + return nil + }) + }) +} + +func (a *archive) repackSignedEntries(ctx context.Context) error { + targetPath := filepath.Join(a.workDir, a.name+".WithSignedContent") + if a.archiveType == zipArchive { + log.Printf("Repacking signed content to %q", targetPath) + if err := withZipOpen(a.path, func(zr *zip.ReadCloser) error { + return withZipCreate(targetPath, func(zw *zip.Writer) error { + return eachZipEntry(zr, func(f *zip.File) error { + if err := ctx.Err(); err != nil { + return err + } + return a.writeZipRepackEntry(f, zw) + }) + }) + }); err != nil { + return err + } + a.repackedPath = targetPath + } else if a.archiveMacOS { + log.Printf("Repacking hardened content to %q", targetPath) + // Open the original tar.gz for header info and to read unchanged files from. + if err := withTarGzOpen(a.path, func(originalTR *tar.Reader) error { + // Create the new tar.gz that we're assembling. + return withTarGzCreate(targetPath, func(outTW *tar.Writer) error { + // Open the zip payload we got back from the signing service. + return withZipOpen(a.macIndividualNotarizePackPath(), func(zrc *zip.ReadCloser) error { + // Iterate through the original tar.gz file to populate the target. + return eachTarEntry(originalTR, func(hdr *tar.Header, originalR io.Reader) error { + if err := ctx.Err(); err != nil { + return err + } + return a.writeTarRepackEntry(hdr, originalR, &zrc.Reader, outTW) + }) + }) + }) + }); err != nil { + return err + } + a.repackedPath = targetPath + } + return nil +} + +// writeZipRepackEntry looks at one entry in the original zip and creates a corresponding entry in +// the output zip. Reads signed entry content from the signed file on disk. If the file hasn't been +// signed, the content is read from the original zip. +func (a *archive) writeZipRepackEntry(original *zip.File, out *zip.Writer) error { + w, err := out.CreateHeader(&zip.FileHeader{ + // Copy necessary original file metadata. + Name: original.Name, + Method: original.Method, + Comment: original.Comment, + Modified: original.Modified, + Extra: original.Extra, + }) + if err != nil { + return err + } + var r io.ReadCloser + // If we have a signed version of this file, read from that. + // Otherwise, read from the original. + if info := a.entrySignInfo(original.Name); info != nil { + log.Printf("Replacing with signed version: %q", original.Name) + r, err = os.Open(info.fullPath) + if err != nil { + return err + } + } else { + r, err = original.Open() + if err != nil { + return err + } + } + _, err = io.Copy(w, r) + return cmp.Or(err, r.Close()) +} + +// writeTarRepackEntry looks at one entry in the original tar.gz and creates a corresponding entry +// in the output tar.gz. Reads signed/hardened entry content from signedPack. Otherwise, the entry +// content is copied from the original. +func (a *archive) writeTarRepackEntry(hdr *tar.Header, original io.Reader, signedPack *zip.Reader, out *tar.Writer) error { + // Always start with header info from the original tar.gz even if we're going to replace the + // file content. This means we don't need to worry about lost metadata due to the zip + // round-trip. + newHeader := &tar.Header{ + // Follow tar.Header documented compat guidance by copying over our selection of fields. + Name: hdr.Name, + Linkname: hdr.Linkname, + + Size: hdr.Size, + Mode: hdr.Mode, + Uid: hdr.Uid, + Gid: hdr.Gid, + Uname: hdr.Uname, + Gname: hdr.Gname, + + ModTime: hdr.ModTime, + AccessTime: hdr.AccessTime, + ChangeTime: hdr.ChangeTime, + } + isFile := hdr.Typeflag == tar.TypeReg + if info := a.entrySignInfo(hdr.Name); info != nil && isFile { + log.Printf("Replacing with signed version: %q", hdr.Name) + replacementFile, err := signedPack.Open(filepath.Base(hdr.Name)) + if err != nil { + return err + } + defer replacementFile.Close() + // Get the file size to prepare to copy. + stat, err := replacementFile.Stat() + if err != nil { + return err + } + newHeader.Size = stat.Size() + original = replacementFile + } + if err := out.WriteHeader(newHeader); err != nil { + return fmt.Errorf( + "failed to write header for %q: %v", + newHeader.Name, err) + } + if isFile { + _, err := io.Copy(out, original) + if err != nil { + return fmt.Errorf("failed to write %q: %v", newHeader.Name, err) + } + } + // Call Flush to make sure our write was correct. We don't technically need to call Flush here + // because the next WriteHeader will confirm that we e.g. wrote the correct number of bytes. + // However, calling Flush ourselves lets us emit an error that mentions the bad filename + // (rather than the next, unrelated filename). + if err := out.Flush(); err != nil { + return fmt.Errorf("failed to flush %q: %v", newHeader.Name, err) + } + return nil +} + +func (a *archive) prepareBundleNotarize(ctx context.Context) ([]*fileToSign, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + if !a.archiveMacOS { + return nil, nil + } + + // Currently, we don't produce any macOS artifacts that can accept stapled notarization, like + // app bundles, disk images, or installers. + // + // The executable binaries inside our tar.gz archive are already notarized by the earlier + // "MacDeveloperHarden" step, and that's the best we can do. Individual file notarizations are + // not stapled: they are stored by Apple and downloaded on demand. + // + // If we do produce notarizable artifacts in the future, add the logic here to pack them in a + // zip and add logic to unpackBundleNotarize to extract them back out, if zip submission is + // still a MicroBuild and/or ESRP requirement. + return nil, nil +} + +func (a *archive) unpackBundleNotarize(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + + if !a.archiveMacOS { + return nil + } + + return nil +} + +func (a *archive) prepareArchiveSignatures(ctx context.Context) ([]*fileToSign, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + // Copy the archive file to have .sig suffix, e.g. "tar.gz" to "tar.gz.sig". The signing + // process sends the "tar.gz.sig" file to get a signature, then replaces the "tar.gz.sig" + // file's content in-place with the result. We need to preemptively make a renamed copy of the + // file so we end up with both the original file and sig on the machine. + log.Printf("Copying file for signature generation: %q -> %q", a.latestPath(), a.sigPath()) + if err := copyFile(a.sigPath(), a.latestPath()); err != nil { + return nil, err + } + return []*fileToSign{ + { + originalPath: a.path, + fullPath: a.sigPath(), + authenticode: "LinuxSignManagedLanguageCompiler", + }, + }, nil +} + +func (a *archive) copyToDestination(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + // Create destination if it doesn't exist. + if err := os.MkdirAll(*destinationDir, 0o777); err != nil { + return fmt.Errorf("failed to create destination directory: %v", err) + } + + log.Printf("Copying finished files to destination: %q", a.latestPath()) + if err := copyFile(filepath.Join(*destinationDir, a.name), a.latestPath()); err != nil { + return err + } + if err := copyFile(filepath.Join(*destinationDir, a.name+".sig"), a.sigPath()); err != nil { + return err + } + return nil +} + +func consolidateDiagnosticFiles() error { + // Do as much as possible, accumulating errors to report to the user. + var err error + // Signing process logs. + err = errors.Join(err, copyGlobFilesToDir( + *consolidateDiagDir, + filepath.Join(*signingCsprojDir, "*.log"), + filepath.Join(*tempDir, "*.binlog"), + filepath.Join(*tempDir, "*.props"), + )) + // .NET diag data, for package versions etc. + err = errors.Join(err, copyGlobFilesToDir( + filepath.Join(*consolidateDiagDir, "obj"), + filepath.Join(*signingCsprojDir, "obj", "*"), + )) + + // Signing working dirs. These contain the files actually sent to sign. Make + // it clear that they're not production-ready files through the filename. + cleanTempDir := filepath.Clean(*tempDir) + err = errors.Join(err, withTarGzCreate( + filepath.Join(*consolidateDiagDir, "sign-work-archives.nonproduction.tar.gz"), + func(tw *tar.Writer) error { + if err := filepath.WalkDir(cleanTempDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + // At the top level, only walk into working dirs. + if d.IsDir() && + filepath.Dir(path) == cleanTempDir && + !strings.HasPrefix(d.Name(), signingWorkingDirPrefix) { + + return filepath.SkipDir + } + // Inside a given working dir, we are free to walk recursively. + if d.IsDir() { + return nil + } + // This is a file: archive it. + df, err := d.Info() + if err != nil { + return err + } + // Basic sanity checks. + if !filepath.IsLocal(path) { + return fmt.Errorf("path %q is not a local path", path) + } + tarPath, ok := strings.CutPrefix(path, cleanTempDir+string(filepath.Separator)) + if !ok { + return fmt.Errorf("path %q is not under temp dir %q", path, *tempDir) + } + tarPath = filepath.ToSlash(tarPath) + + err = tw.WriteHeader(&tar.Header{ + Name: tarPath, + Size: df.Size(), + Mode: 0o644, + }) + if err != nil { + return err + } + return copyFromFile(tw, path) + }); err != nil { + return err + } + return nil + }, + )) + return err +} diff --git a/eng/_util/cmd/sign/archiveutil.go b/eng/_util/cmd/sign/archiveutil.go new file mode 100644 index 00000000000..2abe41dd7b7 --- /dev/null +++ b/eng/_util/cmd/sign/archiveutil.go @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "archive/tar" + "archive/zip" + "cmp" + "compress/gzip" + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +func eachZipEntry(r *zip.ReadCloser, f func(*zip.File) error) error { + for _, file := range r.File { + // Disallow absolute path, "..", etc. + if !filepath.IsLocal(file.Name) { + return fmt.Errorf("zip contains non-local path: %s", file.Name) + } + if err := f(file); err != nil { + return err + } + } + return nil +} + +func eachTarEntry(r *tar.Reader, f func(*tar.Header, io.Reader) error) error { + for { + header, err := r.Next() + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + // Disallow absolute path, "..", etc. + if !filepath.IsLocal(header.Name) { + return fmt.Errorf("tar contains non-local path: %s", header.Name) + } + if err := f(header, r); err != nil { + return err + } + } +} + +func withFileOpen(path string, f func(*os.File) error) error { + file, err := os.Open(path) + if err != nil { + return err + } + return cmp.Or(f(file), file.Close()) +} + +func withZipOpen(path string, f func(*zip.ReadCloser) error) error { + r, err := zip.OpenReader(path) + if err != nil { + return err + } + return cmp.Or(f(r), r.Close()) +} + +func withTarGzOpen(path string, f func(*tar.Reader) error) error { + return withFileOpen(path, func(file *os.File) error { + gz, err := gzip.NewReader(file) + if err != nil { + return err + } + r := tar.NewReader(gz) + return f(r) + }) +} + +func withFileCreate(path string, f func(*os.File) error) error { + if err := os.MkdirAll(filepath.Dir(path), 0o777); err != nil { + return err + } + file, err := os.Create(path) + if err != nil { + return err + } + return cmp.Or(f(file), file.Close()) +} + +func withZipCreate(path string, f func(*zip.Writer) error) error { + return withFileCreate(path, func(file *os.File) error { + w := zip.NewWriter(file) + return cmp.Or(f(w), w.Close()) + }) +} + +func withTarGzCreate(path string, f func(*tar.Writer) error) error { + return withFileCreate(path, func(file *os.File) error { + gzw, err := gzip.NewWriterLevel(file, gzip.BestCompression) + if err != nil { + return err + } + tw := tar.NewWriter(gzw) + return cmp.Or(f(tw), tw.Close(), gzw.Close()) + }) +} + +func copyFile(dst, src string) error { + f, err := os.Open(src) + if err != nil { + return err + } + return cmp.Or(copyToFile(dst, f), f.Close()) +} + +func copyToFile(path string, r io.Reader) error { + if err := os.MkdirAll(filepath.Dir(path), 0o777); err != nil { + return err + } + f, err := os.Create(path) + if err != nil { + return err + } + _, err = io.Copy(f, r) + return cmp.Or(err, f.Close()) +} + +func copyFromFile(w io.Writer, path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + _, err = io.Copy(w, f) + return cmp.Or(err, f.Close()) +} + +func copyGlobFilesToDir(dir string, globs ...string) error { + for _, glob := range globs { + files, err := filepath.Glob(glob) + if err != nil { + return err + } + for _, f := range files { + if err := copyFile(filepath.Join(dir, filepath.Base(f)), f); err != nil { + return err + } + } + } + return nil +} + +// matchOrPanic returns whether name matches the pattern glob, or panics if pattern is invalid. +func matchOrPanic(pattern, name string) bool { + ok, err := filepath.Match(pattern, name) + if err != nil { + panic(err) + } + return ok +} diff --git a/eng/_util/cmd/sign/sign.go b/eng/_util/cmd/sign/sign.go new file mode 100644 index 00000000000..c0f5332dfe5 --- /dev/null +++ b/eng/_util/cmd/sign/sign.go @@ -0,0 +1,342 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "context" + "crypto/sha256" + "errors" + "flag" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/microsoft/go/_util/internal/checksum" +) + +const description = ` +This command signs build artifacts using MicroBuild. It is used in the Microsoft build of Go build pipeline. +Use '-n' to test the command locally. + +Signs in multiple passes. Some steps only apply to certain types of archives: + +1. Archive entries. Extracts specific entries from inside each archive, signs, and repacks. +2. Notarize. macOS archives get a notarization ticket attached to the tar.gz. +3. Signatures. Creates sig files for each archive. +4. Locally creates a .sha256 file for each archive. + +See /eng/_util/cmd/sign/README.md for more information. +` + +var ( + filesGlob = flag.String("files", "eng/signing/tosign/*", "Glob of Go archives to sign.") + destinationDir = flag.String("o", "eng/signing/signed", "Directory to store signed files.") + consolidateDiagDir = flag.String("consolidate-diag-dir", "eng/signing/diag", "Directory to store consolidated diagnostic files.") + tempDir = flag.String("temp-dir", "eng/signing/signing-temp", "Directory to store temporary files.") + signingCsprojDir = flag.String("signing-csproj-dir", "eng/signing", "Directory containing Sign.csproj and related files.") + + signType = flag.String("sign-type", "test", + "Type of signing to perform. Options: test, real. "+ + // https://github.com/microsoft/go-lab/issues/231 + "Test signing skips using MicroBuild tooling because it throws exception 'The test signing method for cert (8020) has NOT been implemented.'") + + timeout = flag.Duration("timeout", 0, + "Timeout for signing operations. Zero means no timeout. "+ + "Any MSBuild processes launched by this tool are be manually killed. "+ + "If set to a value lower than AzDO pipeline timeout, this helps avoid pipeline breakage when uploading MSBuild outputs.") + dryRun = flag.Bool("n", false, "Dry run: don't run the MSBuild signing tooling at all, even in test mode. This works on non-Windows platforms.") +) + +func main() { + help := flag.Bool("h", false, "Print this help message.") + + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n") + flag.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) + } + + flag.Parse() + if *help { + flag.Usage() + return + } + + if err := run(); err != nil { + log.Printf("error: %v", err) + os.Exit(1) + } +} + +func run() (err error) { + // A context for timeout. This timeout is mainly here to make sure child MSBuild processes are + // terminated. There are some ctx.Err() checks sprinkled into the Go code, but canceling + // quickly during the packaging/repackaging work in Go is not currently important: the Go work + // takes an insignificant amount of time compared to the signing service calls in MSBuild. + var ctx context.Context + if *timeout == 0 { + ctx = context.Background() + } else { + var cancel context.CancelFunc + ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(*timeout)) + defer cancel() + } + + defer func() { + log.Println("Consolidating diagnostic files") + log.Printf("Consolidated diagnostic directory: %q", *consolidateDiagDir) + + consolidateErr := consolidateDiagnosticFiles() + err = errors.Join(err, consolidateErr) + }() + + archives, err := findArchives(ctx, *filesGlob) + if err != nil { + return err + } + + log.Println("Signing individual files extracted from archives") + + individualFilesToSign, err := flatMapSlice(archives, func(a *archive) ([]*fileToSign, error) { + return a.prepareEntriesToSign(ctx) + }) + if err != nil { + return err + } + + if err := sign(ctx, "1-Individual", individualFilesToSign); err != nil { + return err + } + + log.Println("Notarizing macOS individual files") + + individualFilesToNotarize, err := flatMapSlice(archives, func(a *archive) ([]*fileToSign, error) { + return a.prepareIndividualNotarize(ctx) + }) + if err != nil { + return err + } + + if err := sign(ctx, "2-Notarize-Individual", individualFilesToNotarize); err != nil { + return err + } + + for _, a := range archives { + if err := a.repackSignedEntries(ctx); err != nil { + return err + } + } + + log.Println("Notarizing macOS bundles") + + filesToNotarize, err := flatMapSlice(archives, func(a *archive) ([]*fileToSign, error) { + return a.prepareBundleNotarize(ctx) + }) + if err != nil { + return err + } + + if err := sign(ctx, "3-Notarize-Bundles", filesToNotarize); err != nil { + return err + } + + for _, a := range archives { + if err := a.unpackBundleNotarize(ctx); err != nil { + return err + } + } + + log.Println("Creating signature files") + + signatureFiles, err := flatMapSlice(archives, func(a *archive) ([]*fileToSign, error) { + return a.prepareArchiveSignatures(ctx) + }) + if err != nil { + return err + } + + if err := sign(ctx, "4-Sigs", signatureFiles); err != nil { + return err + } + + log.Println("Copying finished files to destination") + log.Printf("Destination directory: %q", *destinationDir) + + for _, a := range archives { + if err := a.copyToDestination(ctx); err != nil { + return err + } + } + + log.Println("Generating checksum files") + + for _, a := range archives { + if err := checksum.WriteSHA256ChecksumFile(filepath.Join(*destinationDir, a.name)); err != nil { + return err + } + } + + return nil +} + +func findArchives(ctx context.Context, glob string) ([]*archive, error) { + files, err := filepath.Glob(glob) + if err != nil { + return nil, fmt.Errorf("failed to glob files: %v", err) + } + + archives := make([]*archive, 0, len(files)) + + // Check for duplicate filenames. At the end of signing, we will put all the results in the + // same directory (even if the sources came from different directories), so catching this + // early saves time. + // + // Use lowercase because we sign on a Windows machine with a case-insensitive filesystem. + archiveFilenames := make(map[string]string) + + for _, f := range files { + if err := ctx.Err(); err != nil { + return nil, err + } + // Ignore checksum files: we always generate new ones. + if strings.HasSuffix(f, ".sha256") { + continue + } + + filenameLower := strings.ToLower(filepath.Base(f)) + if existingF, ok := archiveFilenames[filenameLower]; ok { + return nil, fmt.Errorf("duplicate archive %q, already found %q (comparing lowercase filename)", f, existingF) + } + archiveFilenames[filenameLower] = f + + a, err := newArchive(f) + if err != nil { + return nil, fmt.Errorf("failed to process %q: %v", f, err) + } + archives = append(archives, a) + } + + if len(archives) == 0 { + return nil, fmt.Errorf("no archives found to sign matching glob %q", *filesGlob) + } + + return archives, nil +} + +func sign(ctx context.Context, step string, files []*fileToSign) error { + if len(files) == 0 { + log.Printf("No files to sign for step %q", step) + return nil + } + + var sb strings.Builder + sb.WriteString("\n") + sb.WriteString(" \n") + for _, f := range files { + f.WriteMSBuildItem(&sb) + } + sb.WriteString(" \n") + sb.WriteString("\n") + + log.Printf("Signing with props file content:\n%s\n", sb.String()) + if *dryRun { + log.Printf("Dry run: skipping signing.") + return nil + } + if *signType == "test" { + log.Printf("Testing signing: skipping MicroBuild tooling.") + for _, f := range files { + if strings.HasSuffix(f.fullPath, ".sig") { + log.Printf("Replacing file with placeholder content to reduce size and simulate signature: %q", f.fullPath) + // Get a checksum to make the file content unique. Otherwise, + // publishing rejects the repeated file content. + data, err := os.ReadFile(f.fullPath) + if err != nil { + return fmt.Errorf("failed to read file %q: %v", f.fullPath, err) + } + checksum := fmt.Sprintf("%x", sha256.Sum256(data)) + if err := os.WriteFile( + f.fullPath, + fmt.Appendf(nil, "This is a placeholder test signature file. Original content checksum: %v\n", checksum), + 0o666, + ); err != nil { + return fmt.Errorf("failed to write test signature file %q: %v", f.fullPath, err) + } + } + } + return nil + } + + if err := os.MkdirAll(*tempDir, 0o777); err != nil { + return err + } + // Get an absolute path to pass to MSBuild, because our working dirs may not be the same. + // MSBuild in general will resolve paths relative to the csproj. + absTemp, err := filepath.Abs(*tempDir) + if err != nil { + return err + } + propsFilePath := filepath.Join(absTemp, "Sign"+step+".props") + if err := os.WriteFile(propsFilePath, []byte(sb.String()), 0o666); err != nil { + return err + } + + cmd := exec.CommandContext( + ctx, + "dotnet", "build", "Sign.csproj", + "/p:SignFilesDir="+absTemp, + "/p:FilesToSignPropsFile="+propsFilePath, + "/t:AfterBuild", + "/p:SignType="+*signType, + "/bl:"+filepath.Join(absTemp, "Sign"+step+".binlog"), + "/v:n", + ) + cmd.Dir = *signingCsprojDir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + log.Printf("Running: %v", cmd) + return cmd.Run() +} + +type fileToSign struct { + originalPath string + fullPath string + authenticode string + // This file is part of a zip payload, e.g. for macOS hardening. + zip bool + // macAppName for notarization. + macAppName string +} + +func (f *fileToSign) WriteMSBuildItem(w io.Writer) { + fmt.Fprintf(w, " \n") +} + +// flatMapSlice sequentially maps each element of es to a slice using f and flattens the resulting +// slices. If any call to f returns an error, the error is returned immediately. +func flatMapSlice[E, R any](es []E, f func(E) ([]R, error)) ([]R, error) { + var results []R + for _, e := range es { + rs, err := f(e) + if err != nil { + return nil, err + } + results = append(results, rs...) + } + return results, nil +} diff --git a/eng/_util/cmd/submodule-refresh/submodule-refresh.go b/eng/_util/cmd/submodule-refresh/submodule-refresh.go new file mode 100644 index 00000000000..a4c62bbb947 --- /dev/null +++ b/eng/_util/cmd/submodule-refresh/submodule-refresh.go @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "log" + "os" + "path/filepath" + + "github.com/microsoft/go-infra/patch" + "github.com/microsoft/go-infra/submodule" +) + +const description = ` +This command refreshes the Go submodule: initializes it, resets the content, and +applies patches to the stage by default, or optionally as commits. +` + +var ( + commits = flag.Bool("commits", false, "Apply the patches as commits.") + skipPatch = flag.Bool("skip-patch", false, "Skip applying patches.") + take = flag.Int("take", -1, "Only apply the first N patches. -1 means apply all.") + origin = flag.String("origin", "", "Use this origin instead of the default defined in '.gitmodules' to fetch the repository.") + shallow = flag.Bool("shallow", false, "Clone the submodule with depth 1.") + fetchBearerToken = flag.String("fetch-bearer-token", "", "Use this bearer token to fetch the submodule repository.") +) + +func main() { + repoRootDir, err := os.Getwd() + if err != nil { + panic(err) + } + + help := flag.Bool("h", false, "Print this help message.") + + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n") + flag.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) + } + + flag.Parse() + if *help { + flag.Usage() + return + } + + if err := refresh(repoRootDir); err != nil { + panic(err) + } +} + +func refresh(rootDir string) error { + if err := submodule.Init(rootDir, *origin, *fetchBearerToken, *shallow); err != nil { + return err + } + + config, err := patch.FindAncestorConfig(rootDir) + if err != nil { + return err + } + + if err := submodule.Reset(rootDir, filepath.Join(config.RootDir, config.SubmoduleDir), true); err != nil { + return err + } + + if *skipPatch { + return nil + } + + mode := patch.ApplyModeIndex + if *commits { + mode = patch.ApplyModeCommits + } + + if *take >= 0 { + // The patch API applies all patches in a directory. To apply only the + // first N, copy them into a temporary directory and point the config there. + tmpDirRelative := filepath.Join("eng", "artifacts", "submodule-refresh", "patch-subset") + tmpDir := filepath.Join(config.RootDir, tmpDirRelative) + if err := os.RemoveAll(tmpDir); err != nil { + return err + } + if err := os.MkdirAll(tmpDir, 0o777); err != nil { + return err + } + i := 0 + if err := patch.WalkGoPatches(config, func(path string) error { + if i >= *take { + log.Printf("Not including patch %q\n", path) + return nil + } + i++ + log.Printf("Taking patch %q\n", path) + return copyFile(path, filepath.Join(tmpDir, filepath.Base(path))) + }); err != nil { + return err + } + if *take > i { + return fmt.Errorf("-take %d exceeds number of patches (%d)", *take, i) + } + config.PatchesDir = tmpDirRelative + } + + if err := patch.Apply(config, mode); err != nil { + return err + } + return nil +} + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return err + } + _, copyErr := io.Copy(out, in) + return errors.Join(copyErr, out.Close()) +} diff --git a/eng/_util/cmd/updatecryptodocs/docs.go b/eng/_util/cmd/updatecryptodocs/docs.go new file mode 100644 index 00000000000..65ff3207e3f --- /dev/null +++ b/eng/_util/cmd/updatecryptodocs/docs.go @@ -0,0 +1,865 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import _ "embed" + +//go:embed header.md +var header string + +type SupportStatus int + +const ( + Supported SupportStatus = iota + NotSupported + Warn + N_A +) + +type PlatformStatus struct { + Supported SupportStatus + MinGoVersion string + Notes []string + MinVersion string +} + +type Item struct { + Name string + MinGoVersion string + Notes []string + Platforms Platforms +} + +var SupportedPlatforms = []string{"windows", "linux", "macos"} + +type Platforms struct { + Windows PlatformStatus + Linux PlatformStatus + MacOS PlatformStatus +} + +func (s Platforms) Get(platform string) PlatformStatus { + switch platform { + case "windows": + return s.Windows + case "linux": + return s.Linux + case "macos": + return s.MacOS + default: + panic("unknown platform: " + platform) + } +} + +type Section struct { + Title string + ShortTitle string + ColumnHeader string + Packages []string + Description string + DescriptionParagraphs []string + MinGoVersion string + Items []Item + Subsections []Section + Footnotes []string + Footer string + FooterParagraphs []string +} + +type Document struct { + Sections []Section +} + +var doc = Document{ + Sections: []Section{ + { + Title: "Hash and Message Authentication Algorithms", + Packages: []string{ + "crypto/md5", + "crypto/sha1", + "crypto/sha256", + "crypto/sha512", + "crypto/sha3", + "crypto/hmac", + }, + Items: []Item{ + {Name: "MD5"}, + {Name: "SHA-1"}, + { + Name: "SHA-2-224", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + {Name: "SHA-2-256"}, + {Name: "SHA-2-384"}, + {Name: "SHA-2-512"}, + { + Name: "SHA-2-512_224", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + Linux: PlatformStatus{MinVersion: "1.1.1"}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "SHA-2-512_256", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + Linux: PlatformStatus{MinVersion: "1.1.1"}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "SHA-3-224", + MinGoVersion: "1.26", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + Linux: PlatformStatus{MinVersion: "1.1.1"}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "SHA-3-256", + MinGoVersion: "1.26", + Platforms: Platforms{ + Windows: PlatformStatus{MinVersion: "11 (24H2)"}, + Linux: PlatformStatus{MinVersion: "1.1.1"}, + MacOS: PlatformStatus{MinVersion: "26"}, + }, + }, + { + Name: "SHA-3-384", + MinGoVersion: "1.26", + Platforms: Platforms{ + Windows: PlatformStatus{MinVersion: "11 (24H2)"}, + Linux: PlatformStatus{MinVersion: "1.1.1"}, + MacOS: PlatformStatus{MinVersion: "26"}, + }, + }, + { + Name: "SHA-3-512", + MinGoVersion: "1.26", + Platforms: Platforms{ + Windows: PlatformStatus{MinVersion: "11 (24H2)"}, + Linux: PlatformStatus{MinVersion: "1.1.1"}, + MacOS: PlatformStatus{MinVersion: "26"}, + }, + }, + { + Name: "SHAKE-128", + Platforms: Platforms{ + Linux: PlatformStatus{MinVersion: "3.3"}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "SHAKE-256", + Platforms: Platforms{ + Linux: PlatformStatus{MinVersion: "3.3"}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "CSHAKE-128", + Platforms: Platforms{ + Linux: PlatformStatus{Supported: NotSupported}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "CSHAKE-256", + Platforms: Platforms{ + Linux: PlatformStatus{Supported: NotSupported}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "HMAC", + Notes: []string{ + "Supports only hash algorithms that are supported as standalone hash functions.", + }, + }, + }, + }, + { + Title: "Symmetric encryption", + ColumnHeader: "Cipher + Mode", + Packages: []string{"crypto/aes", "crypto/cipher", "crypto/des", "crypto/rc4"}, + FooterParagraphs: []string{ + "- Key Sizes", + " AES-GCM works with 128, 192, and 256-bit keys.", + "- Nonce Sizes", + " AES-GCM works with 12-byte nonces.", + "- Tag Sizes", + " AES-GCM works with 16-byte tags.", + }, + Items: []Item{ + {Name: "AES-ECB"}, + {Name: "AES-CBC"}, + { + Name: "AES-CTR", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "AES-CFB", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + Linux: PlatformStatus{Supported: NotSupported}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "AES-OFB", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + Linux: PlatformStatus{Supported: NotSupported}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "AES-GCM", + Notes: []string{"AES-GCM supports specific keys, nonces, and tags:"}, + }, + { + Name: "DES-CBC", + Platforms: Platforms{ + Linux: PlatformStatus{ + Supported: Warn, + Notes: []string{ + "When using OpenSSL 3, requires the legacy provider to be enabled.", + }, + }, + }, + }, + { + Name: "DES-ECB", + Platforms: Platforms{ + Linux: PlatformStatus{ + Supported: Warn, + Notes: []string{ + "When using OpenSSL 3, requires the legacy provider to be enabled.", + }, + }, + }, + }, + {Name: "3DES-ECB"}, + {Name: "3DES-CBC"}, + { + Name: "RC4", + Platforms: Platforms{ + Linux: PlatformStatus{ + Supported: Warn, + Notes: []string{ + "When using OpenSSL 3, requires the legacy provider to be enabled.", + }, + }, + }, + }, + }, + }, + { + Title: "Asymmetric encryption", + Packages: []string{"RSA", "ECDSA", "ECDH", "Ed25519", "DSA"}, + Description: "", + Subsections: []Section{ + { + Title: "RSA", + ColumnHeader: "Padding Mode", + Packages: []string{"crypto/rsa"}, + DescriptionParagraphs: []string{ + "Multi-prime RSA keys are not supported.", + "The RSA key size is subject to the limitations of the underlying cryptographic library.\n" + + "For example, on some Windows and SCOSSL configurations, the key size should be multiple of 8.\n" + + "Please refer to the documentation of the underlying cryptographic library for the specific limitations.", + "Operations that require random numbers (rand io.Reader) only support [rand.Reader](https://pkg.go.dev/crypto/rand#Reader).", + }, + Items: []Item{ + { + Name: "OAEP (MD5)", + Platforms: Platforms{ + MacOS: PlatformStatus{ + Notes: []string{ + "macOS doesn't support passing a custom label to OAEP functions.", + }, + }, + }, + }, + { + Name: "OAEP (SHA-1)", + Platforms: Platforms{ + MacOS: PlatformStatus{ + Notes: []string{ + "macOS doesn't support passing a custom label to OAEP functions.", + }, + }, + }, + }, + { + Name: "OAEP (SHA-2)", + Notes: []string{ + "Supports only hash algorithms that are [supported as standalone hash functions](#hash-and-message-authentication-algorithms).", + }, + Platforms: Platforms{ + MacOS: PlatformStatus{ + Notes: []string{ + "macOS doesn't support passing a custom label to OAEP functions.", + }, + }, + }, + }, + { + Name: "OAEP (SHA-3)", + Notes: []string{ + "Supports only hash algorithms that are [supported as standalone hash functions](#hash-and-message-authentication-algorithms).", + }, + MinGoVersion: "1.26", + Platforms: Platforms{ + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "PSS (MD5)", + Platforms: Platforms{ + Windows: PlatformStatus{ + Notes: []string{ + "Verifying PSS signatures with [rsa.PSSSaltLengthAuto](https://pkg.go.dev/crypto/rsa#pkg-constants) is not supported.", + }, + }, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "PSS (SHA-1)", + Platforms: Platforms{ + Windows: PlatformStatus{ + Notes: []string{ + "Verifying PSS signatures with [rsa.PSSSaltLengthAuto](https://pkg.go.dev/crypto/rsa#pkg-constants) is not supported.", + }, + }, + MacOS: PlatformStatus{ + Notes: []string{ + "Custom salt lengths are not supported. PSS always uses the [`rsa.PSSSaltLengthEqualsHash`](https://pkg.go.dev/crypto/rsa#pkg-constants).", + }, + }, + }, + }, + { + Name: "PSS (SHA-2)", + Notes: []string{ + "Supports only hash algorithms that are [supported as standalone hash functions](#hash-and-message-authentication-algorithms).", + }, + Platforms: Platforms{ + Windows: PlatformStatus{ + Notes: []string{ + "Verifying PSS signatures with [rsa.PSSSaltLengthAuto](https://pkg.go.dev/crypto/rsa#pkg-constants) is not supported.", + }, + }, + MacOS: PlatformStatus{ + Notes: []string{ + "Custom salt lengths are not supported. PSS always uses the [`rsa.PSSSaltLengthEqualsHash`](https://pkg.go.dev/crypto/rsa#pkg-constants).", + }, + }, + }, + }, + { + Name: "PSS (SHA-3)", + Notes: []string{ + "Supports only hash algorithms that are [supported as standalone hash functions](#hash-and-message-authentication-algorithms).", + }, + Platforms: Platforms{ + Windows: PlatformStatus{MinGoVersion: "1.26"}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + {Name: "PKCS1v15 Signature (Unhashed)"}, + { + Name: "PKCS1v15 Signature (RIPMED160)", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "PKCS1v15 Signature (MD5)", + Platforms: Platforms{ + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "PKCS1v15 Signature (MD5-SHA1)", + Platforms: Platforms{ + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + {Name: "PKCS1v15 Signature (SHA-1)"}, + { + Name: "PKCS1v15 Signature (SHA-2)", + Notes: []string{ + "Supports only hash algorithms that are [supported as standalone hash functions](#hash-and-message-authentication-algorithms).", + }, + }, + { + Name: "PKCS1v15 Signature (SHA-3)", + Platforms: Platforms{ + Windows: PlatformStatus{MinVersion: "11 (24H2)", MinGoVersion: "1.26"}, + Linux: PlatformStatus{MinVersion: "1.1.1"}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + }, + }, + { + Title: "ECDSA", + ColumnHeader: "Elliptic Curve", + Packages: []string{"crypto/ecdsa", "crypto/elliptic"}, + Description: "Operations that require random numbers (rand io.Reader) only support [rand.Reader](https://pkg.go.dev/crypto/rand#Reader).", + Items: []Item{ + { + Name: "NIST P-224 (secp224r1)", + Platforms: Platforms{ + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + {Name: "NIST P-256 (secp256r1)"}, + {Name: "NIST P-384 (secp384r1)"}, + {Name: "NIST P-521 (secp521r1)"}, + }, + }, + { + Title: "ECDH", + ColumnHeader: "Elliptic Curve", + Packages: []string{"crypto/ecdh"}, + Description: "Operations that require random numbers (rand io.Reader) only support [rand.Reader](https://pkg.go.dev/crypto/rand#Reader).", + Items: []Item{ + { + Name: "NIST P-224 (secp224r1)", + Platforms: Platforms{ + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + {Name: "NIST P-256 (secp256r1)"}, + {Name: "NIST P-384 (secp384r1)"}, + {Name: "NIST P-521 (secp521r1)"}, + { + Name: "X25519 (curve25519)", + MinGoVersion: "1.26", + Platforms: Platforms{ + Linux: PlatformStatus{MinVersion: "1.1.1"}, + }, + }, + }, + }, + { + Title: "Ed25519", + ColumnHeader: "Schemes", + Packages: []string{"crypto/ed25519"}, + Description: "Operations that require random numbers (rand io.Reader) only support [rand.Reader](https://pkg.go.dev/crypto/rand#Reader).", + Items: []Item{ + { + Name: "Ed25519", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "Ed25519ctx", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + Linux: PlatformStatus{Supported: NotSupported}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "Ed25519ph", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + Linux: PlatformStatus{Supported: NotSupported}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + }, + }, + { + Title: "DSA", + ColumnHeader: "Parameters", + Items: []Item{ + { + Name: "L1024N160", + Platforms: Platforms{ + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "L2048N224", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "L2048N256", + Platforms: Platforms{ + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "L3072N256", + Platforms: Platforms{ + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + }, + }, + }, + }, + { + Title: "Key derivation functions (KDFs)", + ColumnHeader: "Functions", + Packages: []string{"crypto/hkdf", "crypto/pbkdf2"}, + Items: []Item{ + { + Name: "PBKDF2", + Notes: []string{ + "Supports only hash algorithms that are [supported as standalone hash functions](#hash-and-message-authentication-algorithms).", + }, + }, + { + Name: "HKDF", + Notes: []string{ + "Supports only hash algorithms that are [supported as standalone hash functions](#hash-and-message-authentication-algorithms).", + }, + }, + }, + }, + { + Title: "Key Encapsulation Mechanisms (KEMs)", + Packages: []string{"ML-KEM"}, + Subsections: []Section{ + { + Title: "ML-KEM", + ColumnHeader: "Parameters", + Packages: []string{"crypto/mlkem"}, + MinGoVersion: "1.26", + Items: []Item{ + { + Name: "768", + Platforms: Platforms{ + Windows: PlatformStatus{MinVersion: "11 (24H2)"}, + Linux: PlatformStatus{MinVersion: "3.5.0"}, + MacOS: PlatformStatus{MinVersion: "26"}, + }, + }, + { + Name: "1024", + Platforms: Platforms{ + Windows: PlatformStatus{MinVersion: "11 (24H2)"}, + Linux: PlatformStatus{MinVersion: "3.5.0"}, + MacOS: PlatformStatus{MinVersion: "26"}, + }, + }, + }, + }, + }, + }, + { + Title: "Higher-level protocols", + Packages: []string{"HPKE", "TLS"}, + DescriptionParagraphs: []string{ + "High-level protocols are algorithms that combine multiple cryptographic primitives to provide a specific functionality,\n" + + "such as TLS or Hybrid Public Key Encryption (HPKE).", + "These protocols are implemented using native Go code, but they rely on the underlying OS cryptographic libraries for the cryptographic operations.", + "This section includes the following subsections:", + }, + Subsections: []Section{ + { + Title: "Hybrid Public Key Encryption (HPKE)", + Packages: []string{"AEAD Functions", "KDFs", "KEMs", "crypto/hpke"}, + Subsections: []Section{ + { + Title: "HPKE Authenticated Encryption with Associated Data (AEAD) Functions", + ColumnHeader: "Functions", + ShortTitle: "AEAD Functions", + Items: []Item{ + {Name: "AES-128-GCM"}, + {Name: "AES-256-GCM"}, + { + Name: "ChaCha20Poly1305", + MinGoVersion: "1.26", + }, + { + Name: "Export-only", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: N_A}, + Linux: PlatformStatus{Supported: N_A}, + MacOS: PlatformStatus{Supported: N_A}, + }, + }, + }, + }, + { + Title: "HPKE Key Derivation Functions (KDFs)", + ColumnHeader: "Functions", + ShortTitle: "KDFs", + Items: []Item{ + {Name: "HKDF-SHA256"}, + {Name: "HKDF-SHA384"}, + {Name: "HKDF-SHA512"}, + }, + }, + { + Title: "HPKE Key Encapsulation Mechanisms (KEMs)", + ColumnHeader: "Functions", + ShortTitle: "KEMs", + Items: []Item{ + {Name: "DHKEM(P-256, HKDF-SHA256)"}, + {Name: "DHKEM(P-384, HKDF-SHA384)"}, + {Name: "DHKEM(P-521, HKDF-SHA512)"}, + { + Name: "DHKEM(X25519, HKDF-SHA256)", + Notes: []string{"See the [X25519](#ecdh) section for requirements."}, + }, + { + Name: "ML-KEM-768", + Notes: []string{ + "See the [ML-KEM](#ml-kem) section for requirements.", + }, + }, + { + Name: "ML-KEM-1024", + Notes: []string{ + "See the [ML-KEM](#ml-kem) section for requirements.", + }, + }, + { + Name: "MLKEM768-P256", + Notes: []string{ + "See the [ML-KEM](#ml-kem) section for requirements.", + }, + }, + { + Name: "MLKEM1024-P384", + Notes: []string{ + "See the [ML-KEM](#ml-kem) section for requirements.", + }, + }, + { + Name: "MLKEM768-X25519", + Notes: []string{ + "See the [X25519](#ecdh) section for requirements.", + "See the [ML-KEM](#ml-kem) section for requirements.", + }, + }, + }, + }, + }, + }, + { + Title: "TLS", + Packages: []string{ + "TLS Versions", + "TLS Cipher Suites", + "TLS Curves and Groups", + "TLS Signature Schemes", + "crypto/tls", + }, + Subsections: []Section{ + { + Title: "TLS Versions", + ColumnHeader: "Version", + Description: "The TLS stack is implemented using native Go code but the crypto primitives are provided by the system cryptographic libraries.", + Items: []Item{ + { + Name: "SSL 3.0", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + Linux: PlatformStatus{Supported: NotSupported}, + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "TLS 1.0", + Platforms: Platforms{ + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + { + Name: "TLS 1.1", + Platforms: Platforms{ + MacOS: PlatformStatus{Supported: NotSupported}, + }, + }, + {Name: "TLS 1.2"}, + {Name: "TLS 1.3"}, + }, + }, + { + Title: "TLS Cipher Suites", + ColumnHeader: "Name", + Footer: "On Windows, it is possible to restrict and reorder the cipher suites following the " + + "[Schannel preferences](https://learn.microsoft.com/en-us/windows/win32/secauthn/cipher-suites-in-schannel) " + + "by building with the `ms_tls_config_schannel` goexperiment enabled.", + Items: []Item{ + { + Name: "TLS_RSA_WITH_RC4_128_SHA", + Platforms: Platforms{ + Linux: PlatformStatus{ + Supported: Warn, + Notes: []string{ + "When using OpenSSL 3, requires the legacy provider to be enabled.", + }, + }, + }, + }, + { + Name: "TLS_RSA_WITH_3DES_EDE_CBC_SHA", + Platforms: Platforms{ + Linux: PlatformStatus{ + Supported: Warn, + Notes: []string{ + "When using OpenSSL 3, requires the legacy provider to be enabled.", + }, + }, + }, + }, + {Name: "TLS_RSA_WITH_AES_128_CBC_SHA"}, + {Name: "TLS_RSA_WITH_AES_256_CBC_SHA"}, + {Name: "TLS_RSA_WITH_AES_128_CBC_SHA256"}, + {Name: "TLS_RSA_WITH_AES_128_GCM_SHA256"}, + {Name: "TLS_RSA_WITH_AES_256_GCM_SHA384"}, + { + Name: "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA", + Platforms: Platforms{ + Linux: PlatformStatus{ + Supported: Warn, + Notes: []string{ + "When using OpenSSL 3, requires the legacy provider to be enabled.", + }, + }, + }, + }, + {Name: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA"}, + {Name: "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA"}, + { + Name: "TLS_ECDHE_RSA_WITH_RC4_128_SHA", + Platforms: Platforms{ + Linux: PlatformStatus{ + Supported: Warn, + Notes: []string{ + "When using OpenSSL 3, requires the legacy provider to be enabled.", + }, + }, + }, + }, + { + Name: "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA", + Platforms: Platforms{ + Linux: PlatformStatus{ + Supported: Warn, + Notes: []string{ + "When using OpenSSL 3, requires the legacy provider to be enabled.", + }, + }, + }, + }, + {Name: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA"}, + {Name: "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA"}, + {Name: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256"}, + {Name: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256"}, + {Name: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"}, + {Name: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"}, + {Name: "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"}, + {Name: "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"}, + { + Name: "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", + MinGoVersion: "1.26", + }, + { + Name: "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + MinGoVersion: "1.26", + }, + {Name: "TLS_AES_128_GCM_SHA256"}, + {Name: "TLS_AES_256_GCM_SHA384"}, + { + Name: "TLS_CHACHA20_POLY1305_SHA256", + MinGoVersion: "1.26", + }, + { + Name: "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", + MinGoVersion: "1.26", + }, + { + Name: "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + MinGoVersion: "1.26", + }, + }, + }, + { + Title: "TLS Curves and Groups", + ColumnHeader: "Name", + Description: "Below are the supported [`tls.CurveIDs`](https://pkg.go.dev/crypto/tls#CurveID).", + Items: []Item{ + {Name: "CurveP256"}, + {Name: "CurveP384"}, + {Name: "CurveP521"}, + { + Name: "X25519", + Notes: []string{"See the [X25519](#ecdh) section for requirements."}, + }, + { + Name: "X25519MLKEM768", + Notes: []string{ + "See the [X25519](#ecdh) section for requirements.", + "See the [ML-KEM](#ml-kem) section for requirements.", + }, + }, + { + Name: "SecP256r1MLKEM768", + Notes: []string{ + "See the [ML-KEM](#ml-kem) section for requirements.", + }, + }, + { + Name: "SecP384r1MLKEM1024", + Notes: []string{ + "See the [ML-KEM](#ml-kem) section for requirements.", + }, + }, + }, + }, + { + Title: "TLS Signature Schemes", + ColumnHeader: "Name", + Description: "Below are the supported [`tls.SignatureSchemes`](https://pkg.go.dev/crypto/tls#SignatureScheme).", + Items: []Item{ + {Name: "PKCS1WithSHA1"}, + {Name: "PKCS1WithSHA256"}, + {Name: "PKCS1WithSHA384"}, + {Name: "PKCS1WithSHA512"}, + {Name: "PSSWithSHA256"}, + {Name: "PSSWithSHA384"}, + {Name: "PSSWithSHA512"}, + {Name: "ECDSAWithSHA1"}, + {Name: "ECDSAWithP256AndSHA256"}, + {Name: "ECDSAWithP384AndSHA384"}, + {Name: "ECDSAWithP521AndSHA512"}, + { + Name: "Ed25519", + Platforms: Platforms{ + Windows: PlatformStatus{Supported: NotSupported}, + }, + }, + }, + }, + }, + }, + }, + }, + }, +} diff --git a/eng/_util/cmd/updatecryptodocs/header.md b/eng/_util/cmd/updatecryptodocs/header.md new file mode 100644 index 00000000000..c2ef73d0739 --- /dev/null +++ b/eng/_util/cmd/updatecryptodocs/header.md @@ -0,0 +1,45 @@ +# Cross-Platform Cryptography in the Microsoft build of Go + +Cryptographic operations in the Microsoft build of Go are delegated to the operating system (OS) libraries in some conditions. +The high level conditions and the benefits of delegating cryptographic operations are described in the [Microsoft build of Go FIPS README](./fips/README.md). +At a fine-grained level, Go apps will fall back to the native Go implementation of an algorithm if the OS libraries don't support it. +This article identifies the features that are supported on each platform. + +This article assumes you have a working familiarity with cryptography in Go. + +## Platform support + +The Microsoft build of Go supports the following platforms: + +### Windows + +On Windows, the Microsoft build of Go uses the [CNG library (Cryptography API: Next Generation)](https://learn.microsoft.com/en-us/windows/win32/seccng/cng-portal) for cryptographic operations. +CNG is available since Windows Vista and Windows Server 2008 and it doesn't require any additional installation nor configuration. + +### Linux + +On Linux, the Microsoft build of Go uses the [OpenSSL crypto library](https://docs.openssl.org/3.0/man7/crypto/) for cryptographic operations. +OpenSSL is normally available on Linux distributions, but it may not be installed by default. +If it is not installed, you can install it using the package manager of your distribution. + +OpenSSL 3 implements all the cryptographic algorithms using [Providers](https://docs.openssl.org/3.0/man7/crypto/#providers). +The Microsoft build of Go officially supports the built-in providers and [SCOSSL (SymCrypt provider for OpenSSL)](https://github.com/microsoft/SymCrypt-OpenSSL) v1.6.1 or later. +SCOSSL is expected to be used with the default built-in provider enabled as a fallback (which is the case when using [Azure Linux 3](https://github.com/microsoft/AzureLinux)). + +### macOS + +On macOS, the Microsoft build of Go uses [CommonCrypto](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/Common%20Crypto.3cc.html) and [CryptoKit](https://developer.apple.com/documentation/cryptokit) for cryptographic operations. +CommonCrypto and CryptoKit are shipped with macOS and don't require any additional installation nor configuration. +Currently macOS 13 and above is supported. + +## Table legend + +The following legend describes the symbols used in the tables to indicate the level of support for each cryptographic algorithm: + +| Symbol | Meaning | +| ------ | ---------------------------------------------------------------------------------------------------------------------------- | +| βœ”οΈ | Supported, possibly with minor limitations that don't require special configuration when using the latest Go and OS versions | +| ⚠️ | Supported with limitations that require special configuration action | +| ❌ | Not supported | + +When an algorithm is not supported or the limitations are exceeded, the Microsoft build of Go will fall back to the Go implementation. diff --git a/eng/_util/cmd/updatecryptodocs/updatecryptodocs.go b/eng/_util/cmd/updatecryptodocs/updatecryptodocs.go new file mode 100644 index 00000000000..6f0daa56377 --- /dev/null +++ b/eng/_util/cmd/updatecryptodocs/updatecryptodocs.go @@ -0,0 +1,360 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "text/tabwriter" +) + +var docPath = flag.String("path", filepath.Join("eng", "doc", "CrossPlatformCryptography.md"), "Path to CrossPlatformCryptography.md") + +var useStdout = flag.Bool("stdout", false, "Write to stdout instead of file") + +func main() { + flag.Parse() + + if err := write(); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} + +func write() error { + s, err := generate() + if err != nil { + return fmt.Errorf("failed to generate document: %v", err) + } + + if *useStdout { + fmt.Print(s) + return nil + } + return os.WriteFile(*docPath, []byte(s), 0o644) +} + +func generate() (string, error) { + if err := validate(doc); err != nil { + return "", fmt.Errorf("validation failed: %v", err) + } + + var s strings.Builder + + fmt.Fprintln(&s, "") + fmt.Fprintln(&s) + normalizedHeader := strings.ReplaceAll(header, "\r\n", "\n") + fmt.Fprintf(&s, "%s", normalizedHeader) + + for _, section := range doc.Sections { + printSection(&s, section, 2) + } + return s.String(), nil +} + +func printSection(w io.Writer, section Section, level int) { + fmt.Fprintf(w, "\n%s %s\n\n", strings.Repeat("#", level), section.Title) + + // Filter packages + var validPackages []string + for _, pkg := range section.Packages { + if strings.Contains(pkg, "/") || strings.Contains(pkg, ".") { + validPackages = append(validPackages, pkg) + } + } + + if len(validPackages) > 0 { + fmt.Fprintln(w, "This section includes the following packages:") + fmt.Fprintln(w) + for _, pkg := range validPackages { + if strings.HasPrefix(pkg, "crypto/") { + fmt.Fprintf(w, "- [%s](https://pkg.go.dev/%s)\n", pkg, pkg) + } else if strings.HasPrefix(pkg, "golang.org/") { + fmt.Fprintf(w, "- [%s](https://pkg.go.dev/%s)\n", pkg, pkg) + } else { + // Fallback + fmt.Fprintf(w, "- %s\n", pkg) + } + } + fmt.Fprintln(w) + } + + // Prepend version requirements to description if present + var versionSentences []string + if section.MinGoVersion != "" { + versionSentences = append(versionSentences, fmt.Sprintf("%s is available starting from the Microsoft build of Go %s.", section.Title, section.MinGoVersion)) + } + if len(versionSentences) > 0 { + fmt.Fprintln(w, strings.Join(versionSentences, " ")) + fmt.Fprintln(w) + } + if section.Description != "" { + fmt.Fprintln(w, section.Description) + fmt.Fprintln(w) + } + for _, para := range section.DescriptionParagraphs { + fmt.Fprintln(w, para) + fmt.Fprintln(w) + } + + // Subsections list + if len(section.Subsections) > 0 { + if section.Description == "" && len(section.DescriptionParagraphs) == 0 { + fmt.Fprintln(w, "This section includes the following subsections:") + fmt.Fprintln(w) + } + for _, sub := range section.Subsections { + // Generate anchor link + anchor := strings.ToLower(sub.Title) + anchor = strings.ReplaceAll(anchor, " ", "-") + anchor = strings.ReplaceAll(anchor, "(", "") + anchor = strings.ReplaceAll(anchor, ")", "") + + title := sub.Title + if sub.ShortTitle != "" { + title = sub.ShortTitle + } + fmt.Fprintf(w, "- [%s](#%s)\n", title, anchor) + + } + } + + if len(section.Items) > 0 { + printTable(w, section) + } + + if section.Footer != "" { + fmt.Fprintf(w, "\n%s\n", section.Footer) + } + for _, line := range section.FooterParagraphs { + fmt.Fprintf(w, "\n%s\n", line) + } + + for _, sub := range section.Subsections { + printSection(w, sub, level+1) + } +} + +func printTable(w io.Writer, section Section) { + colHeader := section.ColumnHeader + if colHeader == "" { + colHeader = "Algorithm" // Default + } + + var wipTable strings.Builder + tw := tabwriter.NewWriter(&wipTable, 0, 0, 1, ' ', 0) + platforms := strings.Join(SupportedPlatforms, "\t|\t") + platforms = strings.Replace(platforms, "windows", "Windows", 1) + platforms = strings.Replace(platforms, "linux", "Linux", 1) + platforms = strings.Replace(platforms, "macos", "macOS", 1) + fmt.Fprintf(tw, "|\t%s\t|\t%s\t|\n", colHeader, platforms) + + // We skip the "| --- | --- | etc." row for now. We need to know the actual + // column lengths to put in the proper number of dashes (for raw source + // readability). So, we need to figure it out first and inject it later. + // fmt.Fprintf(tw, "|\t---\t|\t---\t|\t---\t|\t---\t|\n") + + // Collect footnotes + footnotes := make(map[string]int) + nextFootnote := 1 + + // Pre-populate from section.Footnotes + for _, note := range section.Footnotes { + if _, exists := footnotes[note]; !exists { + footnotes[note] = nextFootnote + nextFootnote++ + } + } + + // Helper to get footnote indices + getFootnotes := func(notes []string) string { + if len(notes) == 0 { + return "" + } + var indices []int + for _, note := range notes { + idx, exists := footnotes[note] + if !exists { + idx = nextFootnote + footnotes[note] = idx + nextFootnote++ + } + indices = append(indices, idx) + } + sort.Ints(indices) + strs := make([]string, len(indices)) + for i, idx := range indices { + strs[i] = fmt.Sprintf("%d", idx) + } + return fmt.Sprintf("%s", strings.Join(strs, ",")) + } + + for _, item := range section.Items { + name := item.Name + + itemNotes := item.Notes + if item.MinGoVersion != "" { + note := fmt.Sprintf("Available starting in the Microsoft build of Go %s.", item.MinGoVersion) + itemNotes = append([]string{note}, itemNotes...) + } + + nameNotes := getFootnotes(itemNotes) + + row := []string{name + nameNotes} + + for _, platform := range SupportedPlatforms { + status := item.Platforms.Get(platform) + symbol := "" + switch status.Supported { + case Supported: + symbol = "βœ”οΈ" + case NotSupported: + symbol = "❌️" + case Warn: + symbol = "⚠️" + case N_A: + symbol = "N/A" + default: + // Should not happen due to prior validation + panic(fmt.Sprintf("unexpected status %q", status.Supported)) + } + + statusNotes := status.Notes + // Prepend version notes in preferred order: minGoVersion, minOpenSSLVersion, minMacOSVersion + var versionNotes []string + if status.MinGoVersion != "" { + versionNotes = append(versionNotes, fmt.Sprintf("Available starting in the Microsoft build of Go %s.", status.MinGoVersion)) + } + if status.MinVersion != "" { + switch platform { + case "windows": + versionNotes = append(versionNotes, fmt.Sprintf("Requires Windows %s or later.", status.MinVersion)) + case "linux": + versionNotes = append(versionNotes, fmt.Sprintf("Requires OpenSSL %s or later.", status.MinVersion)) + case "macos": + versionNotes = append(versionNotes, fmt.Sprintf("Requires macOS %s or later.", status.MinVersion)) + default: + panic("unknown platform" + platform) + } + } + // Remove duplicates from notes + filtered := []string{} + for _, n := range statusNotes { + duplicate := false + for _, vn := range versionNotes { + if n == vn { + duplicate = true + break + } + } + if !duplicate { + filtered = append(filtered, n) + } + } + statusNotes = append(versionNotes, filtered...) + + notes := getFootnotes(statusNotes) + row = append(row, symbol+notes) + } + + fmt.Fprintf(tw, "|\t%s\t|\t%s\t|\t%s\t|\t%s\t|\n", row[0], row[1], row[2], row[3]) + } + + err := tw.Flush() + if err != nil { + panic(err) + } + + ts := wipTable.String() + // Now, inject the separator line with proper dash lengths. + rows := strings.Split(ts, "\n") + firstRow := rows[0] + fmt.Fprintf(w, "%s\n", firstRow) + cells := strings.Split(firstRow, "|") + // Ignore the first and last: empty strings. + // We expect "|