diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index e2f24dc2..16f077b8 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,32 +3,39 @@ "isRoot": true, "tools": { "powershell": { - "version": "7.5.3", + "version": "7.5.4", "commands": [ "pwsh" ], "rollForward": false }, "dotnet-coverage": { - "version": "17.14.2", + "version": "18.5.2", "commands": [ "dotnet-coverage" ], "rollForward": false }, "nbgv": { - "version": "3.7.115", + "version": "3.9.50", "commands": [ "nbgv" ], "rollForward": false }, "docfx": { - "version": "2.78.3", + "version": "2.78.5", "commands": [ "docfx" ], "rollForward": false + }, + "nerdbank.dotnetrepotools": { + "version": "1.2.1", + "commands": [ + "repo" + ], + "rollForward": false } } } diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index 4a440efc..00000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -# Refer to https://hub.docker.com/_/microsoft-dotnet-sdk for available versions -FROM mcr.microsoft.com/dotnet/sdk:9.0.305-noble@sha256:604ef064c6d91068eeb9d946036d8ffadbe25589c4cd77a230fc96e0f6d01d72 - -# Installing mono makes `dotnet test` work without errors even for net472. -# But installing it takes a long time, so it's excluded by default. -#RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -#RUN echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic main" | tee /etc/apt/sources.list.d/mono-official-stable.list -#RUN apt-get update -#RUN DEBIAN_FRONTEND=noninteractive apt-get install -y mono-devel - -# Clear the NUGET_XMLDOC_MODE env var so xml api doc files get unpacked, allowing a rich experience in Intellisense. -# See https://github.com/dotnet/dotnet-docker/issues/2790 for a discussion on this, where the prioritized use case -# was *not* devcontainers, sadly. -ENV NUGET_XMLDOC_MODE= diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 1a3a0086..00000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "Dev space", - "dockerFile": "Dockerfile", - "customizations": { - "vscode": { - "settings": { - "terminal.integrated.shell.linux": "/usr/bin/pwsh" - }, - "extensions": [ - "ms-azure-devops.azure-pipelines", - "ms-dotnettools.csharp", - "k--kato.docomment", - "editorconfig.editorconfig", - "esbenp.prettier-vscode", - "pflannery.vscode-versionlens", - "davidanson.vscode-markdownlint", - "dotjoshjohnson.xml", - "ms-vscode-remote.remote-containers", - "ms-azuretools.vscode-docker", - "tintoy.msbuild-project-tools" - ] - } - }, - "postCreateCommand": "./init.ps1 -InstallLocality machine" -} diff --git a/.github/Prime-ForCopilot.ps1 b/.github/Prime-ForCopilot.ps1 new file mode 100644 index 00000000..e0b1fb79 --- /dev/null +++ b/.github/Prime-ForCopilot.ps1 @@ -0,0 +1,5 @@ +if ((git -C $PSScriptRoot rev-parse --is-shallow-repository) -eq 'true') +{ + Write-Host "Shallow clone detected, disabling NBGV Git engine so the build can succeed." + $env:NBGV_GitEngine='Disabled' +} diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 08593502..9e3933b6 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -3,7 +3,8 @@ ## High level guidance * Review the `CONTRIBUTING.md` file for instructions to build and test the software. -* Set the `NBGV_GitEngine` environment variable to `Disabled` before running any `dotnet` or `msbuild` commands. +* Run the `.github/Prime-ForCopilot.ps1` script (once) before running any `dotnet` or `msbuild` commands. + If you see any build errors about not finding git objects or a shallow clone, it may be time to run this script again. ## Software Design @@ -12,9 +13,68 @@ ## Testing -* There should generally be one test project (under the `test` directory) per shipping project (under the `src` directory). Test projects are named after the project being tested with a `.Test` suffix. -* Tests should use the Xunit testing framework. -* Some tests are known to be unstable. When running tests, you should skip the unstable ones by running `dotnet test --filter "TestCategory!=FailsInCloudTest"`. +**IMPORTANT**: This repository uses Microsoft.Testing.Platform (MTP v2) with xunit v3. Traditional `--filter` syntax does NOT work. Use the options below instead. + +* There should generally be one test project (under the `test` directory) per shipping project (under the `src` directory). Test projects are named after the project being tested with a `.Tests` suffix. +* Tests use xunit v3 with Microsoft.Testing.Platform (MTP v2). Traditional VSTest `--filter` syntax does NOT work. +* Some tests are known to be unstable. When running tests, you should skip the unstable ones by using `-- --filter-not-trait "FailsInCloudTest=true"`. + +### Running Tests + +**Run all tests**: +```bash +dotnet test --no-build -c Release +``` + +**Run tests for a specific test project**: +```bash +dotnet test --project test/Microsoft.VisualStudio.SolutionPersistence.Tests/Microsoft.VisualStudio.SolutionPersistence.Tests.csproj --no-build -c Release +``` + +**Run a single test method**: +```bash +dotnet test --project test/Microsoft.VisualStudio.SolutionPersistence.Tests/Microsoft.VisualStudio.SolutionPersistence.Tests.csproj --no-build -c Release -- --filter-method ClassName.MethodName +``` + +**Run all tests in a test class**: +```bash +dotnet test --project test/Microsoft.VisualStudio.SolutionPersistence.Tests/Microsoft.VisualStudio.SolutionPersistence.Tests.csproj --no-build -c Release -- --filter-class ClassName +``` + +**Run tests with wildcard matching** (supports wildcards at beginning and/or end): +```bash +dotnet test --project test/Microsoft.VisualStudio.SolutionPersistence.Tests/Microsoft.VisualStudio.SolutionPersistence.Tests.csproj --no-build -c Release -- --filter-method "*Pattern*" +``` + +**Run tests with a specific trait** (equivalent to category filtering): +```bash +dotnet test --project test/Microsoft.VisualStudio.SolutionPersistence.Tests/Microsoft.VisualStudio.SolutionPersistence.Tests.csproj --no-build -c Release -- --filter-trait "TraitName=value" +``` + +**Exclude tests with a specific trait** (skip unstable tests): +```bash +dotnet test --project test/Microsoft.VisualStudio.SolutionPersistence.Tests/Microsoft.VisualStudio.SolutionPersistence.Tests.csproj --no-build -c Release -- --filter-not-trait "TestCategory=FailsInCloudTest" +``` + +**Run tests for a specific framework only**: +```bash +dotnet test --project test/Microsoft.VisualStudio.SolutionPersistence.Tests/Microsoft.VisualStudio.SolutionPersistence.Tests.csproj --no-build -c Release --framework net8.0 +``` + +**List all available tests without running them**: +```bash +cd test/Microsoft.VisualStudio.SolutionPersistence.Tests +dotnet run --no-build -c Release --framework net8.0 -- --list-tests +``` + +**Key points about test filtering with MTP v2 / xunit v3**: +- Options after `--` are passed to the test runner, not to `dotnet test` +- Use `--filter-method`, `--filter-class`, `--filter-namespace` for simple filtering +- Use `--filter-trait` and `--filter-not-trait` for trait-based filtering (replaces `--filter "TestCategory=..."`) +- Traditional VSTest `--filter` expressions do NOT work +- Wildcards `*` are supported at the beginning and/or end of filter values +- Multiple simple filters of the same type use OR logic, different types combine with AND +- See `--help` for query filter language for advanced scenarios ## Coding style diff --git a/.github/prompts/repo-merge-notes.md b/.github/prompts/repo-merge-notes.md new file mode 100644 index 00000000..f18d134a --- /dev/null +++ b/.github/prompts/repo-merge-notes.md @@ -0,0 +1,51 @@ +# Repo-specific merge notes for vs-solutionpersistence + +This file contains notes specific to this repo that help resolve Library.Template merge conflicts. +It is referenced by `update-library-template.prompt.md` and kept separate to avoid merge conflicts with template-owned files. + +## Files deleted locally that the template will try to update + +These show as "deleted by us" conflicts. Here are the known cases: + +* **GitHub Actions CI files** (`.github/workflows/docs.yml`, `.github/workflows/docs_validate.yml`, `.github/actions/publish-artifacts/action.yaml`): This repo uses Azure Pipelines, not GitHub Actions for CI/CD. Keep these deleted. +* **`.github/workflows/libtemplate-update.yml`**: Automatic Library.Template merge workflow. Should be **kept/restored** β€” it was mistakenly deleted in a past merge and is useful infrastructure. +* **`.github/renovate.json`**: Dependency update automation. Should be kept/restored if missing. +* **`test/Library.Tests/Library.Tests.csproj`**: Template's generic test project. Re-delete it, but apply any relevant changes to `test/Microsoft.VisualStudio.SolutionPersistence.Tests/`. + +## `azure-pipelines/build.yml` + +The template includes `expand-template.yml` references in build.yml (Windows, Linux, macOS jobs). +This repo is not a template and does not have `Expand-Template.ps1`, so always remove these lines after accepting template changes: +```yaml + - template: expand-template.yml +``` + +## `azure-pipelines/vs-insertion.yml` + +The template has a placeholder for `InsertionReviewers`. This repo customizes it to include the team: +```yaml +InsertionReviewers: $(Build.RequestedFor),VS Core - Solution Experience +``` +When merging, keep the team name but accept any other new properties the template adds (e.g. `CustomScriptExecutionCommand`). + +## Researching template changes + +When a conflict is hard to understand, check the Library.Template commit history for the specific file: +``` +https://github.com/aarnott/Library.Template/commits/microbuild/ +``` +This shows *why* changes were made and helps decide whether to accept them. +You can also compare the template's full current version of a file: +```bash +git show libtemplate/microbuild: +``` + +## History of past merge mistakes + +The commit `2fb9c3d` ("Remove github actions", Feb 2025) bulk-deleted all GitHub workflow files. +Some were genuinely unused (GitHub Actions CI, GitHub Pages deployment), but it also deleted useful automation: +* `.github/workflows/libtemplate-update.yml` β€” Automatic weekly Library.Template merge PRs +* `.github/workflows/docs_validate.yml` β€” Docfx link validation +* `.github/renovate.json` β€” Renovate bot dependency updates + +These have been restored. If they show up as conflicts again in future merges, keep them. diff --git a/.github/prompts/template-release-notes.md b/.github/prompts/template-release-notes.md new file mode 100644 index 00000000..975da315 --- /dev/null +++ b/.github/prompts/template-release-notes.md @@ -0,0 +1,20 @@ +# Template release notes + +This file will describe significant changes in Library.Template as they are introduced, especially if they require special consideration when merging updates into existing repos. +This file is referenced by update-library-template.prompt.md and should remain in place to facilitate future merges, whether done manually or by AI. + +## Solution rename + +Never leave a Library.slnx file in the repository. +You might even see one there even though this particular merge didn't bring it in. +This can be an artifact of having renamed Library.sln to Library.slnx in the template repo, but ultimately the receiving repo should have only one .sln or .slnx file, with a better name than `Library`. +Delete any `Library.slnx` that you see. +Migrate an `.sln` in the repo root to `.slnx` using this command: + +```ps1 +dotnet solution EXISTING.sln migrate +``` + +This will create an EXISTING.slnx file. `git add` that file, then `git rm` the old `.sln` file. +Sometimes a repo will reference the sln filename in a script or doc somewhere. +Search the repo for such references and update them to the slnx file. diff --git a/.github/prompts/update-library-template.prompt.md b/.github/prompts/update-library-template.prompt.md new file mode 100644 index 00000000..c2abe371 --- /dev/null +++ b/.github/prompts/update-library-template.prompt.md @@ -0,0 +1,76 @@ +--- +description: Merges the latest Library.Template into this repo (at position of HEAD) and resolves conflicts. +--- + +# Instructions + +1. Run `tools/MergeFrom-Template.ps1` +2. Resolve merge conflicts, taking into account conflict resolution policy below. +3. Validate the changes, as described in the validation section below. +4. Committing your changes (if applicable). + +## Conflict resolution policy + +There may be special notes in `.github/prompts/template-release-notes.md` that describe special considerations for certain files or scenarios to help you resolve conflicts appropriately. +Always refer to that file before proceeding. +In particular, focus on the *incoming* part of the file, since it represents the changes from the Library.Template that you are merging into your repo. +Also refer to `.github/prompts/repo-merge-notes.md` for notes specific to this repo (e.g. which deleted files to keep, which customizations to preserve). +Also consider that some repos choose to reject certain Library.Template patterns. +For example the template uses MTPv2 for test projects, but a repo might have chosen not to adopt that. +When resolving merge conflicts, consider whether it looks like the relevant code file is older than it should be given the changes the template is bringing in. +Ask the user when in doubt as to whether the conflict should be resolved in favor of 'catching up' with the template or keeping the current changes. + +Use #runSubagent to analyze and resolve merge conflicts across files in parallel. + +### Keep Current files + +Conflicts in the following files should always be resolved by keeping the current version (i.e. discard incoming changes): + +* README.md + +### Deleted files + +Very typically, when the incoming change is to a file that was deleted locally, the correct resolution is to re-delete the file. + +In some cases however, the deleted file may have incoming changes that should be applied to other files. +The `test/Library.Tests/Library.Tests.csproj` file is very typical of this. +Changes to this file should very typically be applied to any and all test projects in the repo. +You are responsible for doing this in addition to re-deleting this template file. + +### Files deleted by the template + +When the template deletes a file that exists locally, check the template's commit history to understand *why* it was removed. +Use the GitHub commit history page for the file (e.g. `https://github.com/aarnott/Library.Template/commits/microbuild/path/to/file`) to find the relevant commit. +If the template restructured or centralized the functionality (e.g. OptProf pipelines moved to a centralized VS pipeline), accept the deletion. +If the file contains repo-specific customizations not present in the template, keep it. + +### Files deleted locally but updated by the template + +These show as "deleted by us" conflicts. Refer to `.github/prompts/repo-merge-notes.md` for this repo's specific decisions on which files to keep or re-delete. + +## Validation + +Validate the merge result (after resolving any conflicts, if applicable). +Use #runSubagent for each step. + +1. Verify that `dotnet restore` succeeds. Fix any issues that come up. +2. Verify that `dotnet build` succeeds. +3. Verify that tests succeed by running `tools/dotnet-test-cloud.ps1`. + +While these validations are described using `dotnet` CLI commands, some repos require using full msbuild.exe. +You can detect this by checking the `azure-pipelines/dotnet.yml` or `.github/workflows/build.yml` files for use of one or the other tool. + +You are *not* responsible for fixing issues that the merge did not cause. +If validation fails for reasons that seem unrelated to the changes brought in by the merge, advise the user and ask how they'd like you to proceed. +That said, sometimes merges will bring in SDK or dependency updates that can cause breaks in seemingly unrelated areas. +In such cases, you should investigate and solve the issues as needed. + +## Committing your changes + +If you have to make any changes for validations to pass, consider whether they qualify as a bad merge conflict resolution or more of a novel change that you're making to work with the Library.Template update. +Merge conflict resolution fixes ideally get amended into the merge commit, while novel changes would go into a novel commit after the merge commit. + +Always author your commits using `git commit --author "πŸ€– Copilot "` (and possibly other parameters). +Describe the nature of the merge conflicts you encountered and how you resolved them in your commit message. + +Later, if asked to review pull request validation breaks, always author a fresh commit with each fix that you push, unless the user directs you to do otherwise. diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 00000000..4cd2b0cb --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "github>microsoft/vs-renovate-presets:microbuild", + "github>microsoft/vs-renovate-presets:vs_main_dependencies" + ], + "packageRules": [] +} diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index eb69d92e..3c032fae 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -26,7 +26,7 @@ jobs: # You can define any steps you want, and they will run before the agent starts. # If you do not check out your code, Copilot will do this for you. steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # avoid shallow clone so nbgv can do its work. - name: βš™ Install prerequisites diff --git a/.github/workflows/libtemplate-update.yml b/.github/workflows/libtemplate-update.yml new file mode 100644 index 00000000..62a6417d --- /dev/null +++ b/.github/workflows/libtemplate-update.yml @@ -0,0 +1,98 @@ +name: β›œ Library.Template update + +# PREREQUISITE: This workflow requires the repo to be configured to allow workflows to create pull requests. +# Visit https://github.com/USER/REPO/settings/actions +# Under "Workflow permissions" check "Allow GitHub Actions to create ...pull requests" +# Click Save. + +on: + schedule: + - cron: "0 3 * * Mon" # Sun @ 8 or 9 PM Mountain Time (depending on DST) + workflow_dispatch: + +jobs: + merge: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 # avoid shallow clone so nbgv can do its work. + + - name: merge + id: merge + shell: pwsh + run: | + $LibTemplateBranch = & ./tools/Get-LibTemplateBasis.ps1 -ErrorIfNotRelated + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + git fetch https://github.com/aarnott/Library.Template $LibTemplateBranch + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + $LibTemplateCommit = git rev-parse FETCH_HEAD + git diff --stat ...FETCH_HEAD + + if ((git rev-list FETCH_HEAD ^HEAD --count) -eq 0) { + Write-Host "There are no Library.Template updates to merge." + echo "uptodate=true" >> $env:GITHUB_OUTPUT + exit 0 + } + + # Pushing commits that add or change files under .github/workflows will cause our workflow to fail. + # But it usually isn't necessary because the target branch already has (or doesn't have) these changes. + # So if the merge doesn't bring in any changes to these files, try the merge locally and push that + # to keep github happy. + if ((git rev-list FETCH_HEAD ^HEAD --count -- .github/workflows) -eq 0) { + # Indeed there are no changes in that area. So merge locally to try to appease GitHub. + git checkout -b auto/libtemplateUpdate + git config user.name "Andrew Arnott" + git config user.email "andrewarnott@live.com" + git merge FETCH_HEAD + if ($LASTEXITCODE -ne 0) { + Write-Host "Merge conflicts prevent creating the pull request. Please run tools/MergeFrom-Template.ps1 locally and push the result as a pull request." + exit 2 + } + + git -c http.extraheader="AUTHORIZATION: bearer $env:GH_TOKEN" push origin -u HEAD + } else { + Write-Host "Changes to github workflows are included in this update. Please run tools/MergeFrom-Template.ps1 locally and push the result as a pull request." + exit 1 + } + - name: pull request + shell: pwsh + if: success() && steps.merge.outputs.uptodate != 'true' + run: | + # If there is already an active pull request, don't create a new one. + $existingPR = gh pr list -H auto/libtemplateUpdate --json url | ConvertFrom-Json + if ($existingPR) { + Write-Host "::warning::Skipping pull request creation because one already exists at $($existingPR[0].url)" + exit 0 + } + + $prTitle = "Merge latest Library.Template" + $prBody = "This merges the latest features and fixes from [Library.Template's branch](https://github.com/AArnott/Library.Template/tree/). + + ⚠️ Do **not** squash this pull request when completing it. You must *merge* it. + +
+ Merge conflicts? + Resolve merge conflicts locally by carrying out these steps: + + ``` + git fetch + git checkout auto/libtemplateUpdate + git merge origin/main + # resolve conflicts + git commit + git push + ``` +
" + + gh pr create -H auto/libtemplateUpdate -b $prBody -t $prTitle + env: + GH_TOKEN: ${{ github.token }} diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 00000000..dd449725 --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +*.md diff --git a/Directory.Build.props b/Directory.Build.props index af3cca15..e0d04b6f 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -16,6 +16,7 @@ true true true + true true @@ -44,7 +45,7 @@ - 13 + 14 16.9 diff --git a/Directory.Packages.props b/Directory.Packages.props index e5423a30..c10fbcd3 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,8 +4,9 @@ true true + 2.1.0 - 2.0.201 + 2.0.208 @@ -16,10 +17,12 @@ - + + + + - - + @@ -30,7 +33,7 @@ - + diff --git a/azure-pipelines/BuildStageVariables.yml b/azure-pipelines/BuildStageVariables.yml index 2dcfbd5c..7c61f8fe 100644 --- a/azure-pipelines/BuildStageVariables.yml +++ b/azure-pipelines/BuildStageVariables.yml @@ -2,4 +2,3 @@ variables: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ - # #codecov_token: # Get a new one from https://codecov.io/ diff --git a/azure-pipelines/Merge-CodeCoverage.ps1 b/azure-pipelines/Merge-CodeCoverage.ps1 index 32438237..c552d25f 100644 --- a/azure-pipelines/Merge-CodeCoverage.ps1 +++ b/azure-pipelines/Merge-CodeCoverage.ps1 @@ -28,6 +28,7 @@ try { if ($reports) { $reports |% { $_.FullName } |% { # In addition to replacing {reporoot}, we also normalize on one kind of slash so that the report aggregates data for a file whether data was collected on Windows or not. + Write-Verbose "Processing $_" $xml = [xml](Get-Content -LiteralPath $_) $xml.coverage.packages.package.classes.class |? { $_.filename} |% { $_.filename = $_.filename.Replace('{reporoot}', $RepoRoot).Replace([IO.Path]::AltDirectorySeparatorChar, [IO.Path]::DirectorySeparatorChar) diff --git a/azure-pipelines/OptProf.yml b/azure-pipelines/OptProf.yml deleted file mode 100644 index dae0f1c2..00000000 --- a/azure-pipelines/OptProf.yml +++ /dev/null @@ -1,121 +0,0 @@ -trigger: none -pr: none -schedules: -- cron: "0 3 * * Fri" # Thu @ 8 or 9 PM Mountain Time (depending on DST) - displayName: Weekly OptProf run - branches: - include: - - 'v*.*' - - main - always: true # we must keep data fresh since optimizationdata drops are purged after 30 days - -# Avoid errant CI builds: https://developercommunity.visualstudio.com/content/problem/1154409/azure-pipeline-is-triggering-due-to-events-that-ne.html -#resources: -# repositories: -# - repository: scripts -# type: git -# name: DeploymentScripts -# ref: refs/heads/test - -parameters: - - name: ShouldSkipOptimize - displayName: Skip OptProf optimization - type: boolean - default: false # Should usually be false so that optprof LKG can apply when tests fail, but may need to be set to true in a manually queued pipeline run if all drops have expired. - -variables: -- template: GlobalVariables.yml -- name: PublicRelease - value: false # avoid using nice version since we're building a preliminary/unoptimized package -- name: IsOptProf - value: true -- name: MicroBuild_NuPkgSigningEnabled - value: false # test-signed nuget packages fail to restore in the VS insertion PR validations. Just don't sign them *at all*. - -stages: -- stage: Library - variables: - - name: OptProf - value: true - - template: BuildStageVariables.yml - jobs: - - template: build.yml - parameters: - Is1ESPT: false - RealSign: false - windowsPool: VSEngSS-MicroBuild2022-1ES - EnableMacOSBuild: false - ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} - IsOptProf: true - RunTests: false - SkipCodesignVerify: true -- stage: QueueVSBuild - jobs: - - job: QueueOptProf - pool: VSEngSS-MicroBuild2022-1ES - variables: - InsertPayloadName: Microsoft.VisualStudio.SolutionPersistence - InsertTopicBranch: team/VS-IDE/Microsoft.VisualStudio.SolutionPersistence-OptProf-run-$(Build.BuildId) - steps: - - checkout: none # We don't need source from our own repo - clean: true - - # Pipeline YAML does not yet support checking out other repos. So we'll do it by hand. -# - checkout: scripts # We DO need source from the DeploymentScripts repo -# clean: true -# path: $(Agent.TempDirectory)/DeploymentScripts -# fetchDepth: 1 - - script: 'git -c http.extraheader="AUTHORIZATION: bearer $(System.AccessToken)" clone https://devdiv.visualstudio.com/DevDiv/_git/DeploymentScripts --depth 1 --branch test "$(Agent.TempDirectory)/DeploymentScripts"' - displayName: Download DeploymentScripts repo - - - task: DownloadBuildArtifacts@0 - displayName: Download insertion artifacts - inputs: - artifactName: VSInsertion-Windows - downloadPath: $(Agent.TempDirectory) - - task: DownloadBuildArtifacts@0 - displayName: Download variables artifacts - inputs: - artifactName: Variables-Windows - downloadPath: $(Agent.TempDirectory) - - task: PowerShell@2 - displayName: Set pipeline variables based on artifacts - inputs: - targetType: filePath - filePath: $(Agent.TempDirectory)/Variables-Windows/_define.ps1 - - task: NuGetCommand@2 - displayName: Push VS-repo packages to VS feed - inputs: - command: push - packagesToPush: $(Agent.TempDirectory)/VSInsertion-Windows/*.nupkg - publishVstsFeed: 97a41293-2972-4f48-8c0e-05493ae82010 # VS feed - allowPackageConflicts: true - - task: MicroBuildInsertVsPayload@5 - displayName: Insert VS Payload - inputs: - TeamName: $(TeamName) - TeamEmail: $(TeamEmail) - SkipCreatePR: true - CustomScriptExecutionCommand: src\VSSDK\NuGet\AllowUnstablePackages.ps1 - - task: benjhuser.tfs-extensions-build-tasks.trigger-build-task.TriggerBuild@3 - displayName: Trigger a new build of DD-CB-TestSignVS-devCI - inputs: - buildDefinition: DD-CB-TestSignVS-devCI - useSameBranch: false - branchToUse: $(InsertTopicBranch) - storeInEnvironmentVariable: true - queueBuildForUserThatTriggeredBuild: false - authenticationMethod: OAuth Token - password: $(System.AccessToken) - - task: PowerShell@2 - displayName: Associate InsertionOutputs artifacts with CloudBuild - inputs: - targetType: filePath - filePath: $(Agent.TempDirectory)/DeploymentScripts/Scripts/Insertion/WriteArtifact.ps1 - arguments: '-oldBuildID $(Build.BuildId) -newBuildID $(TriggeredBuildIds) -artifactName "InsertionOutputs" -accessToken $(System.AccessToken)' - - task: PowerShell@2 - displayName: Tag the build with Microsoft.VisualStudio.SolutionPersistence-insertion - inputs: - targetType: filePath - filePath: $(Agent.TempDirectory)/DeploymentScripts/Scripts/Insertion/TagBuild.ps1 - arguments: '-buildID $(TriggeredBuildIds) -tagName "Microsoft.VisualStudio.SolutionPersistence-insertion" -accessToken $(System.AccessToken)' diff --git a/azure-pipelines/OptProf_part2.yml b/azure-pipelines/OptProf_part2.yml deleted file mode 100644 index 5f9a5b1c..00000000 --- a/azure-pipelines/OptProf_part2.yml +++ /dev/null @@ -1,91 +0,0 @@ -trigger: none -pr: none - -resources: - pipelines: - - pipeline: VisualStudioBuildUnderTest - source: DD-CB-TestSignVS-devCI - trigger: - tags: - - Microsoft.VisualStudio.SolutionPersistence-insertion - - pipeline: DartLab - source: DartLab - branch: main - - pipeline: DartLab.OptProf - source: DartLab.OptProf - branch: main - tags: - - production - repositories: - - repository: DartLabTemplates - type: git - name: DartLab.Templates - ref: refs/heads/main - - repository: DartLabOptProfTemplates - type: git - name: DartLab.OptProf - ref: refs/tags/Production - -parameters: - -# The prefix naming of the OptimizationInputs drop -- name: optimizationDropPrefix - type: string - default: OptimizationInputs/$(System.TeamProject)/$(Build.Repository.Name) - -stages: -- template: \templates\stages\visual-studio\single-runsettings.yml@DartLabOptProfTemplates - parameters: - ##### Required ##### - runSettingsURI: $(Pipeline.Workspace)\VisualStudioBuildUnderTest\BuildArtifacts\runsettings\Microsoft.VisualStudio.SolutionPersistence.OptProf.runsettings - visualStudioBootstrapperURI: https://vsdrop.corp.microsoft.com/file/v1/$(VisualStudio.BuildUnderTest.ProductsDropName);bootstrappers/Enterprise/vs_enterprise.exe - ##### Optional ##### - name: OptProfProfilingWorkflow - displayName: OptProf Profiling Workflow - optOptimizationInputsDropName: $(OptimizationInputsDropName) - previousOptimizationInputsDropName: $(PreviousOptimizationInputsDropName) - testLabPoolName: VS-Platform - ##### Step Hooks ##### - preTestMachineConfigurationStepList: - - download: VisualStudioBuildUnderTest - - task: PowerShell@2 - name: SetProductsDropName - displayName: Set 'VisualStudio.BuildUnderTest.ProductsDropName' - inputs: - filePath: $(DartLab.Path)\Scripts\VisualStudio\Build\Get-VisualStudioDropName.ps1 - arguments: -DropNamePrefix 'Products' -VstsDropUrlsJson '$(Pipeline.Workspace)\VisualStudioBuildUnderTest\BuildArtifacts\VstsDropUrls.json' -OutVariableName 'VisualStudio.BuildUnderTest.ProductsDropName' - preDeployAndRunTestsStepList: - - download: VisualStudioBuildUnderTest - prePublishOptimizationInputsDropStepList: - # Set parameter for PreviousOptimizationInputsDropName, MicroBuildCommitID, and OptimizationInputsDropName - - powershell: | - try { - $artifactName = 'InsertionOutputs' - $BuildID = $(resources.pipeline.VisualStudioBuildUnderTest.runID) - $artifact = Get-BuildArtifact -InstanceURL 'https://dev.azure.com/devdiv' -ProjectName 'DevDiv' -BuildID $BuildID -ArtifactName $artifactName -OAuthAccessToken (ConvertTo-SecureString '$(System.AccessToken)' -AsPlainText -Force) - $containerName = $artifact.Resource.Data -Split '/' | Select-Object -Last 1 - $fileName = Join-Path $containerName 'Metadata.json' - $jsonString = Read-BuildArtifactFile -InstanceURL 'https://dev.azure.com/devdiv' -ProjectName 'DevDiv' -BuildID $BuildID -ArtifactName $artifactName -FileName $fileName -OAuthAccessToken (ConvertTo-SecureString '$(System.AccessToken)' -AsPlainText -Force) - $json = $jsonString | ConvertFrom-Json - - Write-Host "The content of the metadata.json file was $json" - - $dropname = $json.OptimizationData - $commitID = $json.CommitID - $OptimizationInputsDropName = "${{parameters.optimizationDropPrefix}}/$($commitID)/$(Build.BuildId)/$(System.StageId)/$(System.StageAttempt)" - - Write-Host "PreviousOptimizationInputsDropName: $dropname" - Set-AzurePipelinesVariable 'PreviousOptimizationInputsDropName' $dropname - - Write-Host "MicroBuildCommitID: $commitID" - Set-AzurePipelinesVariable 'MicroBuildCommitID' $commitID - - Write-Host "OptimizationInputsDropName: $OptimizationInputsDropName" - Set-AzurePipelinesVariable 'OptimizationInputsDropName' $OptimizationInputsDropName - } - catch { - Write-Host $_ - Write-Error "Failed to set OptimizationInputsDropName pipeline variable" - throw - } - displayName: Set MicroBuildCommitID, PreviousOptimizationInputsDropName, and OptimizationInputsDropName diff --git a/azure-pipelines/archive-sourcecode.yml b/azure-pipelines/archive-sourcecode.yml index 77f243bc..84b70ec3 100644 --- a/azure-pipelines/archive-sourcecode.yml +++ b/azure-pipelines/archive-sourcecode.yml @@ -35,8 +35,10 @@ variables: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: + settings: + networkIsolationPolicy: Permissive,CFSClean2 sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: VSEng-MicroBuildVSStable stages: - stage: archive diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index e7d2e299..8262e4ef 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -29,14 +29,17 @@ parameters: - name: artifact_names type: object default: - - build_logs - - coverageResults - - deployables - - projectAssetsJson - - symbols - - testResults - - test_symbols - - Variables + - name: build_logs + - name: coverageResults + - name: deployables + sbomEnabled: true + - name: projectAssetsJson + - name: symbols + - name: testResults + testOnly: true + - name: test_symbols + testOnly: true + - name: Variables # The Enable*Build parameters turn non-Windows agents on or off. # Their default value should be based on whether the build and tests are expected/required to pass on that platform. # Callers (e.g. Official.yml) *may* expose these parameters at queue-time in order to turn OFF optional agents. @@ -47,8 +50,8 @@ parameters: type: boolean default: true -##### Γ°ΕΈβ€˜β€ Γ°ΕΈΒΒΌ You MAY change the defaults above. -##### Γ°ΕΈβ€˜β€‘Γ°ΕΈΒΒΌ You should NOT change the defaults below. +##### πŸ‘†πŸΌ You MAY change the defaults above. +##### πŸ‘‡πŸΌ You should NOT change the defaults below. ##### The following parameters are expected to be set by other YAML files that import this one. ##### Those without defaults require explicit values to be provided by our importers. @@ -57,13 +60,12 @@ parameters: - name: Is1ESPT type: boolean -- name: RealSign +# Indicates whether the 'official' 1ES PT templates are being used (as opposed to the unofficial ones). +- name: Is1ESPTOfficial type: boolean default: false -# Whether this particular run is an OptProf profiling run. -# This is used to skip unit tests and other non-essential work to improve reliability of the OptProf pipeline. -- name: IsOptProf +- name: RealSign type: boolean default: false @@ -96,15 +98,15 @@ parameters: - name: windowsPool type: object default: - vmImage: windows-2022 + vmImage: windows-2025 - name: linuxPool type: object default: - vmImage: ubuntu-22.04 + vmImage: ubuntu-24.04 - name: macOSPool type: object default: - vmImage: macOS-14 + vmImage: macOS-15 jobs: - job: Windows @@ -131,8 +133,11 @@ jobs: optprof: enabled: ${{ parameters.EnableOptProf }} ProfilingInputsDropName: $(ProfilingInputsDropName) - OptimizationInputsLookupMethod: DropPrefix - DropNamePrefix: OptimizationInputs/$(System.TeamProject)/$(Build.Repository.Name) + GeneratePropsFile: true + PropsPath: $(Build.ArtifactStagingDirectory)/InsertionOutputs/$(ProfilingInputsPropsName) + OptimizationInputsLookupMethod: GitTagRepo + GitTagProject: DevDiv + GitTagRepo: VS ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} AccessToken: $(System.AccessToken) mbpresteps: @@ -140,21 +145,31 @@ jobs: fetchDepth: 0 # avoid shallow clone so nbgv can do its work. clean: true - ${{ if parameters.EnableOptProf }}: - - powershell: Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSDROPNAME]$(tools/variables/ProfilingInputsDropName.ps1)" - displayName: βš™ Set ProfilingInputsDropName for optprof + - powershell: | + Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSDROPNAME]$(tools/variables/ProfilingInputsDropName.ps1)" + Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSPROPSNAME]$(tools/variables/ProfilingInputsPropsName.ps1)" + displayName: βš™ Setting variables for optprof sdl: binskim: analyzeTargetGlob: $(Build.ArtifactStagingDirectory)\symbols-Windows\** outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - - ${{ each artifact_name in parameters.artifact_names }}: - - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: + - ${{ each artifact in parameters.artifact_names }}: + - ${{ if or(ne(artifact.testOnly, 'true'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: πŸ“’ Publish ${{ artifact.name }}-Windows + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact.name }}-Windows + artifactName: ${{ artifact.name }}-Windows + ${{ if and(parameters.Is1ESPTOfficial, eq(artifact.sbomEnabled, 'true')) }}: + sbomEnabled: true - output: pipelineArtifact - displayName: πŸ“’ Publish ${{ artifact_name }}-Windows - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Windows - artifactName: ${{ artifact_name }}-Windows - condition: succeededOrFailed() + displayName: πŸ“’ Publish ${{ artifact.name }}-Windows (for failed attempts) + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact.name }}-Windows + artifactName: ${{ artifact.name }}-Windows-$(System.PhaseAttempt) + ${{ if and(parameters.Is1ESPTOfficial, eq(artifact.sbomEnabled, 'true')) }}: + sbomEnabled: true + condition: failed() - output: pipelineArtifact displayName: πŸ“’ Publish VSInsertion-Windows targetPath: $(Build.ArtifactStagingDirectory)/VSInsertion-Windows @@ -169,6 +184,15 @@ jobs: displayName: πŸ“’ Publish APIScanInputs targetPath: $(Build.ArtifactStagingDirectory)/APIScanInputs-Windows artifactName: APIScanInputs + - ${{ if parameters.EnableOptProf }}: + - output: artifactsDrop + displayName: πŸ“’ Publish to Artifact Services - ProfilingInputs + dropServiceURI: https://devdiv.artifacts.visualstudio.com + buildNumber: $(ProfilingInputsDropName) + sourcePath: $(Build.ArtifactStagingDirectory)\OptProf\ProfilingInputs + toLowerCase: false + retentionDays: 500 + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) steps: - ${{ if not(parameters.Is1ESPT) }}: - checkout: self @@ -178,9 +202,6 @@ jobs: - powershell: Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSDROPNAME]$(tools/variables/ProfilingInputsDropName.ps1)" displayName: βš™ Set ProfilingInputsDropName for optprof - - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: - - template: schedule-only-steps.yml - - template: install-dependencies.yml - script: dotnet nbgv cloud -ca @@ -191,8 +212,6 @@ jobs: - template: microbuild.before.yml parameters: EnableLocalization: ${{ parameters.EnableLocalization }} - EnableOptProf: ${{ parameters.EnableOptProf }} - IsOptProf: ${{ parameters.IsOptProf }} ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} RealSign: ${{ parameters.RealSign }} @@ -200,7 +219,6 @@ jobs: parameters: Is1ESPT: ${{ parameters.Is1ESPT }} RunTests: ${{ parameters.RunTests }} - IsOptProf: ${{ parameters.IsOptProf }} - ${{ if and(parameters.EnableDotNetFormatCheck, not(parameters.EnableLinuxBuild)) }}: - script: dotnet format --verify-no-changes @@ -211,123 +229,136 @@ jobs: - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - template: microbuild.after.yml parameters: - EnableOptProf: ${{ parameters.EnableOptProf }} - IsOptProf: ${{ parameters.IsOptProf }} SkipCodesignVerify: ${{ parameters.SkipCodesignVerify }} -- ${{ if not(parameters.IsOptProf) }}: - - ${{ if parameters.EnableLinuxBuild }}: - - job: Linux - pool: ${{ parameters.linuxPool }} - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - templateContext: - mb: - ${{ if parameters.RealSign }}: - signing: - enabled: false # enable when building unique artifacts on this agent that must be signed - signType: real - signWithProd: true - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - ${{ each artifact_name in parameters.artifact_names }}: - - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: - - output: pipelineArtifact - displayName: πŸ“’ Publish ${{ artifact_name }}-Linux - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Linux - artifactName: ${{ artifact_name }}-Linux - condition: succeededOrFailed() - steps: - - checkout: self - fetchDepth: 0 # avoid shallow clone so nbgv can do its work. - clean: true - - template: install-dependencies.yml - - template: dotnet.yml - parameters: - Is1ESPT: ${{ parameters.Is1ESPT }} - RunTests: ${{ parameters.RunTests }} - BuildRequiresAccessToken: ${{ parameters.RealSign }} # Real signing on non-Windows machines requires passing through access token to build steps that sign - - ${{ if parameters.EnableDotNetFormatCheck }}: - - script: dotnet format --verify-no-changes - displayName: πŸ’… Verify formatted code - env: - dotnetformat: true # part of a workaround for https://github.com/dotnet/sdk/issues/44951 - - - ${{ if parameters.EnableMacOSBuild }}: - - job: macOS - pool: ${{ parameters.macOSPool }} - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - templateContext: - mb: - ${{ if parameters.RealSign }}: - signing: - enabled: false # enable when building unique artifacts on this agent that must be signed - signType: real - signWithProd: true - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - ${{ each artifact_name in parameters.artifact_names }}: - - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: - - output: pipelineArtifact - displayName: πŸ“’ Publish ${{ artifact_name }}-macOS - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-macOS - artifactName: ${{ artifact_name }}-macOS - condition: succeededOrFailed() - steps: - - checkout: self - fetchDepth: 0 # avoid shallow clone so nbgv can do its work. - clean: true - - template: install-dependencies.yml - - template: dotnet.yml - parameters: - Is1ESPT: ${{ parameters.Is1ESPT }} - RunTests: ${{ parameters.RunTests }} - BuildRequiresAccessToken: ${{ parameters.RealSign }} # Real signing on non-Windows machines requires passing through access token to build steps that sign +- ${{ if parameters.EnableLinuxBuild }}: + - job: Linux + pool: ${{ parameters.linuxPool }} + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + templateContext: + mb: + ${{ if parameters.RealSign }}: + signing: + enabled: false # enable when building unique artifacts on this agent that must be signed + signType: real + signWithProd: true + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ each artifact in parameters.artifact_names }}: + - ${{ if or(ne(artifact.testOnly, 'true'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: πŸ“’ Publish ${{ artifact.name }}-Linux + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact.name }}-Linux + artifactName: ${{ artifact.name }}-Linux + ${{ if and(parameters.Is1ESPTOfficial, eq(artifact.sbomEnabled, 'true')) }}: + sbomEnabled: true + - output: pipelineArtifact + displayName: πŸ“’ Publish ${{ artifact.name }}-Linux (for failed attempts) + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact.name }}-Linux + artifactName: ${{ artifact.name }}-Linux-$(System.PhaseAttempt) + ${{ if and(parameters.Is1ESPTOfficial, eq(artifact.sbomEnabled, 'true')) }}: + sbomEnabled: true + condition: failed() + steps: + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true + - template: install-dependencies.yml + - template: dotnet.yml + parameters: + Is1ESPT: ${{ parameters.Is1ESPT }} + RunTests: ${{ parameters.RunTests }} + BuildRequiresAccessToken: ${{ parameters.RealSign }} # Real signing on non-Windows machines requires passing through access token to build steps that sign + - ${{ if parameters.EnableDotNetFormatCheck }}: + - script: dotnet format --verify-no-changes + displayName: πŸ’… Verify formatted code + env: + dotnetformat: true # part of a workaround for https://github.com/dotnet/sdk/issues/44951 - - job: WrapUp - dependsOn: - - Windows - - ${{ if parameters.EnableLinuxBuild }}: - - Linux - - ${{ if parameters.EnableMacOSBuild }}: - - macOS - pool: ${{ parameters.windowsPool }} # Use Windows agent because PublishSymbols task requires it (https://github.com/microsoft/azure-pipelines-tasks/issues/13821). - condition: succeededOrFailed() - variables: - ONEES_ENFORCED_CODEQL_ENABLED: false # CodeQL runs on build jobs, we don't need it here +- ${{ if parameters.EnableMacOSBuild }}: + - job: macOS + pool: ${{ parameters.macOSPool }} ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: templateContext: - ${{ if not(parameters.RealSign) }}: - mb: - signing: # if the build is test-signed, install the signing plugin so that CSVTestSignPolicy.xml is available - enabled: true - zipSources: false - signType: test + mb: + ${{ if parameters.RealSign }}: + signing: + enabled: false # enable when building unique artifacts on this agent that must be signed + signType: real + signWithProd: true outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - - output: pipelineArtifact - displayName: πŸ“’ Publish symbols-legacy - targetPath: $(Build.ArtifactStagingDirectory)/symbols-legacy - artifactName: symbols-legacy - condition: succeededOrFailed() + - ${{ each artifact in parameters.artifact_names }}: + - ${{ if or(ne(artifact.testOnly, 'true'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: πŸ“’ Publish ${{ artifact.name }}-macOS + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact.name }}-macOS + artifactName: ${{ artifact.name }}-macOS + ${{ if and(parameters.Is1ESPTOfficial, eq(artifact.sbomEnabled, 'true')) }}: + sbomEnabled: true + - output: pipelineArtifact + displayName: πŸ“’ Publish ${{ artifact.name }}-macOS (for failed attempts) + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact.name }}-macOS + artifactName: ${{ artifact.name }}-macOS-$(System.PhaseAttempt) + ${{ if and(parameters.Is1ESPTOfficial, eq(artifact.sbomEnabled, 'true')) }}: + sbomEnabled: true + condition: failed() steps: - checkout: self fetchDepth: 0 # avoid shallow clone so nbgv can do its work. clean: true - template: install-dependencies.yml + - template: dotnet.yml parameters: - initArgs: -NoRestore - - template: publish-symbols.yml + Is1ESPT: ${{ parameters.Is1ESPT }} + RunTests: ${{ parameters.RunTests }} + BuildRequiresAccessToken: ${{ parameters.RealSign }} # Real signing on non-Windows machines requires passing through access token to build steps that sign + +- job: WrapUp + dependsOn: + - Windows + - ${{ if parameters.EnableLinuxBuild }}: + - Linux + - ${{ if parameters.EnableMacOSBuild }}: + - macOS + pool: ${{ parameters.windowsPool }} # Use Windows agent because PublishSymbols task requires it (https://github.com/microsoft/azure-pipelines-tasks/issues/13821). + condition: succeededOrFailed() + variables: + ONEES_ENFORCED_CODEQL_ENABLED: false # CodeQL runs on build jobs, we don't need it here + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + templateContext: + ${{ if not(parameters.RealSign) }}: + mb: + signing: # if the build is test-signed, install the signing plugin so that CSVTestSignPolicy.xml is available + enabled: true + zipSources: false + signType: test + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + displayName: πŸ“’ Publish symbols-legacy + targetPath: $(Build.ArtifactStagingDirectory)/symbols-legacy + artifactName: symbols-legacy + condition: succeededOrFailed() + steps: + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true + - template: install-dependencies.yml + parameters: + initArgs: -NoRestore + - template: publish-symbols.yml + parameters: + EnableLinuxBuild: ${{ parameters.EnableLinuxBuild }} + EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} + - ${{ if and(parameters.RunTests, parameters.PublishCodeCoverage) }}: + - template: publish-codecoverage.yml parameters: EnableLinuxBuild: ${{ parameters.EnableLinuxBuild }} EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} - - ${{ if and(parameters.RunTests, parameters.PublishCodeCoverage) }}: - - template: publish-codecoverage.yml - parameters: - EnableLinuxBuild: ${{ parameters.EnableLinuxBuild }} - EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} - - ${{ if parameters.EnableAPIScan }}: - - template: apiscan.yml - parameters: - windowsPool: ${{ parameters.windowsPool }} - RealSign: ${{ parameters.RealSign }} +- ${{ if parameters.EnableAPIScan }}: + - template: apiscan.yml + parameters: + windowsPool: ${{ parameters.windowsPool }} + RealSign: ${{ parameters.RealSign }} diff --git a/azure-pipelines/dotnet.yml b/azure-pipelines/dotnet.yml index 947cfc1a..b67b4a62 100644 --- a/azure-pipelines/dotnet.yml +++ b/azure-pipelines/dotnet.yml @@ -1,8 +1,5 @@ parameters: - name: RunTests -- name: IsOptProf - type: boolean - default: false - name: Is1ESPT type: boolean - name: BuildRequiresAccessToken @@ -17,14 +14,9 @@ steps: env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) -- ${{ if not(parameters.IsOptProf) }}: - - powershell: tools/dotnet-test-cloud.ps1 -Configuration $(BuildConfiguration) -Agent $(Agent.JobName) -PublishResults - displayName: πŸ§ͺ dotnet test - condition: and(succeeded(), ${{ parameters.RunTests }}) - -- ${{ if parameters.IsOptProf }}: - - script: dotnet pack src\VSInsertionMetadata -c $(BuildConfiguration) -warnaserror /bl:"$(Build.ArtifactStagingDirectory)/build_logs/VSInsertion-Pack.binlog" - displayName: πŸ”§ dotnet pack VSInsertionMetadata +- powershell: tools/dotnet-test-cloud.ps1 -Configuration $(BuildConfiguration) -Agent $(Agent.JobName) -PublishResults + displayName: πŸ§ͺ dotnet test + condition: and(succeeded(), ${{ parameters.RunTests }}) - powershell: tools/variables/_define.ps1 failOnStderr: true @@ -42,11 +34,14 @@ steps: displayName: πŸ“’ Publish artifacts condition: succeededOrFailed() -- ${{ if and(ne(variables['codecov_token'], ''), parameters.RunTests) }}: +- ${{ if parameters.RunTests }}: - powershell: | $ArtifactStagingFolder = & "tools/Get-ArtifactsStagingDirectory.ps1" $CoverageResultsFolder = Join-Path $ArtifactStagingFolder "coverageResults-$(Agent.JobName)" - tools/publish-CodeCov.ps1 -CodeCovToken "$(codecov_token)" -PathToCodeCoverage "$CoverageResultsFolder" -Name "$(Agent.JobName) Coverage Results" -Flags "$(Agent.JobName)" + tools/publish-CodeCov.ps1 -CodeCovToken "$(CODECOV_TOKEN)" -PathToCodeCoverage "$CoverageResultsFolder" -Name "$(Agent.JobName) Coverage Results" -Flags "$(Agent.JobName)" displayName: πŸ“’ Publish code coverage results to codecov.io timeoutInMinutes: 3 continueOnError: true + # Set the CODECOV_TOKEN variable in your Azure Pipeline to enable code coverage reporting + # Get a token from https://codecov.io/ + condition: and(succeeded(), ne(variables['CODECOV_TOKEN'], '')) diff --git a/azure-pipelines/libtemplate-update.yml b/azure-pipelines/libtemplate-update.yml index 384be7cf..6d0051a0 100644 --- a/azure-pipelines/libtemplate-update.yml +++ b/azure-pipelines/libtemplate-update.yml @@ -30,6 +30,8 @@ variables: extends: template: azure-pipelines/MicroBuild.1ES.Unofficial.yml@MicroBuildTemplate parameters: + settings: + networkIsolationPolicy: Permissive,CFSClean2 sdl: sourceAnalysisPool: name: AzurePipelines-EO diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 025de4f5..c0e1e7bd 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -1,10 +1,4 @@ parameters: -- name: EnableOptProf - type: boolean - default: false -- name: IsOptProf - type: boolean - default: false - name: SkipCodesignVerify type: boolean @@ -18,21 +12,3 @@ steps: TargetFolders: | $(Build.SourcesDirectory)/bin/Packages/$(BuildConfiguration) condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -- ${{ if parameters.IsOptProf }}: - - task: ms-vscs-artifact.build-tasks.artifactDropTask-1.artifactDropTask@0 - inputs: - dropServiceURI: https://devdiv.artifacts.visualstudio.com - buildNumber: $(ProfilingInputsDropName) - sourcePath: $(Build.ArtifactStagingDirectory)\OptProf\ProfilingInputs - toLowerCase: false - usePat: true - displayName: πŸ“’ Publish to Artifact Services - ProfilingInputs - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - - - task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: $(Build.ArtifactStagingDirectory)/InsertionOutputs - ArtifactName: InsertionOutputs - ArtifactType: Container - displayName: πŸ“’ Publish InsertionOutputs as Azure DevOps artifacts diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index d09310b1..29823532 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -2,12 +2,6 @@ parameters: - name: EnableLocalization type: boolean default: false -- name: EnableOptProf - type: boolean - default: false -- name: IsOptProf - type: boolean - default: false - name: ShouldSkipOptimize type: boolean default: false @@ -15,7 +9,7 @@ parameters: type: boolean steps: -- ${{ if and(not(parameters.IsOptProf), ne(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: # notice@0 requires CG detection to run first, and non-default branches don't inject it automatically. # default branch injection (main) is happening too late for notice@0 to run successfully. Adding this as a workaround. - task: ComponentGovernanceComponentDetection@0 @@ -26,28 +20,5 @@ steps: inputs: outputfile: $(System.DefaultWorkingDirectory)/obj/NOTICE outputformat: text - retryCountOnTaskFailure: 3 # fails when the cloud service is overloaded + retryCountOnTaskFailure: 10 # fails when the cloud service is overloaded continueOnError: ${{ not(parameters.RealSign) }} # Tolerate failures when we're not building something that may ship. - -- ${{ if parameters.IsOptProf }}: - # We have to install these plugins ourselves for Optprof runs because those pipelines haven't migrated to 1ES PT yet. - - task: MicroBuildOptProfPlugin@6 - inputs: - ProfilingInputsDropName: $(ProfilingInputsDropName) - OptimizationInputsLookupMethod: DropPrefix - DropNamePrefix: OptimizationInputs/$(System.TeamProject)/$(Build.Repository.Name) - ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} - AccessToken: $(System.AccessToken) - displayName: πŸ”§ Install OptProf Plugin - - - task: MicroBuildSigningPlugin@4 - inputs: - signType: Test - zipSources: false - displayName: πŸ”§ Install MicroBuild Signing Plugin - - - ${{ if parameters.EnableLocalization }}: - - task: MicroBuildLocalizationPlugin@4 - inputs: - languages: $(LocLanguages) - displayName: πŸ”§ Install MicroBuild Localization Plugin diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 6d7c0213..9f0a89d8 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -45,12 +45,14 @@ variables: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: + settings: + networkIsolationPolicy: Permissive,CFSClean2 sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: VSEng-MicroBuildVSStable codeSignValidation: enabled: true break: true - additionalTargetsGlobPattern: -|Variables-*\*.ps1;-|LocBin-*\**;-|APIScanInputs-*\**;-|test_symbols-*\**;-|MicroBuild\** + additionalTargetsGlobPattern: -|Variables-*\*.ps1;-|LocBin-*\**;-|APIScanInputs-*\**;-|test_symbols-*\**;-|MicroBuild\**;-|$(Build.ArtifactStagingDirectory)\VSInsertion-Windows\vs-insertion-script.ps1 policheck: enabled: true exclusionsFile: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml @@ -66,10 +68,11 @@ extends: - template: /azure-pipelines/build.yml@self parameters: Is1ESPT: true + Is1ESPTOfficial: true RealSign: true # ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} EnableAPIScan: ${{ parameters.EnableAPIScan }} - windowsPool: VSEngSS-MicroBuild2022-1ES + windowsPool: VSEng-MicroBuildVSStable linuxPool: name: AzurePipelines-EO demands: @@ -77,7 +80,7 @@ extends: os: Linux macOSPool: name: Azure Pipelines - vmImage: macOS-14 + vmImage: macOS-15 os: macOS EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} RunTests: ${{ parameters.RunTests }} diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index fdce906a..9f934f92 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -17,7 +17,7 @@ stages: - ${{ if parameters.ArchiveSymbols }}: - job: symbol_archive displayName: Archive symbols - pool: VSEngSS-MicroBuild2022-1ES + pool: VSEng-MicroBuildVSStable variables: ONEES_ENFORCED_CODEQL_ENABLED: false # CodeQL runs on build stages, we don't need it here steps: diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index e9559ebe..244f6b43 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -20,13 +20,16 @@ variables: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: + settings: + networkIsolationPolicy: Permissive,CFSClean2 sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: VSEng-MicroBuildVSStable stages: - stage: release jobs: - - job: release + - job: nuget + displayName: πŸ“¦ Push nuget.org packages pool: name: AzurePipelines-EO demands: @@ -43,6 +46,28 @@ extends: publishFeedCredentials: VisualStudioExtensibility (nuget.org) steps: - checkout: none + - download: CI + artifact: deployables-Windows + displayName: πŸ”» Download deployables-Windows artifact + patterns: 'NuGet/*' + - job: github + displayName: πŸ“’ GitHub release + dependsOn: nuget + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals 1ESPT-Ubuntu22.04 + os: Linux + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + pipeline: CI + artifactName: deployables-Windows + targetPath: $(Pipeline.Workspace)/CI/deployables-Windows + steps: + - checkout: none - powershell: | Write-Host "##vso[build.updatebuildnumber]$(resources.pipeline.CI.runName)" if ('$(resources.pipeline.CI.runName)'.Contains('-')) { @@ -51,10 +76,6 @@ extends: Write-Host "##vso[task.setvariable variable=IsPrerelease]false" } displayName: βš™ Set up pipeline - - download: CI - artifact: deployables-Windows - displayName: πŸ”» Download deployables-Windows artifact - patterns: 'NuGet/*' - task: GitHubRelease@1 displayName: πŸ“’ GitHub release (create) inputs: diff --git a/azure-pipelines/schedule-only-steps.yml b/azure-pipelines/schedule-only-steps.yml deleted file mode 100644 index ad07a341..00000000 --- a/azure-pipelines/schedule-only-steps.yml +++ /dev/null @@ -1,3 +0,0 @@ -steps: -- powershell: echo "##vso[build.addbuildtag]auto-insertion" - displayName: Tag for auto-insertion diff --git a/azure-pipelines/unofficial.yml b/azure-pipelines/unofficial.yml index d7232735..47a2a2e2 100644 --- a/azure-pipelines/unofficial.yml +++ b/azure-pipelines/unofficial.yml @@ -55,8 +55,10 @@ variables: extends: template: azure-pipelines/MicroBuild.1ES.Unofficial.yml@MicroBuildTemplate parameters: + settings: + networkIsolationPolicy: Permissive,CFSClean2 sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: VSEng-MicroBuildVSStable credscan: enabled: false suppression: @@ -65,7 +67,7 @@ extends: codeSignValidation: enabled: ${{ parameters.EnableProductionSDL }} break: true - additionalTargetsGlobPattern: -|Variables-*\*.ps1;-|APIScanInputs-*\**;-|test_symbols-*\**;-|MicroBuild\** + additionalTargetsGlobPattern: -|Variables-*\*.ps1;-|APIScanInputs-*\**;-|test_symbols-*\**;-|MicroBuild\**;-|$(Build.ArtifactStagingDirectory)\VSInsertion-Windows\vs-insertion-script.ps1 policyFile: $(MBSIGN_APPFOLDER)\CSVTestSignPolicy.xml policheck: enabled: ${{ parameters.EnableProductionSDL }} @@ -83,7 +85,7 @@ extends: RealSign: false # ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} EnableAPIScan: ${{ parameters.EnableAPIScan }} - windowsPool: VSEngSS-MicroBuild2022-1ES + windowsPool: VSEng-MicroBuildVSStable linuxPool: name: AzurePipelines-EO demands: @@ -91,7 +93,7 @@ extends: os: Linux macOSPool: name: Azure Pipelines - vmImage: macOS-14 + vmImage: macOS-15 os: macOS EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} RunTests: ${{ parameters.RunTests }} diff --git a/azure-pipelines/vs-insertion-script.ps1 b/azure-pipelines/vs-insertion-script.ps1 new file mode 100644 index 00000000..53076254 --- /dev/null +++ b/azure-pipelines/vs-insertion-script.ps1 @@ -0,0 +1,17 @@ +# List of build artifact files [Source => Destination] to be committed into the VS repo. +$FilesToCommit = @{ + "$env:PROFILINGINPUTSPROPSNAME" = "src/Tests/config/runsettings/Official/OptProf/External/$env:PROFILINGINPUTSPROPSNAME"; +} + +foreach ($File in $FilesToCommit.GetEnumerator()) { + $SourcePath = Join-Path $PSScriptRoot $File.Key + if (Test-Path $SourcePath) { + $DestinationPath = Join-Path (Get-Location) $File.Value + Write-Host "Copying $SourcePath to $DestinationPath" + Copy-Item -Path $SourcePath -Destination $DestinationPath + git add $DestinationPath + } + else { + Write-Host "$SourcePath is not present, skipping" + } +} diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index a7daf393..8f8196be 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -15,7 +15,6 @@ resources: trigger: tags: - Real signed - - auto-insertion variables: - template: GlobalVariables.yml @@ -23,15 +22,19 @@ variables: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: + settings: + networkIsolationPolicy: Permissive,CFSClean2 sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: VSEng-MicroBuildVSStable + sbom: + enabled: false stages: - stage: insertion jobs: - job: insertion displayName: VS insertion - pool: VSEngSS-MicroBuild2022-1ES + pool: VSEng-MicroBuildVSStable templateContext: outputParentDirectory: $(Pipeline.Workspace)/CI steps: @@ -64,9 +67,15 @@ extends: InsertionPayloadName: $(Build.Repository.Name) $(Build.BuildNumber) InsertionBuildPolicies: Request Perf DDRITs InsertionReviewers: $(Build.RequestedFor),VS Core - Solution Experience # Append `,Your team name` (without quotes) + CustomScriptExecutionCommand: $(Pipeline.Workspace)\CI\VSInsertion-Windows\vs-insertion-script.ps1 AutoCompletePR: true AutoCompleteMergeStrategy: Squash ShallowClone: true + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + ConnectedVSDropServiceName: 'VSEng-VSDrop-MI' + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - powershell: | $contentType = 'application/json'; $headers = @{ Authorization = 'Bearer $(System.AccessToken)' }; diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index b9d46b7c..f6237313 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -26,8 +26,10 @@ variables: extends: template: azure-pipelines/MicroBuild.1ES.Unofficial.yml@MicroBuildTemplate parameters: + settings: + networkIsolationPolicy: Permissive,CFSClean2 sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: VSEng-MicroBuildVSStable credscan: enabled: false @@ -44,7 +46,7 @@ extends: Is1ESPT: true RealSign: false # ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} - windowsPool: VSEngSS-MicroBuild2022-1ES + windowsPool: VSEng-MicroBuildVSStable linuxPool: name: AzurePipelines-EO demands: @@ -52,7 +54,7 @@ extends: os: Linux macOSPool: name: Azure Pipelines - vmImage: macOS-14 + vmImage: macOS-15 os: macOS EnableMacOSBuild: false RunTests: false @@ -68,7 +70,7 @@ extends: jobs: - job: insertion displayName: VS insertion - pool: VSEngSS-MicroBuild2022-1ES + pool: VSEng-MicroBuildVSStable steps: - checkout: self clean: true @@ -103,12 +105,16 @@ extends: InsertionPayloadName: $(Build.Repository.Name) VALIDATION BUILD $(Build.BuildNumber) ($(Build.SourceBranch)) [Skip-SymbolCheck] [Skip-HashCheck] [Skip-SignCheck] InsertionDescription: | This PR is for **validation purposes only** for !$(System.PullRequest.PullRequestId). **Do not complete**. - CustomScriptExecutionCommand: src/VSSDK/NuGet/AllowUnstablePackages.ps1 + CustomScriptExecutionCommand: $(Pipeline.Workspace)\VSInsertion-Windows\vs-insertion-script.ps1; src\VSSDK\NuGet\AllowUnstablePackages.ps1 InsertionBuildPolicies: Request Perf DDRITs InsertionReviewers: $(Build.RequestedFor) DraftPR: false # set to true and update InsertionBuildPolicy when we can specify all the validations we want to run (https://dev.azure.com/devdiv/DevDiv/_workitems/edit/2224288) AutoCompletePR: false ShallowClone: true + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + ConnectedVSDropServiceName: 'VSEng-VSDrop-MI' + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) - powershell: | $insertionPRId = azure-pipelines/Get-InsertionPRId.ps1 $Markdown = @" diff --git a/global.json b/global.json index a5461699..b95fef0b 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,12 @@ { "sdk": { - "version": "9.0.305", + "version": "10.0.200", "rollForward": "patch", "allowPrerelease": false }, + "test": { + "runner": "Microsoft.Testing.Platform" + }, "msbuild-sdks": { "Microsoft.Build.NoTargets": "3.7.56" } diff --git a/src/OptProf.targets b/src/OptProf.targets index d0167d7c..0b6cc92c 100644 --- a/src/OptProf.targets +++ b/src/OptProf.targets @@ -5,11 +5,8 @@ Common7\IDE\PrivateAssemblies\$(TargetFileName) /ExeConfig:"%VisualStudio.InstallationUnderTest.Path%\Common7\IDE\vsn.exe" - - - + - diff --git a/src/VSInsertionMetadata/Library.VSInsertionMetadata.proj b/src/VSInsertionMetadata/Library.VSInsertionMetadata.proj deleted file mode 100644 index 0caaedb9..00000000 --- a/src/VSInsertionMetadata/Library.VSInsertionMetadata.proj +++ /dev/null @@ -1,11 +0,0 @@ - - - netstandard2.0 - true - $(RepoRootPath)bin\Packages\$(Configuration)\VSRepo\ - false - false - Contains metadata for insertion into VS. - - - diff --git a/src/VSInsertionMetadata/ProfilingInputs.props b/src/VSInsertionMetadata/ProfilingInputs.props deleted file mode 100644 index fb19d604..00000000 --- a/src/VSInsertionMetadata/ProfilingInputs.props +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/src/VSInsertionMetadata/VSInsertionMetadata.targets b/src/VSInsertionMetadata/VSInsertionMetadata.targets deleted file mode 100644 index 84ebb5a8..00000000 --- a/src/VSInsertionMetadata/VSInsertionMetadata.targets +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - $(TargetsForTfmSpecificContentInPackage); - SubstituteProfilingInputsMacro; - - - - - - - - - - - - - - - - - - - - - - $(PackageVersion).$(Build_BuildId) - - - - - - diff --git a/test/Directory.Build.props b/test/Directory.Build.props index 395359dd..466be54e 100644 --- a/test/Directory.Build.props +++ b/test/Directory.Build.props @@ -5,6 +5,7 @@ false true + true diff --git a/test/Directory.Build.targets b/test/Directory.Build.targets index 9f32cd06..3758bb8c 100644 --- a/test/Directory.Build.targets +++ b/test/Directory.Build.targets @@ -1,5 +1,11 @@ + + + + + + diff --git a/test/Microsoft.VisualStudio.SolutionPersistence.Tests/Microsoft.VisualStudio.SolutionPersistence.Tests.csproj b/test/Microsoft.VisualStudio.SolutionPersistence.Tests/Microsoft.VisualStudio.SolutionPersistence.Tests.csproj index f0663bf9..de8660de 100644 --- a/test/Microsoft.VisualStudio.SolutionPersistence.Tests/Microsoft.VisualStudio.SolutionPersistence.Tests.csproj +++ b/test/Microsoft.VisualStudio.SolutionPersistence.Tests/Microsoft.VisualStudio.SolutionPersistence.Tests.csproj @@ -12,9 +12,7 @@ - - - + diff --git a/tools/Get-3rdPartySymbolFiles.ps1 b/tools/Get-ExternalSymbolFiles.ps1 similarity index 93% rename from tools/Get-3rdPartySymbolFiles.ps1 rename to tools/Get-ExternalSymbolFiles.ps1 index ef6bbef2..5ad2a196 100644 --- a/tools/Get-3rdPartySymbolFiles.ps1 +++ b/tools/Get-ExternalSymbolFiles.ps1 @@ -79,11 +79,13 @@ Function Get-PackageVersion($id) { $version } -# All 3rd party packages for which symbols packages are expected should be listed here. +# All 1st party packages for which symbols packages are expected should be listed here. # These must all be sourced from nuget.org, as it is the only feed that supports symbol packages. -$3rdPartyPackageIds = @() +# We should NOT add 3rd party packages to this list because PDBs may be unsafe for our debuggers to load, +# so we should only archive 1st party symbols. +$1stPartyPackageIds = @() -$3rdPartyPackageIds | % { +$1stPartyPackageIds | % { $version = Get-PackageVersion $_ if ($version) { Get-SymbolsFromPackage -id $_ -version $version diff --git a/tools/Install-DotNetSdk.ps1 b/tools/Install-DotNetSdk.ps1 index 402b4307..590581e2 100644 --- a/tools/Install-DotNetSdk.ps1 +++ b/tools/Install-DotNetSdk.ps1 @@ -125,14 +125,14 @@ Function Get-InstallerExe( } if ($TypedVersion.Build -eq -1) { - $versionInfo = -Split (Invoke-WebRequest -Uri "https://dotnetcli.blob.core.windows.net/dotnet/$sku/$Version/latest.version" -UseBasicParsing) + $versionInfo = -Split (Invoke-WebRequest -Uri "https://builds.dotnet.microsoft.com/dotnet/$sku/$Version/latest.version" -UseBasicParsing) $Version = $versionInfo[-1] } $majorMinor = "$($TypedVersion.Major).$($TypedVersion.Minor)" $ReleasesFile = Join-Path $DotNetInstallScriptRoot "$majorMinor\releases.json" if (!(Test-Path $ReleasesFile)) { - Get-FileFromWeb -Uri "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/$majorMinor/releases.json" -OutDir (Split-Path $ReleasesFile) | Out-Null + Get-FileFromWeb -Uri "https://builds.dotnet.microsoft.com/dotnet/release-metadata/$majorMinor/releases.json" -OutDir (Split-Path $ReleasesFile) | Out-Null } $releases = Get-Content $ReleasesFile | ConvertFrom-Json @@ -197,7 +197,7 @@ if ($InstallLocality -eq 'machine') { $restartRequired = $false $sdks |% { if ($_.Version) { $version = $_.Version } else { $version = $_.Channel } - if ($PSCmdlet.ShouldProcess(".NET SDK $_", "Install")) { + if ($PSCmdlet.ShouldProcess(".NET SDK $version ($arch)", "Install")) { Install-DotNet -Version $version -Architecture $arch $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) @@ -281,10 +281,10 @@ if ($IncludeX86) { } if ($IsMacOS -or $IsLinux) { - $DownloadUri = "https://raw.githubusercontent.com/dotnet/install-scripts/0b09de9bc136cacb5f849a6957ebd4062173c148/src/dotnet-install.sh" + $DownloadUri = "https://raw.githubusercontent.com/dotnet/install-scripts/a3fbd0fd625032bac207f1f590e5353fe26faa59/src/dotnet-install.sh" $DotNetInstallScriptPath = "$DotNetInstallScriptRoot/dotnet-install.sh" } else { - $DownloadUri = "https://raw.githubusercontent.com/dotnet/install-scripts/0b09de9bc136cacb5f849a6957ebd4062173c148/src/dotnet-install.ps1" + $DownloadUri = "https://raw.githubusercontent.com/dotnet/install-scripts/a3fbd0fd625032bac207f1f590e5353fe26faa59/src/dotnet-install.ps1" $DotNetInstallScriptPath = "$DotNetInstallScriptRoot/dotnet-install.ps1" } @@ -306,7 +306,7 @@ $global:LASTEXITCODE = 0 $sdks |% { if ($_.Version) { $parameters = '-Version', $_.Version } else { $parameters = '-Channel', $_.Channel } - if ($PSCmdlet.ShouldProcess(".NET SDK $_", "Install")) { + if ($PSCmdlet.ShouldProcess(".NET SDK $_ ($arch)", "Install")) { $anythingInstalled = $true Invoke-Expression -Command "$DotNetInstallScriptPathExpression $parameters -Architecture $arch -InstallDir $DotNetInstallDir $switches" diff --git a/tools/Prepare-Legacy-Symbols.ps1 b/tools/Prepare-Legacy-Symbols.ps1 index ae0bc40c..8a007c2a 100644 --- a/tools/Prepare-Legacy-Symbols.ps1 +++ b/tools/Prepare-Legacy-Symbols.ps1 @@ -20,6 +20,20 @@ Get-ChildItem "$ArtifactStagingFolder\*.pdb" -Recurse |% { } if ($BinaryImagePath) { + # Native binaries can't have their PDBs converted to legacy (Windows) format so just skip them + try { + [System.Reflection.AssemblyName]::GetAssemblyName($BinaryImagePath) | Out-Null + $isManaged = $true + } + catch { + $isManaged = $false + } + + if (-not $isManaged) { + Write-Host "Skipping native binary PDB: $_" -ForegroundColor DarkYellow + continue + } + # Convert the PDB to legacy Windows PDBs Write-Host "Converting PDB for $_" -ForegroundColor DarkGray $WindowsPdbDir = "$($_.Directory.FullName)\$WindowsPdbSubDirName" diff --git a/tools/artifacts/VSInsertion.ps1 b/tools/artifacts/VSInsertion.ps1 index a5b940b5..700b24f6 100644 --- a/tools/artifacts/VSInsertion.ps1 +++ b/tools/artifacts/VSInsertion.ps1 @@ -19,6 +19,10 @@ if (!$BuildConfiguration) { $PackagesRoot = "$RepoRoot/bin/Packages/$BuildConfiguration" $NuGetPackages = "$PackagesRoot/NuGet" $VsixPackages = "$PackagesRoot/Vsix" +$AzurePipelinesPath = "$RepoRoot/azure-pipelines" +if ($env:BUILD_ARTIFACTSTAGINGDIRECTORY) { + $InsertionOutputs = "$env:BUILD_ARTIFACTSTAGINGDIRECTORY/InsertionOutputs" +} if (!(Test-Path $NuGetPackages) -and !(Test-Path $VsixPackages)) { Write-Warning "Skipping because NuGet and VSIX packages haven't been built yet." @@ -26,16 +30,16 @@ if (!(Test-Path $NuGetPackages) -and !(Test-Path $VsixPackages)) { } $result = @{ - "$NuGetPackages" = (Get-ChildItem $NuGetPackages -Recurse) + "$AzurePipelinesPath" = (Get-ChildItem "$AzurePipelinesPath/vs-insertion-script.ps1"); + "$NuGetPackages" = (Get-ChildItem $NuGetPackages -Recurse); } if (Test-Path $VsixPackages) { $result["$PackagesRoot"] += Get-ChildItem $VsixPackages -Recurse } -if ($env:IsOptProf) { - $VSRepoPackages = "$PackagesRoot/VSRepo" - $result["$VSRepoPackages"] = (Get-ChildItem "$VSRepoPackages\*.VSInsertionMetadata.*.nupkg"); +if ($InsertionOutputs -and $env:PROFILINGINPUTSPROPSNAME -and (Test-Path "$InsertionOutputs/$env:PROFILINGINPUTSPROPSNAME")) { + $result[$InsertionOutputs] = (Get-ChildItem "$InsertionOutputs/$env:PROFILINGINPUTSPROPSNAME"); # OptProf ProfilingInputs } $result diff --git a/tools/artifacts/Variables.ps1 b/tools/artifacts/Variables.ps1 index 7a320c7e..c4d97665 100644 --- a/tools/artifacts/Variables.ps1 +++ b/tools/artifacts/Variables.ps1 @@ -26,7 +26,7 @@ Get-ChildItem "$PSScriptRoot/../variables" |% { if ($value) { # We got something, so wrap it with quotes so it's treated like a literal value. - $value = "'$value'" + $value = "'" + $value.Replace("'", "''") + "'" } } diff --git a/tools/artifacts/coverageResults.ps1 b/tools/artifacts/coverageResults.ps1 index 8c68216e..1aadbb74 100644 --- a/tools/artifacts/coverageResults.ps1 +++ b/tools/artifacts/coverageResults.ps1 @@ -1,25 +1,26 @@ -$RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..") +$RepoRoot = Resolve-Path "$PSScriptRoot\..\.." -$coverageFiles = @(Get-ChildItem "$RepoRoot/test/*.cobertura.xml" -Recurse | Where {$_.FullName -notlike "*/In/*" -and $_.FullName -notlike "*\In\*" }) +$coverageFilesUnderRoot = @(Get-ChildItem "$RepoRoot/*.cobertura.xml" -Recurse | Where-Object {$_.FullName -notlike "*/In/*" -and $_.FullName -notlike "*\In\*" }) + +# Under MTP, coverage files are written directly to the artifacts output directory, +# so we need to look there too. +$ArtifactStagingFolder = & "$PSScriptRoot/../Get-ArtifactsStagingDirectory.ps1" +$directTestLogs = Join-Path $ArtifactStagingFolder test_logs +$coverageFilesUnderArtifacts = if (Test-Path $directTestLogs) { @(Get-ChildItem "$directTestLogs/*.cobertura.xml" -Recurse) } else { @() } # Prepare code coverage reports for merging on another machine -$repoRoot = $env:SYSTEM_DEFAULTWORKINGDIRECTORY -if (!$repoRoot) { $repoRoot = $env:GITHUB_WORKSPACE } -if ($repoRoot) { - Write-Host "Substituting $repoRoot with `"{reporoot}`"" - $coverageFiles |% { - $content = Get-Content -LiteralPath $_ |% { $_ -Replace [regex]::Escape($repoRoot), "{reporoot}" } - Set-Content -LiteralPath $_ -Value $content -Encoding UTF8 - } -} else { - Write-Warning "coverageResults: Cloud build not detected. Machine-neutral token replacement skipped." +Write-Host "Substituting $repoRoot with `"{reporoot}`"" +@($coverageFilesUnderRoot + $coverageFilesUnderArtifacts) |? { $_ }|% { + $content = Get-Content -LiteralPath $_ |% { $_ -Replace [regex]::Escape($repoRoot), "{reporoot}" } + Set-Content -LiteralPath $_ -Value $content -Encoding UTF8 } if (!((Test-Path $RepoRoot\bin) -and (Test-Path $RepoRoot\obj))) { return } @{ + $directTestLogs = $coverageFilesUnderArtifacts; $RepoRoot = ( - $coverageFiles + + $coverageFilesUnderRoot + (Get-ChildItem "$RepoRoot\obj\*.cs" -Recurse) ); } diff --git a/tools/artifacts/symbols.ps1 b/tools/artifacts/symbols.ps1 index b5882678..91f83f0d 100644 --- a/tools/artifacts/symbols.ps1 +++ b/tools/artifacts/symbols.ps1 @@ -1,10 +1,10 @@ $BinPath = [System.IO.Path]::GetFullPath("$PSScriptRoot/../../bin") -$3rdPartyPath = [System.IO.Path]::GetFullPath("$PSScriptRoot/../../obj/SymbolsPackages") +$ExternalPath = [System.IO.Path]::GetFullPath("$PSScriptRoot/../../obj/SymbolsPackages") if (!(Test-Path $BinPath)) { return } $symbolfiles = & "$PSScriptRoot/../Get-SymbolFiles.ps1" -Path $BinPath | Get-Unique -$3rdPartyFiles = & "$PSScriptRoot/../Get-3rdPartySymbolFiles.ps1" +$ExternalFiles = & "$PSScriptRoot/../Get-ExternalSymbolFiles.ps1" @{ "$BinPath" = $SymbolFiles; - "$3rdPartyPath" = $3rdPartyFiles; + "$ExternalPath" = $ExternalFiles; } diff --git a/tools/artifacts/testResults.ps1 b/tools/artifacts/testResults.ps1 index 5310fb52..a841967e 100644 --- a/tools/artifacts/testResults.ps1 +++ b/tools/artifacts/testResults.ps1 @@ -4,7 +4,8 @@ Param( $result = @{} -$testRoot = Resolve-Path "$PSScriptRoot\..\..\test" +$RepoRoot = Resolve-Path "$PSScriptRoot\..\.." +$testRoot = Join-Path $RepoRoot test $result[$testRoot] = (Get-ChildItem "$testRoot\TestResults" -Recurse -Directory | Get-ChildItem -Recurse -File) $artifactStaging = & "$PSScriptRoot/../Get-ArtifactsStagingDirectory.ps1" diff --git a/tools/dotnet-test-cloud.ps1 b/tools/dotnet-test-cloud.ps1 index 02891d97..5d204819 100644 --- a/tools/dotnet-test-cloud.ps1 +++ b/tools/dotnet-test-cloud.ps1 @@ -25,6 +25,7 @@ Param( $RepoRoot = (Resolve-Path "$PSScriptRoot/..").Path $ArtifactStagingFolder = & "$PSScriptRoot/Get-ArtifactsStagingDirectory.ps1" +$OnCI = ($env:CI -or $env:TF_BUILD) $dotnet = 'dotnet' if ($x86) { @@ -45,23 +46,68 @@ if ($x86) { } $testBinLog = Join-Path $ArtifactStagingFolder (Join-Path build_logs test.binlog) -$testDiagLog = Join-Path $ArtifactStagingFolder (Join-Path test_logs diag.log) +$testLogs = Join-Path $ArtifactStagingFolder test_logs -& $dotnet test $RepoRoot ` - --no-build ` - -c $Configuration ` - --filter "TestCategory!=FailsInCloudTest" ` - --collect "Code Coverage;Format=cobertura" ` - --settings "$PSScriptRoot/test.runsettings" ` - --blame-hang-timeout 60s ` - --blame-crash ` - -bl:"$testBinLog" ` - --diag "$testDiagLog;TraceLevel=info" ` - --logger trx ` +$globalJson = Get-Content $PSScriptRoot/../global.json | ConvertFrom-Json +$isMTP = $globalJson.test.runner -eq 'Microsoft.Testing.Platform' +$extraArgs = @() +$failedTests = 0 + +if ($isMTP) { + if ($OnCI) { $extraArgs += '--no-progress' } + + $dumpSwitches = @( + ,'--hangdump' + ,'--hangdump-timeout','120s' + ,'--crashdump' + ) + $mtpArgs = @( + ,'--coverage' + ,'--coverage-output-format','cobertura' + ,'--diagnostic' + ,'--diagnostic-output-directory',$testLogs + ,'--diagnostic-verbosity','Information' + ,'--results-directory',$testLogs + ,'--report-trx' + ) + + & $dotnet test --solution $RepoRoot ` + --no-build ` + -c $Configuration ` + -bl:"$testBinLog" ` + --filter-not-trait 'TestCategory=FailsInCloudTest' ` + --coverage-settings "$PSScriptRoot/test.runsettings" ` + @mtpArgs ` + @dumpSwitches ` + @extraArgs + if ($LASTEXITCODE -ne 0) { $failedTests += 1 } + + $trxFiles = Get-ChildItem -Recurse -Path $testLogs\*.trx +} else { + $testDiagLog = Join-Path $ArtifactStagingFolder (Join-Path test_logs diag.log) + & $dotnet test $RepoRoot ` + --no-build ` + -c $Configuration ` + --filter "TestCategory!=FailsInCloudTest" ` + --collect "Code Coverage;Format=cobertura" ` + --settings "$PSScriptRoot/test.runsettings" ` + --blame-hang-timeout 60s ` + --blame-crash ` + -bl:"$testBinLog" ` + --diag "$testDiagLog;TraceLevel=info" ` + --logger trx ` + @extraArgs + if ($LASTEXITCODE -ne 0) { $failedTests += 1 } + + $trxFiles = Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx +} $unknownCounter = 0 -Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx |% { - Copy-Item $_ -Destination $ArtifactStagingFolder/test_logs/ +$trxFiles |% { + New-Item $testLogs -ItemType Directory -Force | Out-Null + if (!($_.FullName.StartsWith($testLogs, [StringComparison]::OrdinalIgnoreCase))) { + Copy-Item $_ -Destination $testLogs + } if ($PublishResults) { $x = [xml](Get-Content -LiteralPath $_) @@ -84,3 +130,7 @@ Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx |% { Write-Host "##vso[results.publish type=VSTest;runTitle=$runTitle;publishRunAttachments=true;resultFiles=$_;failTaskOnFailedTests=true;testRunSystem=VSTS - PTR;]" } } + +if ($failedTests -ne 0) { + exit $failedTests +} diff --git a/tools/variables/InsertJsonValues.ps1 b/tools/variables/InsertJsonValues.ps1 index 7aa8d30c..fb9390c5 100644 --- a/tools/variables/InsertJsonValues.ps1 +++ b/tools/variables/InsertJsonValues.ps1 @@ -8,11 +8,25 @@ $BasePath = "$PSScriptRoot\..\..\bin\Packages\$BuildConfiguration\Vsix" if (Test-Path $BasePath) { $vsmanFiles = @() - Get-ChildItem $BasePath *.vsman -Recurse -File |% { + Get-ChildItem $BasePath *.vsman -Recurse -File | % { $version = (Get-Content $_.FullName | ConvertFrom-Json).info.buildVersion + $fullPath = (Resolve-Path $_.FullName).Path + $basePath = (Resolve-Path $BasePath).Path + # Cannot use RelativePath or GetRelativePath due to Powershell Core v2.0 limitation + if ($fullPath.StartsWith($basePath, [StringComparison]::OrdinalIgnoreCase)) { + # Get the relative paths then make sure the directory separators match URL format. + $rfn = $fullPath.Substring($basePath.Length).TrimStart('\', '/').Replace('\', '/') + } + else { + $rfn = $fullPath # fallback to full path if it doesn't start with base path + } + $fn = $_.Name - $vsmanFiles += "$fn{$version}=https://vsdrop.corp.microsoft.com/file/v1/$vstsDropNames;$fn" + + # The left side is filename followed by the version and the right side is the drop url and the relative filename + $thisVsManFile = "$fn{$version}=https://vsdrop.corp.microsoft.com/file/v1/$vstsDropNames;$rfn" + $vsmanFiles += $thisVsManFile } - [string]::join(',',$vsmanFiles) + [string]::join(',', $vsmanFiles) } diff --git a/tools/variables/ProfilingInputsPropsName.ps1 b/tools/variables/ProfilingInputsPropsName.ps1 new file mode 100644 index 00000000..9e6bc4a1 --- /dev/null +++ b/tools/variables/ProfilingInputsPropsName.ps1 @@ -0,0 +1,6 @@ +if ($env:SYSTEM_TEAMPROJECT) { + $repoName = $env:BUILD_REPOSITORY_NAME.Replace('/', '.') + "$env:SYSTEM_TEAMPROJECT.$repoName.props" +} else { + Write-Warning "No Azure Pipelines build detected. No profiling inputs filename will be computed." +}