diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index 827b97f33ac..3cc556049a4 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -509,6 +509,70 @@ stages: continueOnError: true condition: failed() + # Windows Apex VS integration tests (non-gating — published for signal only) + - job: WindowsApexIntegration + variables: + - name: _configuration + value: Release + - name: __VSNeverShowWhatsNew + value: 1 + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(_WindowsMachineQueueName) + timeoutInMinutes: 90 + steps: + - checkout: self + clean: true + + - powershell: eng\SetupVSHive.ps1 + displayName: Setup VS Hive + + # Detect the installed VS's Roslyn version and set FSHARP_APEX_ROSLYN_VERSION so the VSIX below + # is built against it (the repo's forward Roslyn pin is newer than any CI/scout VS, so a VSIX + # built against the pin cannot load its Roslyn assemblies in the installed VS). Non-fatal. + - powershell: eng\SetApexRoslynVersion.ps1 + displayName: Detect CI VS Roslyn version for Apex VSIX build + + # Build + deploy the F# VSIX into the RoslynDev hive, then run the Apex tests. The Apex host + # resolves the installed VS via LocateVisualStudio (set as VisualStudio.InstallationUnderTest.Path + # by the -testApex path), so it does not depend on the local-only 'Canary' milestone. + # continueOnError keeps this leg non-gating: failures surface as signal but do not fail the PR. + - script: eng\CIBuildNoPublish.cmd -configuration $(_configuration) -deployExtensions -testApex + env: + NativeToolsOnMachine: true + displayName: Build, deploy extension and run Apex integration tests + continueOnError: true + + - task: PublishTestResults@2 + displayName: Publish Apex Test Results + inputs: + testResultsFormat: 'VSTest' + testRunTitle: WindowsApexIntegration + mergeTestResults: true + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)' + continueOnError: true + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: Publish Apex Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration)' + ArtifactName: Windows $(_configuration) Apex test logs + publishLocation: Container + continueOnError: true + condition: always() + + - task: PublishBuildArtifacts@1 + displayName: Publish Apex Build Logs + condition: always() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\$(_configuration)' + ArtifactName: Windows $(_configuration) Apex build logs + ArtifactType: Container + parallel: true + # Windows With Compressed Metadata Desktop (split into 3 batches) - job: WindowsCompressedMetadata_Desktop strategy: diff --git a/azure-pipelines-integration-dartlab.yml b/azure-pipelines-integration-dartlab.yml new file mode 100644 index 00000000000..3c7a09f6c5c --- /dev/null +++ b/azure-pipelines-integration-dartlab.yml @@ -0,0 +1,78 @@ +# F# VS integration (Apex) test gate — runs against a Visual Studio installed via the VS +# bootstrapper on internal DartLab test machines, so the VS matches what this repo targets. +# +# SCAFFOLD — modeled on dotnet/roslyn's azure-pipelines-integration-dartlab.yml. +# This pipeline is INTERNAL-only and requires DevDiv/DartLab access that is not part of this +# repo. See eng/pipelines/apex-integration/README.md for the P0 prerequisites and every +# `# TODO(P0):` below for the real values that must be supplied before it can run. + +trigger: none +pr: none + +resources: + repositories: + - repository: DartLabTemplates + type: git + name: DevDiv/DartLab.Templates + ref: refs/heads/main + - repository: VSTemplates + type: git + name: DevDiv/VS.Templates + ref: refs/heads/main + # TODO(P0): provision an internal mirror of dotnet/fsharp and a matching service connection. + # Roslyn uses `internal/dotnet-roslyn` via the `dnceng-internal-code-access` endpoint. + - repository: FSharpMirror + endpoint: dnceng-internal-code-access + type: git + name: internal/dotnet-fsharp + ref: $(Build.SourceBranch) + # TODO(P0): the VS build under test supplies the bootstrapper to install. Interim: omit this and + # hard-code `visualStudioBootstrapperURI` in stage.yml to a pinned int.main Products drop. End + # state: reference the VS build pipeline so this runs as a gate on the consuming VS build. + # - pipeline: VisualStudioBuildUnderTest + # source: '' + +parameters: + - name: prNumber + type: string + default: 'None' + - name: sha + type: string + default: 'None' + - name: EnforceLatestCommit + type: boolean + default: true + # '(default)' => stage.yml resolves the bootstrapper from the VS-build-under-test drop. + - name: visualStudioBootstrapperURI + type: string + default: '(default)' + - name: testCaseFilter + type: string + default: '(all)' + # Set to 'stop' to keep the test machine around for investigation after the run. + - name: testMachineCleanUpStrategy + displayName: Test Machine Clean Up Strategy + type: string + default: delete + values: + - delete + - stop + # Retention period for the test machine (format "D:H:M:S"). + - name: testMachineRetention + displayName: Test Machine Retention (format "D:H:M:S") + type: string + default: 1:0:0:0 + +extends: + template: /DartLab/1ES/pipelines/ci/real-sign.yml@VSTemplates + parameters: + testStages: + - template: /eng/pipelines/apex-integration/stage.yml@self + parameters: + prNumber: ${{ parameters.prNumber }} + sha: ${{ parameters.sha }} + EnforceLatestCommit: ${{ parameters.EnforceLatestCommit }} + visualStudioBootstrapperURI: ${{ parameters.visualStudioBootstrapperURI }} + testCaseFilter: ${{ parameters.testCaseFilter }} + testMachineCleanUpStrategy: ${{ parameters.testMachineCleanUpStrategy }} + testMachineRetention: ${{ parameters.testMachineRetention }} diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 41a52df6395..6149e223f33 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -60,6 +60,7 @@ param ( [switch]$testIntegration, [switch]$testScripting, [switch]$testVs, + [switch]$testApex, [switch]$testAll, [switch]$testAllButIntegration, [switch]$testAllButIntegrationAndAot, @@ -128,6 +129,7 @@ function Print-Usage() { Write-Host " -testIntegration Run F# integration tests" Write-Host " -testScripting Run Scripting tests" Write-Host " -testVs Run F# editor unit tests" + Write-Host " -testApex Run F# Apex VS integration tests (requires -deployExtensions)" Write-Host " -testpack Verify built packages" Write-Host " -testAOT Run AOT/Trimming tests" Write-Host " -testEditor Run VS Editor tests" @@ -402,6 +404,245 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str Exec-Console $dotnetExe $test_args } +# Runs the Apex VS integration tests. These are a classic MSTest (VSTest) project, NOT part of the +# repo's Microsoft.Testing.Platform 'dotnet test' path, and they drive a real VS instance via the Apex +# UI-automation framework — so they are launched with vstest.console.exe (from the installed VS). +# The project is intentionally excluded from VisualFSharp.slnx (its dependencies must version-match the +# VS it drives), so this builds it explicitly. It also must match a specific VS version +# (MicrosoftTestApexVisualStudioVersion); if the installed VS does not match, the run is skipped rather +# than producing confusing Apex/VS-mismatch failures. The F# VSIX must be deployed into the RoslynDev +# hive first (build with -deployExtensions); Apex launches devenv /rootsuffix RoslynDev. + +# ===== BEGIN Apex CI diagnostics helpers (Phase 2, TEMPORARY — revert once the CI hang is understood) ===== +# On CI the Apex host failed with "Timed out ... trying get running object": devenv started but never +# registered its DTE automation object in the ROT. These helpers (a) make sure devenv has an interactive +# console desktop (the usual reason DTE never registers) and (b) collect the RoslynDev-hive logs so we can +# tell an interactive-desktop problem apart from an F# VSIX / package load failure. +function Capture-ApexScreenshot([string] $path) { + # Returns $true only if a screenshot could be taken, which implies an interactive desktop is attached. + try { + Add-Type -AssemblyName System.Windows.Forms -ErrorAction Stop + Add-Type -AssemblyName System.Drawing -ErrorAction Stop + $bounds = [System.Windows.Forms.SystemInformation]::VirtualScreen + $bmp = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height + $gfx = [System.Drawing.Graphics]::FromImage($bmp) + try { + $gfx.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size) + $bmp.Save($path, [System.Drawing.Imaging.ImageFormat]::Png) + Write-Host "Captured screenshot to '$path'." + return $true + } finally { + $gfx.Dispose(); $bmp.Dispose() + } + } catch { + Write-Host "Screenshot capture failed (no interactive desktop?): $($_.Exception.Message)" + return $false + } +} + +function Prepare-ApexInteractiveSession([string] $screenshotPath) { + # Mirrors dotnet/roslyn's Setup-IntegrationTestRun: if the desktop is not interactive, reconnect the + # session to the console so devenv can publish its DTE automation object; then minimize other windows. + if (-not (Capture-ApexScreenshot $screenshotPath)) { + Write-Host "No interactive desktop detected; attempting to reconnect the session to the console." + try { + $quserItems = ((quser $env:USERNAME | Select-Object -Skip 1) -split '\s+') + $sessionId = $quserItems[2] + if ($sessionId -eq 'Disc') { $sessionId = $quserItems[1] } + Write-Host "tscon $sessionId /dest:console" + tscon $sessionId /dest:console + Start-Sleep -Seconds 3 + [void](Capture-ApexScreenshot $screenshotPath) + } catch { + Write-Host "##vso[task.logissue type=warning]Failed to reconnect session to console: $($_.Exception.Message)" + } + } + + try { + (New-Object -ComObject "Shell.Application").MinimizeAll() + } catch { + Write-Host "MinimizeAll failed: $($_.Exception.Message)" + } +} + +function Collect-ApexDiagnostics([string] $screenshotPath, [string] $destination) { + # After-run screenshot plus the RoslynDev experimental-hive logs. ActivityLog.xml records whether the + # F# package failed to load; ComponentModelCache\*.err records MEF composition failures. + [void](Capture-ApexScreenshot $screenshotPath) + + $roamingVs = Join-Path $env:USERPROFILE "AppData\Roaming\Microsoft\VisualStudio" + $localVs = Join-Path $env:USERPROFILE "AppData\Local\Microsoft\VisualStudio" + + if (Test-Path $roamingVs) { + foreach ($hiveDir in @(Get-ChildItem -Path $roamingVs -Directory -Filter "*RoslynDev" -ErrorAction SilentlyContinue)) { + foreach ($name in @("ActivityLog.xml", "ActivityLog.xsl")) { + $src = Join-Path $hiveDir.FullName $name + if (Test-Path $src) { + Copy-Item $src (Join-Path $destination "$($hiveDir.Name)-$name") -Force -ErrorAction SilentlyContinue + Write-Host "Collected $src" + } + } + } + } + if (Test-Path $localVs) { + foreach ($hiveDir in @(Get-ChildItem -Path $localVs -Directory -Filter "*RoslynDev" -ErrorAction SilentlyContinue)) { + $mefErr = Join-Path $hiveDir.FullName "ComponentModelCache\Microsoft.VisualStudio.Default.err" + if (Test-Path $mefErr) { + Copy-Item $mefErr (Join-Path $destination "$($hiveDir.Name)-Microsoft.VisualStudio.Default.err") -Force -ErrorAction SilentlyContinue + Write-Host "Collected $mefErr" + } + } + } +} +# ===== END Apex CI diagnostics helpers (Phase 2) ===== + +function TestApexUsingVSTest([string] $targetFramework) { + $projectName = "FSharp.Editor.Apex.IntegrationTests" + $apexProject = Join-Path $RepoRoot "vsintegration\tests\$projectName\$projectName.csproj" + + # Create the results directory up-front so the CI 'publish test logs' step never fails on a missing + # path, even when the run is skipped below (a marker file records the skip reason). + $testResultsDir = "$ArtifactsDir\TestResults\$configuration" + Create-Directory $testResultsDir + + # The Apex package (MicrosoftTestApexVisualStudioVersion) is pinned to a VS 18.x build. Because the + # installed VS differs by machine (CI's image trails a dev's local install and they rarely share the + # same minor), gate on the MAJOR version by default so any VS 18.x can run the tests; set + # FSHARP_APEX_VS_VERSION to a stricter prefix (e.g. "18.9") to force an exact minor. + $expectedVsVersion = if (${env:FSHARP_APEX_VS_VERSION}) { ${env:FSHARP_APEX_VS_VERSION} } else { "18" } + + # Determine which VS will run the tests, and gate on its version. An explicit + # VisualStudio.InstallationUnderTest.Path override is trusted (gate skipped); otherwise enumerate + # ALL installed VS instances (not just the latest) and pick one whose version starts with the target + # prefix, skipping the run if none match rather than producing confusing Apex/VS-mismatch errors. + $overridePath = ${env:VisualStudio.InstallationUnderTest.Path} + if ($overridePath) { + Write-Host "Using explicit VisualStudio.InstallationUnderTest.Path=$overridePath (VS version gate skipped)." + $vsDir = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $overridePath)) + } + else { + $vswhere = Join-Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe" + # Query the properties separately (parallel arrays in a stable order) instead of parsing + # `-format json`: Windows PowerShell 5.1's ConvertFrom-Json mishandles vswhere's multi-instance + # JSON array (it collapses the instances into one object with array-valued properties), which + # would make every version comparison fail. Build.ps1 runs under Windows PowerShell 5.1. + $versions = @() + $paths = @() + $names = @() + if (Test-Path $vswhere) { + $versions = @(& $vswhere -all -prerelease -products * -property installationVersion) + $paths = @(& $vswhere -all -prerelease -products * -property installationPath) + $names = @(& $vswhere -all -prerelease -products * -property displayName) + } + $found = $versions -join ', ' + + # Print the full VS inventory for diagnostics (helps explain a skip on CI). Always logged. + Write-Host "Visual Studio instances found by vswhere ($($versions.Count)) [vswhere: $vswhere]:" + if ($versions.Count -eq 0) { + Write-Host " (none)" + } + for ($k = 0; $k -lt $versions.Count; $k++) { + $mm = ($versions[$k] -split '\.')[0..1] -join '.' + $hasIde = Test-Path (Join-Path $paths[$k] "Common7\IDE\devenv.exe") + $name = if ($k -lt $names.Count) { $names[$k] } else { "" } + Write-Host (" [{0}] version={1} (major.minor {2}) devenv={3} name='{4}' path={5}" -f $k, $versions[$k], $mm, $hasIde, $name, $paths[$k]) + } + + # Among installs whose version starts with the target prefix (major, or an explicit major.minor) + # and that actually have an IDE (devenv.exe — excludes Build Tools / incomplete installs), pick + # the LOWEST version: it is closest to the pinned Apex minor, and "Apex older than VS" is the + # safer cross-minor direction than the reverse. + $vsDir = $null + $selectedVersion = $null + $versionMatchPath = $null + $candidates = for ($k = 0; $k -lt $versions.Count; $k++) { + if ($versions[$k] -like "$expectedVsVersion.*") { + [PSCustomObject]@{ Version = $versions[$k]; Path = $paths[$k] } + } + } + $candidates = @($candidates | Sort-Object { [version]$_.Version }) + foreach ($c in $candidates) { + if (-not $versionMatchPath) { $versionMatchPath = $c.Path } + if (Test-Path (Join-Path $c.Path "Common7\IDE\devenv.exe")) { + $vsDir = $c.Path.TrimEnd("\") + $selectedVersion = $c.Version + break + } + } + + if (-not $vsDir) { + if ($versionMatchPath) { + Write-Host "##vso[task.logissue type=warning]Found VS $expectedVsVersion ($versionMatchPath) but no devenv.exe under Common7\IDE (incomplete install, Build Tools, or a VS update in progress); skipping Apex integration tests (non-gating)." + Write-Host "Skipping Apex tests: VS $expectedVsVersion present but devenv.exe missing at [$versionMatchPath]. If VS was updating, retry once it finishes." + } + else { + Write-Host "##vso[task.logissue type=warning]No installed VS matches the Apex target $expectedVsVersion (found: $found); skipping Apex integration tests (non-gating)." + Write-Host "Skipping Apex tests: no VS $expectedVsVersion found among installs [$found]. Set FSHARP_APEX_VS_VERSION or VisualStudio.InstallationUnderTest.Path to run against a different install." + } + Set-Content -Path (Join-Path $testResultsDir "apex-tests-skipped.txt") -Value "Apex integration tests skipped: no usable VS matching target $expectedVsVersion (installed: $found)." + return + } + Write-Host "Selected VS $selectedVersion at '$vsDir' for the Apex tests (target $expectedVsVersion)." + # This process env var propagates to the child vstest.console -> testhost -> Apex processes. + ${env:VisualStudio.InstallationUnderTest.Path} = Join-Path $vsDir "Common7\IDE\devenv.exe" + } + + $vstestConsole = @( + (Join-Path $vsDir "Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe"), + (Join-Path $vsDir "Common7\IDE\Extensions\TestPlatform\vstest.console.exe") + ) | Where-Object { Test-Path $_ } | Select-Object -First 1 + if (-not $vstestConsole) { + throw "vstest.console.exe not found under '$vsDir' (looked in CommonExtensions\Microsoft\TestWindow and Extensions\TestPlatform)." + } + + # Build the Apex test project explicitly (it is intentionally excluded from VisualFSharp.slnx). + $dotnetPath = InitializeDotNetCli + $dotnetExe = Join-Path $dotnetPath "dotnet.exe" + $buildBinLog = "$LogDir\${projectName}_$targetFramework.binlog" + Exec-Console $dotnetExe "build `"$apexProject`" -c $configuration /bl:`"$buildBinLog`"" + + $testAssembly = Join-Path $ArtifactsDir "bin\$projectName\$configuration\$targetFramework\$projectName.dll" + if (-not (Test-Path $testAssembly)) { + throw "Apex test assembly not found at '$testAssembly' after build." + } + + $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" } + $trxFileName = "$projectName.$targetFramework.$jobName.trx" + + $vstestArgs = @( + $testAssembly, + "/Platform:x64", + "/Framework:.NETFramework,Version=v4.7.2", + "/logger:trx;LogFileName=$trxFileName", + "/ResultsDirectory:$testResultsDir" + ) + + Write-Host "$vstestConsole $($vstestArgs -join ' ')" + + # ===== BEGIN Apex CI diagnostics (Phase 2, TEMPORARY — revert once the CI hang is understood) ===== + if ($ci) { + Prepare-ApexInteractiveSession (Join-Path $testResultsDir "apex-before-run.png") + } + # ===== END Apex CI diagnostics (Phase 2) ===== + + & $vstestConsole @vstestArgs + $vstestExitCode = $LASTEXITCODE + + # ===== BEGIN Apex CI diagnostics (Phase 2, TEMPORARY — revert once the CI hang is understood) ===== + if ($ci) { + Collect-ApexDiagnostics (Join-Path $testResultsDir "apex-after-run.png") $testResultsDir + } + # ===== END Apex CI diagnostics (Phase 2) ===== + + # This leg is non-gating (signal only): a test failure must NOT fail the build. Surface it as a + # warning and let the published TRX / Tests tab carry the signal. Genuine infra errors (missing + # assembly / vstest.console) already threw above and remain hard failures. + if ($vstestExitCode -ne 0) { + Write-Host "##vso[task.logissue type=warning]Apex integration tests reported failures (vstest.console exit code $vstestExitCode). This leg is non-gating; see the published TRX and the Tests tab." + Write-Host "Apex tests exit code $vstestExitCode (non-gating; not failing the build)." + } +} + function Prepare-TempDir() { Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.props") $TempDir Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.targets") $TempDir @@ -669,6 +910,10 @@ try { TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework } + if ($testApex) { + TestApexUsingVSTest -targetFramework $script:desktopTargetFramework + } + if ($testAOT) { Push-Location "$RepoRoot\tests\AheadOfTime" ./check.ps1 diff --git a/eng/SetApexRoslynVersion.ps1 b/eng/SetApexRoslynVersion.ps1 new file mode 100644 index 00000000000..e95f42e721f --- /dev/null +++ b/eng/SetApexRoslynVersion.ps1 @@ -0,0 +1,82 @@ +# Detects the Roslyn version of the *installed* Visual Studio and exports it as the pipeline variable +# FSHARP_APEX_ROSLYN_VERSION so the Apex integration-test leg builds the F# VSIX against it. +# +# Why: the repo pins Roslyn forward (to the VS being inserted into), which is newer than any VS on the CI +# scout image. A VSIX built against that pin cannot bind Microsoft.VisualStudio.LanguageServices at +# runtime in the older CI VS, so FSharpPackage/FSharpProjectPackage fail to load and every Apex test +# fails. The F# editor references Roslyn compile-only (ExcludeAssets=runtime), so building against the +# VS's own Roslyn version is sufficient and ships nothing extra. The eng/Versions.props override consumes +# this variable; the committed pin is unchanged. If detection fails we leave the variable unset (the +# build falls back to the committed pin) rather than break the build. +[CmdletBinding()] +param() + +$ErrorActionPreference = 'Stop' + +function Write-Skip([string] $message) { + Write-Host "##vso[task.logissue type=warning]$message" + Write-Host "Apex Roslyn override not set; the VSIX will build against the repo's pinned Roslyn." +} + +$vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe' +if (-not (Test-Path $vswhere)) { + Write-Skip "vswhere.exe not found; cannot detect the installed VS Roslyn version." + return +} + +# Select the same install the Apex gate runs against: the lowest matching (default major '18') VS that +# actually has devenv.exe. On CI there is a single VS, so any selection resolves to it. +$prefix = if (${env:FSHARP_APEX_VS_VERSION}) { ${env:FSHARP_APEX_VS_VERSION} } else { '18' } +$versions = @(& $vswhere -all -prerelease -products * -property installationVersion) +$paths = @(& $vswhere -all -prerelease -products * -property installationPath) + +$candidates = for ($i = 0; $i -lt $versions.Count; $i++) { + if ($versions[$i] -like "$prefix.*") { + [PSCustomObject]@{ Version = $versions[$i]; Path = $paths[$i] } + } +} +$candidates = @($candidates | Sort-Object { [version]$_.Version }) + +$vsDir = $null +foreach ($c in $candidates) { + if (Test-Path (Join-Path $c.Path 'Common7\IDE\devenv.exe')) { $vsDir = $c.Path.TrimEnd('\'); break } +} +if (-not $vsDir) { + Write-Skip "No installed VS $prefix.x with devenv.exe found; cannot detect the Roslyn version." + return +} + +$lsDll = Join-Path $vsDir 'Common7\IDE\CommonExtensions\Microsoft\VBCSharp\LanguageServices\Microsoft.VisualStudio.LanguageServices.dll' +if (-not (Test-Path $lsDll)) { + $lsDll = Get-ChildItem -Path $vsDir -Recurse -Filter 'Microsoft.VisualStudio.LanguageServices.dll' -ErrorAction SilentlyContinue | + Select-Object -First 1 -ExpandProperty FullName +} +if (-not $lsDll -or -not (Test-Path $lsDll)) { + Write-Skip "Microsoft.VisualStudio.LanguageServices.dll not found under '$vsDir'." + return +} + +$asmVersion = [System.Reflection.AssemblyName]::GetAssemblyName($lsDll).Version +$majorMinor = "$($asmVersion.Major).$($asmVersion.Minor)" +Write-Host "Installed VS '$vsDir' ships Roslyn assembly $asmVersion (major.minor $majorMinor)." + +# Resolve a matching Microsoft.CodeAnalysis NuGet version. The whole Roslyn family is published together +# with the same version, and every '..0-*' build has assembly version ..0.0, +# so any such package binds to the installed VS at runtime; pick the newest for freshness. +$feed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3' +try { + $allVersions = (Invoke-RestMethod -Uri "$feed/flat2/microsoft.codeanalysis.externalaccess.fsharp/index.json" -UseBasicParsing).versions +} catch { + Write-Skip "Failed to query the dotnet-tools feed for Roslyn versions: $($_.Exception.Message)" + return +} + +$selected = @($allVersions | Where-Object { $_ -match "^$([regex]::Escape($majorMinor))\.0-" }) | + Sort-Object | Select-Object -Last 1 +if (-not $selected) { + Write-Skip "No Roslyn package '$majorMinor.0-*' found on the feed to match the installed VS." + return +} + +Write-Host "Selected Roslyn NuGet version '$selected' for the Apex VSIX build (matches VS Roslyn $majorMinor)." +Write-Host "##vso[task.setvariable variable=FSHARP_APEX_ROSLYN_VERSION]$selected" diff --git a/eng/SetupVSHive.ps1 b/eng/SetupVSHive.ps1 index 50dcf7531d0..b6ec083d97f 100644 --- a/eng/SetupVSHive.ps1 +++ b/eng/SetupVSHive.ps1 @@ -12,6 +12,46 @@ $vsRegEdit = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'VSRegEdit $hive = "RoslynDev" &$vsRegEdit set "$vsDir" $hive HKCU "Roslyn\Internal\OnOff\Features" OOP64Bit dword 0 +# Suppress modal dialogs and online-profile prompts that block UI automation (Apex) and cause the +# integration tests to hang until they time out. These mirror the settings dotnet/roslyn applies to its +# RoslynDev hive before running VS integration tests. + +# Disable the editor "report exceptions" dialog: fail silently and keep the test running instead of +# popping a modal that stalls automation. +&$vsRegEdit set "$vsDir" $hive HKCU "Text Editor" "Report Exceptions" dword 0 + +# Disable roaming settings so an online user profile / sign-in prompt can't interfere on CI. +&$vsRegEdit set "$vsDir" $hive HKCU "ApplicationPrivateSettings\Microsoft\VisualStudio" RoamingEnabled string "1*System.Boolean*False" + +# Disable background download UI to avoid toasts appearing over the IDE during a run. +&$vsRegEdit set "$vsDir" $hive HKCU "FeatureFlags\Setup\BackgroundDownload" Value dword 0 + +# Disable targeted (remote settings) notifications. This one can't be set via VsRegEdit, so write it +# directly to the user hive (it is not hive-suffix specific). +reg add hkcu\Software\Microsoft\VisualStudio\RemoteSettings /f /t REG_DWORD /v TurnOffSwitch /d 1 | Out-Null + +# Initialize the experimental hive's first-launch state non-interactively. On a fresh hive devenv shows a +# full-screen "Sign in to Visual Studio" first-launch experience that blocks the UI thread, so it never +# registers its DTE automation object and Apex times out waiting for it. Importing settings via +# /resetsettings completes first launch without that dialog. This mirrors what the +# Microsoft.VisualStudio.Extensibility.Testing harness does before running VS integration tests on CI. + +# Start VS with an empty environment (no start window / start page) on launch. +&$vsRegEdit set "$vsDir" $hive HKCU "General" OnEnvironmentStartup dword 10 + +$devenv = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'devenv.exe' +$firstLaunchLog = Join-Path $env:TEMP "vs-firstlaunch-$hive.log" +Write-Host "Initializing '$hive' hive first launch: devenv /rootsuffix $hive /resetsettings General.vssettings /command File.Exit" +$devenvArgs = @("/rootsuffix", $hive, "/resetsettings", "General.vssettings", "/command", "File.Exit", "/logFile:$firstLaunchLog") +$proc = Start-Process -FilePath $devenv -ArgumentList $devenvArgs -PassThru +if (-not $proc.WaitForExit(180000)) { + Write-Host "##vso[task.logissue type=warning]devenv first-launch init did not exit within 180s; terminating it." + try { Stop-Process -Id $proc.Id -Force } catch {} +} +else { + Write-Host "devenv first-launch init exited with code $($proc.ExitCode)." +} + Write-Host "-- VS Info --" $isolationIni = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'devenv.isolation.ini' Get-Content $isolationIni | Write-Host diff --git a/eng/Versions.props b/eng/Versions.props index 60486fabcaa..94aa71dc8f0 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -130,6 +130,8 @@ $(VisualStudioEditorPackagesVersion) $(VisualStudioEditorPackagesVersion) 17.14.0 + 18.7.0-preview-1-11715-043 + 3.6.4 0.1.800-beta $(MicrosoftVisualStudioExtensibilityTestingVersion) @@ -165,4 +167,32 @@ + + + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + $(FSHARP_APEX_ROSLYN_VERSION) + + diff --git a/eng/pipelines/apex-integration/README.md b/eng/pipelines/apex-integration/README.md new file mode 100644 index 00000000000..42b6e54d92e --- /dev/null +++ b/eng/pipelines/apex-integration/README.md @@ -0,0 +1,81 @@ +# F# Apex VS integration tests — DartLab pipeline (scaffold) + +This directory contains an **internal DevDiv DartLab** pipeline that runs the F# Apex VS +integration tests (`vsintegration/tests/FSharp.Editor.Apex.IntegrationTests`) against a +Visual Studio build that **matches** what this repo targets, by *installing* that VS on a +fresh test machine via the VS bootstrapper. + +It is modeled directly on dotnet/roslyn's +[`azure-pipelines-integration-dartlab.yml`](https://github.com/dotnet/roslyn/blob/main/azure-pipelines-integration-dartlab.yml) +and its `eng/pipelines/test-gates/*` stage templates. + +## Why this exists + +The F# editor VSIX is pinned (Roslyn, and the dotnet/runtime `System.*` family) to the VS it +inserts into (F# `main` → VS `main`, ~18.10). Visual Studio has **no binding redirects for our +assemblies**, so a VSIX built against those pins only loads in a matching (or newer) VS. The +public PR CI scout image trails that VS (e.g. 18.7), and the mismatch cannot be worked around +by downgrading dependencies (the VS-SDK imposes a floor — e.g. `Microsoft.VisualStudio.RpcContracts` +requires `System.Composition >= 10.0.8`, so a downgrade fails restore with NU1605). + +The industry-standard fix (used by both NuGet/NuGet.Client and dotnet/roslyn) is to **install a +matching VS via the bootstrapper on internal DartLab test machines**, not to use a pre-baked older +public image. + +## Prerequisites that must be provisioned before this can run (P0 — external) + +These are **not** contained in this repo and require coordination with the DevDiv / DartLab / VS +team (as Roslyn did). The YAML here uses `# TODO(P0):` markers wherever a real value is required. + +1. **DartLab + VS pipeline templates access** — the pipeline `extends`/references + `DevDiv/DartLab.Templates` and `DevDiv/VS.Templates` (internal AzDO repos). +2. **Internal source mirror** — an `internal/dotnet-fsharp` mirror reachable via a + `dnceng-internal-code-access` service connection (Roslyn uses `internal/dotnet-roslyn`). The + test machine checks out from this mirror, not from GitHub. +3. **VS-Platform test lab pool** + **1ES** onboarding for a new internal pipeline definition. +4. **F# DevDiv area path / owner** for `templateContext` (Roslyn uses + `mlinfraswat` / `DevDiv\NET Developer Experience\CSharp and VB IDE`). +5. **VS-build-under-test source** — where the matching VS bootstrapper comes from: + - *Interim:* pin a recent VS `int.main` Products drop and hard-code its bootstrapper URI. + - *End state:* wire a `VisualStudioBuildUnderTest` pipeline resource and compute the drop name + with DartLab's `Get-VisualStudioDropName.ps1` (as Roslyn does), so the tests run as a gate on + the VS build that consumes the F# insertion. + +## Files + +- `../../azure-pipelines-integration-dartlab.yml` — pipeline entry (`trigger: none`), extends the + VS real-sign template and wires resources. +- `stage.yml` — the DartLab VS test stage: provisions a `VS-Platform` machine, installs VS via the + bootstrapper with an F#-minimal component set, then deploys + runs. +- `integration-job.yml` — deploy the F# VSIX + run the Apex tests via `eng/Build.ps1 -testApex` + against the freshly installed VS; publish the TRX. + +## Once matching-VS runs are green + +The interim public-CI workarounds become unnecessary and should be retired (see plan P4): +- `eng/SetApexRoslynVersion.ps1` and the `FSHARP_APEX_ROSLYN_VERSION` override group in + `eng/Versions.props`. +Keep `eng/SetupVSHive.ps1` (first-launch/registry prep) and the `/NoSigninPrompt` launch argument. + +## Concrete request to send to the DartLab / VS test team + +Onboarding is the same shape Roslyn used. Ask for / decide: + +1. Read access for the F# pipeline's service identity to the internal AzDO repos + `DevDiv/DartLab.Templates` and `DevDiv/VS.Templates`. +2. An internal Git mirror of `dotnet/fsharp` (e.g. `internal/dotnet-fsharp`) and a + `dnceng-internal-code-access` service connection so the test machine can check it out. +3. Use of the `VS-Platform` test-lab pool for the new pipeline, and 1ES registration of + `azure-pipelines-integration-dartlab.yml` as an internal pipeline. +4. The F# `templateContext.owner` and `areaPath` to record against test results + (fill into `eng/pipelines/apex-integration/stage.yml`). +5. The VS build to test against: + - *Interim:* a specific VS `int.main` **Products** drop whose + `bootstrappers/Enterprise/vs_enterprise.exe` we can pin as `visualStudioBootstrapperURI`. + - *End state:* register a `VisualStudioBuildUnderTest` pipeline resource so the F# tests run as + a gate on the VS build that consumes the F# insertion (uncomment the resource + the two + `preTestMachineConfigurationStepList` steps in `stage.yml`). + +After that: fill in every `# TODO(P0):` marker, register the pipeline, and iterate to green. Then do +plan P4 (retire the version overrides) and P5 (disable the public `WindowsApexIntegration` job). + diff --git a/eng/pipelines/apex-integration/integration-job.yml b/eng/pipelines/apex-integration/integration-job.yml new file mode 100644 index 00000000000..9d8e3c86f95 --- /dev/null +++ b/eng/pipelines/apex-integration/integration-job.yml @@ -0,0 +1,72 @@ +# Deploy the F# VSIX and run the Apex VS integration tests against the VS installed on this +# DartLab machine (at C:\Test\VisualStudio). SCAFFOLD — mirrors dotnet/roslyn's +# eng/pipelines/test-integration-job.yml, reusing this repo's existing -testApex path. +# +# Because the installed VS MATCHES the repo's pins, the interim public-CI version overrides are +# NOT used here (no eng/SetApexRoslynVersion.ps1 / FSHARP_APEX_ROSLYN_VERSION). Only the +# first-launch/registry prep and the /NoSigninPrompt launch arg from the public leg are reused. + +parameters: + - name: configuration + type: string + default: 'Release' + - name: testCaseFilter + type: string + default: '' + +steps: + # Checkout is performed by the DartLab deployAndRunTestsStepList, so skip it here. + + # Point Apex at the VS the bootstrapper installed on this machine. + - powershell: | + $devenv = 'C:\Test\VisualStudio\Common7\IDE\devenv.exe' + if (-not (Test-Path $devenv)) { Write-Host "##vso[task.logissue type=error]devenv not found at $devenv"; exit 1 } + Write-Host "##vso[task.setvariable variable=VisualStudio.InstallationUnderTest.Path]$devenv" + Write-Host "Apex will target: $devenv" + displayName: Select installed VS for Apex + + - task: PowerShell@2 + displayName: Enable Long Paths + inputs: + targetType: 'inline' + script: reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f + + # First-launch / experimental-hive registry prep + non-interactive settings import (shared with + # the public leg). Suppresses the sign-in ISE so devenv reaches automation-ready state. + - powershell: eng\SetupVSHive.ps1 + displayName: Setup VS Hive + + # Build F#, deploy the VSIX into the RoslynDev hive, and run the Apex tests via -testApex. + # The installed VS matches the repo pins, so no Roslyn/System.* override is applied. + - script: eng\CIBuildNoPublish.cmd -configuration ${{ parameters.configuration }} -deployExtensions -testApex + env: + NativeToolsOnMachine: true + displayName: Build, deploy extension and run Apex integration tests + + - task: PublishTestResults@2 + displayName: Publish Apex Test Results + inputs: + testResultsFormat: 'VSTest' + testRunTitle: 'FSharpVSIntegration ${{ parameters.configuration }}' + mergeTestResults: true + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}' + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: Publish Apex Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\${{ parameters.configuration }}' + ArtifactName: 'FSharp Apex ${{ parameters.configuration }} test logs $(System.JobAttempt)' + publishLocation: Container + continueOnError: true + condition: always() + + - task: PublishBuildArtifacts@1 + displayName: Publish Apex Build Logs + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\${{ parameters.configuration }}' + ArtifactName: 'FSharp Apex ${{ parameters.configuration }} build logs $(System.JobAttempt)' + publishLocation: Container + continueOnError: true + condition: always() diff --git a/eng/pipelines/apex-integration/stage.yml b/eng/pipelines/apex-integration/stage.yml new file mode 100644 index 00000000000..e523b08d9b6 --- /dev/null +++ b/eng/pipelines/apex-integration/stage.yml @@ -0,0 +1,119 @@ +# F# Apex VS integration test stage (DartLab). SCAFFOLD — mirrors dotnet/roslyn's +# eng/pipelines/test-gates/roslyn-vs-integration/stage.yml, trimmed to the F# scenario. +# See eng/pipelines/apex-integration/README.md and every `# TODO(P0):` below. + +parameters: +- name: prNumber + type: string + default: 'None' +- name: sha + type: string + default: 'None' +- name: EnforceLatestCommit + type: boolean + default: true +- name: visualStudioBootstrapperURI + type: string + default: '(default)' +- name: testCaseFilter + type: string + default: '' +- name: testMachineCleanUpStrategy + type: string + default: delete +- name: testMachineRetention + type: string + default: 1:0:0:0 +- name: dartLabTemplatesRepositoryAlias + type: string + default: DartLabTemplates +- name: fsharpRepositoryAlias + type: string + default: self + +stages: +- template: /stages/visual-studio/agent.yml@${{ parameters.dartLabTemplatesRepositoryAlias }} + parameters: + name: FSharpVSIntegration + displayName: F# VS Integration (Apex) + testLabPoolName: VS-Platform + testMachineCleanUpStrategy: ${{ parameters.testMachineCleanUpStrategy }} + testMachineRetention: ${{ parameters.testMachineRetention }} + templateContext: + # TODO(P0): set the F# DevDiv owner / areaPath (Roslyn uses + # `mlinfraswat` / `DevDiv\NET Developer Experience\CSharp and VB IDE`). + owner: fsharp + areaPath: DevDiv\NET Developer Experience\FSharp + # '(default)' => install the VS from the build-under-test Products drop. Interim: replace the + # whole expression with a hard-coded bootstrapper URI for a pinned int.main Products drop, e.g. + # 'https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/VS/int.main/;bootstrappers/Enterprise/vs_enterprise.exe'. + visualStudioBootstrapperURI: ${{ iif(eq(parameters.visualStudioBootstrapperURI, '(default)'), 'https://vsdrop.corp.microsoft.com/file/v1/$(VisualStudio.BuildUnderTest.ProductsDropName);bootstrappers/Enterprise/vs_enterprise.exe', parameters.visualStudioBootstrapperURI) }} + # F#-minimal component set: core editor + MSBuild + .NET SDK/targeting packs + Roslyn language + # services + the F# components + the VS SDK (so the locally-built F# VSIX can be deployed/loaded). + # Expand only if a test needs more; every added component increases install time. + visualStudioInstallationParameters: >- + --add Microsoft.VisualStudio.Component.CoreEditor + --add Microsoft.VisualStudio.Workload.CoreEditor + --add Microsoft.Component.MSBuild + --add Microsoft.NetCore.Component.SDK + --add Microsoft.NetCore.Component.Runtime.8.0 + --add Microsoft.NetCore.Component.DevelopmentTools + --add Microsoft.Net.Component.4.7.2.TargetingPack + --add Microsoft.Net.Component.4.8.SDK + --add Microsoft.VisualStudio.Component.Roslyn.Compiler + --add Microsoft.VisualStudio.Component.Roslyn.LanguageServices + --add Microsoft.VisualStudio.Component.FSharp + --add Microsoft.VisualStudio.Component.FSharp.Desktop + --add Microsoft.VisualStudio.Component.ManagedDesktop.Core + --add Microsoft.VisualStudio.Component.ManagedDesktop.Prerequisites + --add Microsoft.VisualStudio.Workload.ManagedDesktop + --add Microsoft.Component.CodeAnalysis.SDK + --add Microsoft.VisualStudio.Component.VSSDK + --add Microsoft.VisualStudio.ComponentGroup.VisualStudioExtension.Prerequisites + --add Microsoft.VisualStudio.Workload.VisualStudioExtension + --installPath "C:\Test\VisualStudio" --quiet --norestart --wait + testExecutionJobStrategy: + matrix: + release: + _configuration: Release + testMachineTotalCount: 1 + testAgentElevated: true + preTestMachineConfigurationStepList: + - checkout: none + # TODO(P0): only needed for the gate/end-state where the VS bootstrapper comes from a VS build + # resource. For the interim pinned-drop approach (hard-coded bootstrapper URI above), delete + # these two steps. + # - task: DownloadBuildArtifacts@1 + # displayName: Download Visual Studio build artifacts + # inputs: + # buildType: specific + # project: $(resources.pipeline.VisualStudioBuildUnderTest.projectID) + # definition: $(resources.pipeline.VisualStudioBuildUnderTest.pipelineID) + # buildVersionToDownload: specific + # buildId: $(resources.pipeline.VisualStudioBuildUnderTest.runID) + # downloadType: specific + # artifactName: BuildArtifacts + # downloadPath: $(Pipeline.Workspace)\VisualStudioBuildUnderTest + # - task: PowerShell@2 + # name: SetProductsDropName + # displayName: Set 'VisualStudio.BuildUnderTest.ProductsDropName' + # inputs: + # filePath: $(DartLab.Path)\Scripts\VisualStudio\Build\Get-VisualStudioDropName.ps1 + # arguments: -DropNamePrefix 'Products' -VstsDropUrlsJson '$(Pipeline.Workspace)\VisualStudioBuildUnderTest\BuildArtifacts\VstsDropUrls.json' -OutVariableName 'VisualStudio.BuildUnderTest.ProductsDropName' + deployAndRunTestsStepList: + # TODO(P0): check out the internal mirror (see azure-pipelines-integration-dartlab.yml resources). + - checkout: FSharpMirror + fetchDepth: 1 + fetchTags: false + persistCredentials: true + - ${{ if ne(parameters.prNumber, 'None') }}: + - task: PowerShell@2 + displayName: Setup branch for validation + inputs: + filePath: 'eng\setup-pr-validation.ps1' + arguments: "-sourceBranchName $(Build.SourceBranch) -prNumber ${{ parameters.prNumber }} -commitSHA ${{ parameters.sha }} -enforceLatestCommit ${{ iif(parameters.EnforceLatestCommit, '1', '0') }}" + condition: succeeded() + - template: /eng/pipelines/apex-integration/integration-job.yml@${{ parameters.fsharpRepositoryAlias }} + parameters: + configuration: $(_configuration) + testCaseFilter: ${{ parameters.testCaseFilter }} diff --git a/eng/setup-pr-validation.ps1 b/eng/setup-pr-validation.ps1 new file mode 100644 index 00000000000..00cae5630af --- /dev/null +++ b/eng/setup-pr-validation.ps1 @@ -0,0 +1,70 @@ +# Sets up the working tree on a DartLab test machine to validate a specific GitHub PR: fetches the +# PR merge commit, optionally enforces that the PR head still matches the reviewed SHA, and checks +# it out. Referenced by eng/pipelines/apex-integration/stage.yml (only when a PR number is supplied). +# Adapted from dotnet/roslyn's eng/setup-pr-validation.ps1. +[CmdletBinding(PositionalBinding=$false)] +param ( + [string]$sourceBranchName, + [string]$prNumber, + [string]$commitSHA, + [boolean]$enforceLatestCommit) + +try { + # name and email are only used for the merge commit; the values do not matter. + git config user.name "FSharpValidation" + git config user.email "validation@fsharp.net" + + if ($commitSHA.Length -lt 7) { + Write-Host "##vso[task.LogIssue type=error;]The PR Commit SHA must be at least 7 characters long." + exit 1 + } + + git remote add gh https://github.com/dotnet/fsharp.git + + Write-Host "Getting the hash of refs/pull/$prNumber/head..." + $remoteRef = git ls-remote gh refs/pull/$prNumber/head + Write-Host ($remoteRef | Out-String) + + $prHeadSHA = $remoteRef.Split()[0] + + if ($enforceLatestCommit) { + Write-Host "Validating the PR head matches the specified commit SHA ($commitSHA)..." + if (!$prHeadSHA.StartsWith($commitSHA)) { + Write-Host "##vso[task.LogIssue type=error;]The PR's Head SHA ($prHeadSHA) does not begin with the specified commit SHA ($commitSHA). Unreviewed changes may have been pushed to the PR." + exit 1 + } + } + + Write-Host "Setting up the build for PR validation by fetching refs/pull/$prNumber/merge..." + git fetch gh refs/pull/$prNumber/merge + if (!$?) { + Write-Host "##vso[task.LogIssue type=error;]Fetching ref refs/pull/$prNumber/merge failed." + exit 1 + } + + git checkout FETCH_HEAD + if (!$?) { + Write-Host "##vso[task.LogIssue type=error;]Checking out FETCH_HEAD for refs/pull/$prNumber/merge failed." + exit 1 + } + + if (!$enforceLatestCommit) { + if ($prHeadSHA.StartsWith($commitSHA)) { + Write-Host "PR head SHA ($prHeadSHA) already matches the specified commit SHA ($commitSHA), skipping checkout." + } + else { + Write-Host "Checking out the specified commit SHA ($commitSHA)..." + git checkout $commitSHA + if (!$?) { + Write-Host "##vso[task.LogIssue type=error;]Checking out commit SHA $commitSHA failed." + exit 1 + } + } + } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs new file mode 100644 index 00000000000..453cac96bb6 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Apex counterparts of the xUnit/IdeFact tests in +// vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs. The project is created from +// the .NET SDK class-library template (via `dotnet new`, matching the source tests; Apex's +// ProjectTemplate.ClassLibrary is the legacy .NET Framework template). The template auto-opens the +// default source file, so the source is set through the open document and saved to disk (an +// out-of-band disk write would be superseded by the still-open template buffer, and the compiler would +// build the valid template instead of the intended code). Rather than parsing the "Build: 1 succeeded, +// 0 failed ..." output-pane summary string, these assert on the idiomatic Apex build result: +// BuildManager.Succeeded and BuildManager.Verify.HasFailedWithErrors for the error case. + +using System.IO; +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class BuildProjectTests : FSharpLanguageServiceApexTest + { + protected override bool AutomaticallyDismissMessageBoxes => false; + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify a well-formed F# project builds successfully.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void SuccessfulBuild() + { + var code = @"module Test + +let answer = 42"; + + this.SetLibraryContent(code); + + Assert.IsTrue(this.BuildSolutionSucceeded(), "Build was expected to succeed."); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify an F# project with an incomplete binding fails to build with FS0010.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void FailedBuild() + { + var code = @"module Test + +let answer ="; + + this.SetLibraryContent(code); + + this.BuildSolution(); + + var buildManager = this.Library.VisualStudio.ObjectModel.Solution.BuildManager; + Assert.IsFalse(buildManager.Succeeded, "Build was expected to fail."); + + // The Apex error-list verifier matches the error DESCRIPTION (the message text); the code + // 'FS0010' lives in a separate Code column, so match on the FS0010 message instead. + Assert.IsTrue( + buildManager.Verify.HasFailedWithErrors( + new[] { "Incomplete structured construct" }, alsoVerifyInOutputWindow: false), + "Build was expected to fail with the FS0010 'Incomplete structured construct' error."); + } + + /// + /// Creates an F# class library and replaces its default Library.fs with + /// through the open document (saved to disk), verifying the file on disk matches before building. + /// + private void SetLibraryContent(string code) + { + var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("classlib", "Library"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + var document = this.Library.OpenProjectFile(project, "Library.fs"); + document.ReplaceAllAndSave(code); + + AssertSourceEquals(code, File.ReadAllText(document.FilePath)); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs new file mode 100644 index 00000000000..515cd14883f --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Apex counterparts of the xUnit/IdeFact tests in +// vsintegration/tests/FSharp.Editor.IntegrationTests/CodeActionTests.cs. Each case sets up the same +// triggering F# source, places the caret where that test does, and verifies the same code/error fix +// is offered on the light bulb (asserting on the action's display text). The Apex light-bulb model +// exposes a flat action list, so the CodeFix/ErrorFix category asserted by the source tests is not +// mirrored here. + +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class CodeActionTests : FSharpLanguageServiceApexTest + { + protected override bool AutomaticallyDismissMessageBoxes => false; + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the 'Remove unused open declarations' code fix is offered on an unused open.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void UnusedOpenDeclarations() + { + var code = @"module Library + +open System + +let x = 42"; + + this.AssertCodeActionOffered(code, "open System", "Remove unused open declarations"); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the 'Add missing 'fun' keyword' error fix is offered on a lambda missing 'fun'.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void AddMissingFunKeyword() + { + var code = @"module Library + +let original = [] +let transformed = original |> List.map (x -> x)"; + + this.AssertCodeActionOffered(code, "->", "Add missing 'fun' keyword"); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the 'Add 'new' keyword' code fix is offered when constructing a disposable.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void AddNewKeywordToDisposables() + { + var code = @"module Library + +let sr = System.IO.StreamReader("""")"; + + this.AssertCodeActionOffered(code, "let sr", "Add 'new' keyword"); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs new file mode 100644 index 00000000000..1aa66153b95 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Ported verbatim from the TypeScript-VS repository +// (VS/LanguageService/Tests/Integration/CodeFixes/SuggestionCodeFix.cs) and then adjusted so that +// the setup is tailored for the F# extension. A single test case is kept, exercising an F# code fix +// (Wrap expression in parentheses, FS0597). The triggering code is taken from the F# code-fix unit +// tests: vsintegration/tests/FSharp.Editor.Tests/CodeFixes/WrapExpressionInParenthesesTests.fs. + +using System; +using System.Linq; +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class SuggestionCodeFix : FSharpLanguageServiceApexTest + { + #region Private members + protected override bool AutomaticallyDismissMessageBoxes => false; + private const string code = @"module Library + +let rng = System.Random() + +printfn ""Hello %d"" rng.Next(5)"; + private const string expectedCode = @"module Library + +let rng = System.Random() + +printfn ""Hello %d"" (rng.Next(5))"; + #endregion + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify code-fix offers suggestion to wrap an expression in parentheses in an F# file.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void SuggestionCodeFixInFSharpFile() + { + var document = this.OpenFSharpDocument(code, "rng.Next"); + + this.InvokeCodeFix(document, "rng.Next"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + // Poll until the fix rewrites the buffer, then assert it actually applied. Without the + // assertion a fix that never fires would leave the original source and the test would still + // pass. Compare on normalized source so incidental line-ending differences do not race the poll. + this.Library.Synchronization.TryWaitForCondition( + () => NormalizeSource(document.Contents) == NormalizeSource(expectedCode), + TimeSpan.FromSeconds(15)); + + AssertSourceEquals(expectedCode, document.Contents); + } + + #region Helps + public void InvokeCodeFix(TextDocumentView document, string expression) + { + var lightBulb = this.MoveToExpressionAndExpandLightBulb(document, expression); + var globalScopeAction = lightBulb.Actions.Single(action => + action.Text.IndexOf("Wrap", StringComparison.OrdinalIgnoreCase) >= 0 + ); + + globalScopeAction.Invoke(); + } + #endregion + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs new file mode 100644 index 00000000000..4ce393246a3 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Apex counterparts of the xUnit/IdeFact tests in +// vsintegration/tests/FSharp.Editor.IntegrationTests/CreateProjectTests.cs. Each case creates an F# +// project from a .NET SDK template (via `dotnet new`, matching the SDK templates the source tests +// target — Apex's ProjectTemplate enum maps to the legacy .NET Framework F# templates with different +// filenames and content) and asserts the auto-generated default source. + +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class CreateProjectTests : FSharpLanguageServiceApexTest + { + protected override bool AutomaticallyDismissMessageBoxes => false; + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the default source of a new F# class library project.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void ClassLibrary() + { + var expectedCode = @"namespace Library + +module Say = + let hello name = + printfn ""Hello %s"" name"; + + var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("classlib", "Library"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + var document = this.Library.OpenProjectFile(project, "Library.fs"); + + AssertSourceEquals(expectedCode, document.Contents); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the default source of a new F# console application project.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void ConsoleApp() + { + var expectedCode = @"// For more information see https://aka.ms/fsharp-console-apps +printfn ""Hello from F#"""; + + var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("console", "ConsoleApp"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + var document = this.Library.OpenProjectFile(project, "Program.fs"); + + AssertSourceEquals(expectedCode, document.Contents); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the default source of a new F# xUnit test project.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void XUnitTestProject() + { + var expectedCode = @"module Tests + +open System +open Xunit + +[] +let ``My test`` () = + Assert.True(true)"; + + var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("xunit", "Tests"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + var document = this.Library.OpenProjectFile(project, "Tests.fs"); + + AssertSourceEquals(expectedCode, document.Contents); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj new file mode 100644 index 00000000000..924a74297f4 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj @@ -0,0 +1,41 @@ + + + + + net472 + preview + disable + Library + false + + true + $(NoWarn);CS1591;VSTHRD200 + + + + + + + + + + + + + + + + diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs new file mode 100644 index 00000000000..5eff72aa107 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Apex counterparts of the xUnit/IdeFact tests in +// vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs. Go To Definition is +// invoked idiomatically via the editor caret (IVisualStudioCaretTestExtension.GoToDefinition), and the +// result is read from the active document (current line + window caption) rather than the async +// SolutionExplorer/Editor/Shell helpers used by the xUnit harness. + +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.Test.Apex.VisualStudio.Solution; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class GoToDefinitionTests : FSharpLanguageServiceApexTest + { + protected override bool AutomaticallyDismissMessageBoxes => false; + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify Go To Definition navigates from a use of a binding to its definition.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void GoesToDefinition() + { + var code = @"module Test + +let add x y = x + y + +let increment = add 1"; + + var document = this.OpenFSharpDocument(code, "add 1"); + + // Resolve references so the file type-checks and Go To Definition can bind the symbol. + this.BuildSolution(); + + this.GoToDefinitionAndWait(document, "add 1", line => line.Contains("let add x y = x + y")); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify Go To Definition stays within the signature file and the implementation file respectively.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void FsiAndFsFilesGoToCorrespondentDefinitions() + { + var fsi = @"module Module + +type SomeType = +| Number of int +| Letter of char + +val id: t: SomeType -> SomeType"; + + var fs = @"module Module + +type SomeType = + | Number of int + | Letter of char + +let id (t: SomeType) = t"; + + var project = this.Library.ProjectCreation.CreateFSharpProject(ProjectTemplate.ClassLibrary, "Library"); + + // The signature file must precede the implementation file in F# compile order. When a file is + // added, VS inserts it alphabetically, so adding "Module.fsi" then "Module.fs" would place the + // implementation first ("Module.fs" sorts before "Module.fsi") and the signature would not + // apply. Add the signature under a name that sorts first, then rename it — the same approach as + // the source IdeFact test. + var fsiItem = this.Library.ProjectCreation.AddProjectItemFromContent(project, "AModule.fsi", fsi); + this.Library.ProjectCreation.AddProjectItemFromContent(project, "Module.fs", fs); + fsiItem.Rename("Module.fsi"); + this.Library.Synchronization.WaitForSolutionCrawler(); + this.BuildSolution(); + + // From the signature file, Go To Definition on the type usage stays in Module.fsi. + var fsiDocument = this.Library.OpenProjectFile(project, "Module.fsi"); + this.GoToDefinitionAndWait(fsiDocument, "SomeType ->", line => line.Trim() == "type SomeType =", "Module.fsi"); + + // From the implementation file, Go To Definition on the type usage stays in Module.fs. + var fsDocument = this.Library.OpenProjectFile(project, "Module.fs"); + this.GoToDefinitionAndWait(fsDocument, "SomeType)", line => line.Trim() == "type SomeType =", "Module.fs"); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs new file mode 100644 index 00000000000..4583a7b321f --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs @@ -0,0 +1,317 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Web.Script.Serialization; +using Microsoft.Test.Apex.Editor; +using Microsoft.Test.Apex.VisualStudio; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Base test class for F# language service tests built using the Apex framework. + /// Compact, self-contained equivalent of the TypeScript-VS LanguageServiceApexTest, tailored for + /// the F# extension and without the JavaScript/TypeScript BrowserTools layer. + /// + [TestClass] + public abstract class FSharpLanguageServiceApexTest : VisualStudioHostTest + { + protected const int ShortTimeoutMS = 1000 * 60 * 12; + protected const int LongTimeoutMS = 1000 * 60 * 20; + + // Apex launches the Visual Studio instance pointed to by this environment variable, if set. + private const string InstallationUnderTestPathVariable = "VisualStudio.InstallationUnderTest.Path"; + + private FSharpLanguageServiceLibrary libraryCache; + + /// + /// Gets the F# language service test library. + /// + public FSharpLanguageServiceLibrary Library + => this.libraryCache ??= new FSharpLanguageServiceLibrary(this.VisualStudio, this.Operations); + + /// + /// Whether message boxes shown during the test run should be dismissed automatically. + /// + protected virtual bool AutomaticallyDismissMessageBoxes => true; + + /// + /// The registry root suffix (experimental hive) of the Visual Studio instance to launch. + /// Defaults to "RoslynDev", where the locally-built F# extension is deployed. + /// + protected virtual string RootSuffix => "RoslynDev"; + + /// + /// The product milestone of the installed Visual Studio to launch, as reported by vswhere's + /// catalog.productMilestone. Defaults to "Canary" (the IntCanary channel), which selects the + /// Canary install rather than Insiders when both are present locally. On CI the installed VS is + /// a different channel (e.g. "Preview"), so this can be overridden with the + /// FSHARP_APEX_VS_MILESTONE environment variable; setting VisualStudio.InstallationUnderTest.Path + /// directly takes precedence over milestone resolution entirely. + /// + protected virtual string TargetProductMilestone + => Environment.GetEnvironmentVariable("FSHARP_APEX_VS_MILESTONE") is string milestone + && !string.IsNullOrEmpty(milestone) + ? milestone + : "Canary"; + + protected override VisualStudioHostConfiguration GetVisualStudioHostConfiguration() + { + this.EnsureTargetInstallationSelected(); + + var config = base.GetVisualStudioHostConfiguration(); + config.AutomaticallyDismissMessageBoxes = this.AutomaticallyDismissMessageBoxes; + config.RootSuffix = this.RootSuffix; + + // On a fresh experimental hive (e.g. CI), devenv shows a full-screen "Sign in to Visual Studio" + // first-launch dialog that blocks the UI thread, so it never registers its DTE automation + // object and Apex times out waiting for it. The devenv '/NoSigninPrompt' switch (registered by + // Microsoft.VisualStudio.Shell.Connected under AppCommandLine) suppresses that dialog. + config.CommandLineArguments = + string.IsNullOrEmpty(config.CommandLineArguments) + ? "/NoSigninPrompt" + : config.CommandLineArguments + " /NoSigninPrompt"; + + return config; + } + + /// + /// Points Apex at the Visual Studio install matching + /// (e.g. Canary). An explicit VisualStudio.InstallationUnderTest.Path override is respected. + /// + private void EnsureTargetInstallationSelected() + { + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable(InstallationUnderTestPathVariable))) + { + return; + } + + string devenvPath = ResolveDevenvByMilestone(this.TargetProductMilestone); + if (!string.IsNullOrEmpty(devenvPath)) + { + Environment.SetEnvironmentVariable(InstallationUnderTestPathVariable, devenvPath); + } + } + + private static string ResolveDevenvByMilestone(string milestone) + { + string programFilesX86 = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86); + string vswhere = Path.Combine(programFilesX86, "Microsoft Visual Studio", "Installer", "vswhere.exe"); + if (!File.Exists(vswhere)) + { + return null; + } + + var startInfo = new ProcessStartInfo(vswhere, "-all -prerelease -format json -utf8") + { + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + string json; + using (var process = Process.Start(startInfo)) + { + json = process.StandardOutput.ReadToEnd(); + process.WaitForExit(); + } + + if (string.IsNullOrWhiteSpace(json) + || !(new JavaScriptSerializer { MaxJsonLength = int.MaxValue }.DeserializeObject(json) is object[] installations)) + { + return null; + } + + foreach (var installation in installations.OfType>()) + { + if (installation.TryGetValue("catalog", out var catalogObject) + && catalogObject is Dictionary catalog + && catalog.TryGetValue("productMilestone", out var installedMilestone) + && string.Equals(installedMilestone as string, milestone, StringComparison.OrdinalIgnoreCase) + && installation.TryGetValue("productPath", out var productPath)) + { + return productPath as string; + } + } + + return null; + } + + /// + /// Moves the caret onto , triggers the code-fix request and + /// expands the resulting light bulb. + /// + protected ILightBulbTestExtension MoveToExpressionAndExpandLightBulb(TextDocumentView document, string expression) + { + // Nudging the caret into the diagnostic range triggers a request for code fixes. + document.MoveToExpression(expression); + document.MoveRight(); + document.MoveLeft(); + + this.Library.Synchronization.WaitForLightBulb(); + + Assert.IsTrue( + this.Library.Editor.LightBulb.Verify.IsLightBulbPresent(TimeSpan.FromSeconds(60)), + $"Expected a light bulb to be present at expression '{expression}'."); + + var lightBulb = this.Library.Editor.LightBulb.GetActiveLightBulb(); + lightBulb.Expand(); + + // Wait for the suggested actions to populate. Poll the already-expanded session directly: + // re-querying GetActiveLightBulb() here returns null once the light bulb is expanded into + // its flyout (LightBulbBroker no longer reports it as the active session), and the flyout + // taking focus can also null out Library.Editor (ActiveDocumentWindowAsTextEditor) — either + // of which previously threw a NullReferenceException inside the poll. LightBulbTestExtension + // .Actions reads the captured session and returns an empty (never null) set while pending. + this.Library.Synchronization.TryWaitForCondition(() => lightBulb.Actions.Any()); + + return lightBulb; + } + + /// + /// Creates a fresh single-project F# solution whose Library.fs contains + /// and opens it in the editor. Waits for the project to finish loading before editing (so the + /// F# project system does not reload the empty file over the insert) and confirms + /// reached the buffer before returning, so downstream caret + /// searches are not racing document initialization. + /// + protected TextDocumentView OpenFSharpDocument(string code, string expectedContent) + { + var project = this.Library.ProjectCreation.CreateFSharpLibrary(); + var documentItem = this.Library.ProjectCreation.AddProjectItemFromEmptyFile(project, "Library.fs"); + var document = this.Library.OpenDocument(documentItem); + + this.Library.Synchronization.WaitForSolutionCrawler(); + + document.InsertText(code); + + Assert.IsTrue( + this.Library.Synchronization.TryWaitForCondition( + () => document.Contents.Contains(expectedContent), TimeSpan.FromSeconds(15)), + $"Inserted source did not appear in the document buffer (looking for '{expectedContent}')."); + + return document; + } + + /// + /// Builds the current solution and waits for it to finish. Building performs a NuGet restore and + /// resolves references — which the F# language service needs before it can type-check and surface + /// SEMANTIC diagnostics (e.g. the unused-open analyzer, or FS0760 for disposables). This is the + /// Apex counterpart of the source IdeFact tests' RestoreNuGetPackagesAsync + WaitForProjectSystem. + /// Parse-based fixes (FS0597, FS0010) don't need it. Build success is not asserted: some fixture + /// sources intentionally contain errors, and a failed compile still restores references. + /// + protected void BuildSolution() + { + this.Library.VisualStudio.ObjectModel.Solution.BuildManager.Build(waitForBuildToFinish: true); + this.Library.Synchronization.WaitForSolutionCrawler(); + } + + /// + /// Builds the current solution and returns whether it succeeded. + /// + protected bool BuildSolutionSucceeded() + { + this.BuildSolution(); + return this.Library.VisualStudio.ObjectModel.Solution.BuildManager.Succeeded; + } + + /// + /// Places the caret on , focuses the editor and invokes Go To + /// Definition, then waits for navigation to settle. The result is read from the active document + /// afterwards (which may be the same or a different file). + /// + protected void GoToDefinition(TextDocumentView document, string expression) + { + document.MoveToExpression(expression); + document.Focus(); + document.GoToDefinition(); + this.Library.Synchronization.WaitForSolutionCrawler(); + } + + /// + /// Invokes Go To Definition on and polls until the active document + /// settles on a line matching and, when given, a window caption + /// equal to . Go To Definition navigates asynchronously (it may + /// open/activate another document and only then move the caret), so — exactly like the light-bulb + /// helper waits for its actions — we poll for the end state instead of reading it immediately. + /// Reading it immediately is the fire-and-assert race that makes the ported navigation tests flaky. + /// + protected void GoToDefinitionAndWait( + TextDocumentView document, + string expression, + Func lineMatches, + string expectedCaption = null) + { + this.GoToDefinition(document, expression); + + string lastLine = null; + string lastCaption = null; + bool landed = this.Library.Synchronization.TryWaitForCondition( + () => + { + lastLine = this.Library.ActiveDocumentCurrentLineText; + lastCaption = this.Library.ActiveDocumentCaption; + return lastLine != null + && lineMatches(lastLine) + && (expectedCaption == null || string.Equals(lastCaption, expectedCaption, StringComparison.Ordinal)); + }, + TimeSpan.FromSeconds(30)); + + Assert.IsTrue( + landed, + $"Go To Definition on '{expression}' did not settle on the expected location. " + + (expectedCaption != null ? $"Expected caption '{expectedCaption}', actual '{lastCaption}'. " : string.Empty) + + $"Actual current line: '{lastLine}'."); + } + + /// + /// Asserts two pieces of F# source are equal, ignoring line-ending style and any trailing + /// newline. Template output and editor buffers differ in those incidental ways across SDKs, so + /// normalizing avoids brittle failures while still comparing the meaningful content exactly. + /// + protected static void AssertSourceEquals(string expected, string actual) + { + Assert.AreEqual(NormalizeSource(expected), NormalizeSource(actual)); + } + + /// + /// Normalizes F# source for comparison by unifying line-ending style and trimming any trailing + /// newline, so comparisons ignore those incidental differences. + /// + protected static string NormalizeSource(string source) + => (source ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").TrimEnd('\n'); + + /// + /// Verifies that placing the caret on in the given F# source + /// offers exactly one light-bulb action whose text contains . + /// Mirrors the display-text assertions in FSharp.Editor.IntegrationTests CodeActionTests (the Apex + /// light-bulb model exposes a flat action list, so the code-fix/error-fix category is not checked). + /// + protected void AssertCodeActionOffered(string code, string caretExpression, string expectedActionText) + { + var document = this.OpenFSharpDocument(code, caretExpression); + + // Restore/resolve references so the F# language service can type-check and offer semantic + // fixes (the unused-open analyzer, FS0760, ...); without this their light bulb never appears. + this.BuildSolution(); + + var lightBulb = this.MoveToExpressionAndExpandLightBulb(document, caretExpression); + + var matchCount = lightBulb.Actions.Count(action => + action.Text.IndexOf(expectedActionText, StringComparison.OrdinalIgnoreCase) >= 0); + + Assert.AreEqual( + 1, + matchCount, + $"Expected exactly one code action containing '{expectedActionText}'. Offered actions: " + + string.Join(", ", lightBulb.Actions.Select(action => $"'{action.Text}'"))); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs new file mode 100644 index 00000000000..a3b98ab410f --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs @@ -0,0 +1,82 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using System.Linq; +using Microsoft.Test.Apex; +using Microsoft.Test.Apex.Services; +using Microsoft.Test.Apex.VisualStudio; +using Microsoft.Test.Apex.VisualStudio.Editor; +using Microsoft.Test.Apex.VisualStudio.Shell; +using Microsoft.Test.Apex.VisualStudio.Solution; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Compact test library for the F# Apex integration tests. Provides the small facade over the + /// Apex that the code-action tests use (project creation, + /// synchronization, the active editor and document opening), tailored for the F# extension. + /// + public sealed class FSharpLanguageServiceLibrary + { + public FSharpLanguageServiceLibrary(VisualStudioHost visualStudio, IOperations operations) + { + this.VisualStudio = visualStudio; + this.Synchronization = new SynchronizationHelper(visualStudio, operations.Get()); + this.ProjectCreation = new ProjectCreationHelper(visualStudio); + } + + /// The Apex Visual Studio host. + public VisualStudioHost VisualStudio { get; } + + /// Synchronization helpers used to wait for background work. + public SynchronizationHelper Synchronization { get; } + + /// F# project and project-item creation helpers. + public ProjectCreationHelper ProjectCreation { get; } + + /// The editor of the active document window, or null if none is active. + public IVisualStudioTextEditorTestExtension Editor + => this.VisualStudio.ObjectModel.WindowManager.ActiveDocumentWindowAsTextEditor?.Editor; + + /// + /// Opens the given project item in the text editor and returns a view over it. + /// + public TextDocumentView OpenDocument(ProjectItemTestExtension documentItem) + { + var window = documentItem.Open(); + return new TextDocumentView(window); + } + + /// + /// Finds under , opens it in the text + /// editor and returns a view over it. The project's item tree is populated lazily, so the lookup + /// is polled (TryFindChild is single-shot); on timeout it throws listing the project's actual + /// top-level item names, which makes a template-filename surprise obvious. + /// + public TextDocumentView OpenProjectFile(ProjectTestExtension project, string fileName) + { + ProjectItemTestExtension item = null; + this.Synchronization.TryWaitForCondition( + () => (item = project.TryFindChild(fileName, true)) != null); + + if (item == null) + { + var actualNames = string.Join(", ", project.ProjectItems.Select(child => $"'{child.Name}'")); + throw new InvalidOperationException( + $"Project item '{fileName}' was not found in project '{project.Name}'. Items present: {actualNames}."); + } + + return this.OpenDocument(item); + } + + /// The short caption (file name) of the active document window, or null if none is active. + public string ActiveDocumentCaption + => this.VisualStudio.ObjectModel.WindowManager.ActiveDocumentWindow?.Caption; + + /// The text of the caret's current line in the active document editor, or null if none is active. + public string ActiveDocumentCurrentLineText + => this.Editor?.Caret.GetCurrentLineText(); + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs new file mode 100644 index 00000000000..11a837bd233 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs @@ -0,0 +1,117 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Test.Apex.VisualStudio; +using Microsoft.Test.Apex.VisualStudio.Solution; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Creates F# projects and project items for the Apex integration tests. + /// Tailored for the F# extension: the TypeScript-VS ProjectCreationHelper created ASP.NET Core + /// web apps for JavaScript/TypeScript; this one creates plain F# class libraries. + /// + public sealed class ProjectCreationHelper + { + private readonly VisualStudioHost visualStudio; + private int projectCounter = 0; + + public ProjectCreationHelper(VisualStudioHost visualStudio) + { + this.visualStudio = visualStudio; + } + + /// + /// Creates a new F# class library in a fresh solution. + /// + public ProjectTestExtension CreateFSharpLibrary() + { + string projectName = $"FSharpLibrary_{++this.projectCounter}"; + return this.CreateFSharpProject(ProjectTemplate.ClassLibrary, projectName); + } + + /// + /// Creates a new F# project of the given built-in template with the given name. + /// + public ProjectTestExtension CreateFSharpProject(ProjectTemplate template, string projectName) + => this.visualStudio.ObjectModel.Solution.CreateProject( + ProjectLanguage.FSharp, + template, + projectName); + + /// + /// Creates a fresh single-project solution from an F# SDK template (e.g. "classlib", "console", + /// "xunit") scaffolded on disk with dotnet new. Apex's enum + /// maps to the legacy .NET Framework F# templates (which produce Library1.fs/Script.fsx), whereas + /// the source FSharp.Editor.IntegrationTests target the .NET SDK templates; using dotnet new + /// makes the project (filenames and default source) match those. An empty solution is created + /// first because AddProject throws when no solution is open. + /// + public ProjectTestExtension CreateFSharpProjectFromSdkTemplate(string sdkTemplateName, string projectName) + { + string projectDirectory = Path.Combine( + Path.GetTempPath(), "FSharpApexTemplates", Guid.NewGuid().ToString("N"), projectName); + Directory.CreateDirectory(projectDirectory); + + RunDotNet($"new {sdkTemplateName} --language \"F#\" --name \"{projectName}\" --output \"{projectDirectory}\""); + + string projectPath = Path.Combine(projectDirectory, projectName + ".fsproj"); + this.visualStudio.ObjectModel.Solution.CreateEmptySolution(); + return this.visualStudio.ObjectModel.Solution.AddProject(projectPath); + } + + /// + /// Creates an empty file on disk and adds it to the given project or project item. + /// + public ProjectItemTestExtension AddProjectItemFromEmptyFile(IProjectItemsContainer container, string fileName) + { + string containerDirectory = container.IsProject ? container.ProjectDirectory : container.FullPath; + string filePath = Path.Combine(containerDirectory, fileName); + File.WriteAllText(filePath, string.Empty); + return container.AddProjectItemFromFile(filePath); + } + + /// + /// Creates a file with the given content on disk and adds it to the given project or item. + /// + public ProjectItemTestExtension AddProjectItemFromContent(IProjectItemsContainer container, string fileName, string content) + { + string containerDirectory = container.IsProject ? container.ProjectDirectory : container.FullPath; + string filePath = Path.Combine(containerDirectory, fileName); + File.WriteAllText(filePath, content); + return container.AddProjectItemFromFile(filePath); + } + + private static void RunDotNet(string arguments) + { + var startInfo = new ProcessStartInfo("dotnet", arguments) + { + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + + using var process = Process.Start(startInfo) + ?? throw new InvalidOperationException($"Failed to start 'dotnet {arguments}'."); + + // Drain both redirected streams concurrently. Reading one to completion before the other can + // deadlock: if the child fills the stdout pipe buffer while we block on stderr's EOF (which + // only arrives at process exit), neither side can make progress. + Task standardErrorTask = process.StandardError.ReadToEndAsync(); + process.StandardOutput.ReadToEnd(); + string standardError = standardErrorTask.GetAwaiter().GetResult(); + process.WaitForExit(); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException($"'dotnet {arguments}' failed with exit code {process.ExitCode}: {standardError}"); + } + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs new file mode 100644 index 00000000000..a5f834042ec --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Drop-in replacement for that re-runs a test up to + /// maxAttempts times, passing as soon as one attempt passes. These Apex tests drive a real + /// Visual Studio through UI automation and are occasionally flaky (focus, background analysis and + /// build/restore timing); a bounded retry keeps them reliable without masking a genuine failure — + /// every attempt has to fail before the test is reported as failed. + /// + /// Each attempt goes through the full per-test pipeline (a fresh test-class instance plus + /// TestInitialize/TestCleanup, i.e. a fresh VS session for the Apex host), so a retry starts from a + /// clean state rather than reusing the dirtied state of the previous attempt. + /// + /// MSTest's own [Retry] attribute is only available from 3.8; this repo pins 3.6.4, so the + /// classic "derive from TestMethodAttribute and override Execute" idiom is used instead. + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public sealed class RetryTestMethodAttribute : TestMethodAttribute + { + private readonly int maxAttempts; + + public RetryTestMethodAttribute(int maxAttempts = 2) + { + if (maxAttempts < 1) + { + throw new ArgumentOutOfRangeException(nameof(maxAttempts), maxAttempts, "At least one attempt is required."); + } + + this.maxAttempts = maxAttempts; + } + + public override TestResult[] Execute(ITestMethod testMethod) + { + TestResult[] results = null; + + for (var attempt = 1; attempt <= this.maxAttempts; attempt++) + { + results = base.Execute(testMethod); + + if (results.All(result => result.Outcome == UnitTestOutcome.Passed)) + { + return results; + } + + if (attempt < this.maxAttempts) + { + Trace.WriteLine( + $"[RetryTestMethod] '{testMethod.TestMethodName}' failed on attempt {attempt} of {this.maxAttempts}; retrying."); + } + } + + return results; + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs new file mode 100644 index 00000000000..80969ffd74c --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs @@ -0,0 +1,58 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using Microsoft.Test.Apex.Services; +using Microsoft.Test.Apex.VisualStudio; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Synchronization helper for the F# Apex integration tests. + /// This is a compact, self-contained equivalent of the TypeScript-VS SynchronizationHelper: + /// where that type delegated to the Roslyn "WaitForFeatures" async-operation waiter, this one + /// polls via the Apex so the harness has no dependency on + /// the JavaScript/TypeScript BrowserTools layer. + /// + public sealed class SynchronizationHelper + { + private static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(30); + private static readonly TimeSpan DefaultInterval = TimeSpan.FromSeconds(1); + + private readonly VisualStudioHost visualStudio; + private readonly ISynchronizationService synchronizationService; + + public SynchronizationHelper(VisualStudioHost visualStudio, ISynchronizationService synchronizationService) + { + this.visualStudio = visualStudio; + this.synchronizationService = synchronizationService; + } + + /// + /// Waits for the solution to be fully loaded and for background analysis to settle. + /// + public void WaitForSolutionCrawler() + { + this.visualStudio.ObjectModel.Solution.WaitForFullyLoaded(); + this.Settle(); + } + + /// + /// Waits for the light bulb / code-action pipeline to settle before the light bulb is queried. + /// + public void WaitForLightBulb() => this.Settle(); + + /// + /// Polls until it returns true or the timeout elapses. + /// + /// True if the condition became true within the timeout; otherwise false. + public bool TryWaitForCondition(Func condition, TimeSpan? timeout = null) + { + return this.synchronizationService.TryWaitFor(timeout ?? DefaultTimeout, () => condition()); + } + + private void Settle() + => this.synchronizationService.TryWaitFor(DefaultInterval, () => false); + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs new file mode 100644 index 00000000000..47e756ea87c --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs @@ -0,0 +1,132 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using System.Text.RegularExpressions; +using System.Threading; +using Microsoft.Test.Apex.VisualStudio.Editor; +using Microsoft.Test.Apex.VisualStudio.Shell; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Represents the text view of an open VS document. Compact, self-contained equivalent of the + /// TypeScript-VS TextDocumentView / TextDocumentViewBase, wrapping the Apex text editor extension + /// with the small set of operations the code-action tests need. + /// + public sealed class TextDocumentView + { + private readonly TextEditorDocumentWindowTestExtension window; + + public TextDocumentView(TextEditorDocumentWindowTestExtension window) + { + this.window = window; + } + + private IVisualStudioTextEditorTestExtension editor => this.window.Editor; + + /// The full text content of the document. + public string Contents => this.editor.Contents; + + /// The path of the file backing this document. + public string FilePath => this.window.FilePath; + + /// + /// Inserts text verbatim at the current caret position via a direct text-buffer edit. + /// Unlike InsertTextWithReturn, this does not simulate per-character typing, so the F# + /// editor's brace/quote auto-completion, IntelliSense auto-commit and smart indentation do not + /// fire and rewrite the buffer. That keeps the inserted source exactly as given, which the + /// code-fix tests rely on when locating expressions afterwards. + /// + public void InsertText(string text) => this.editor.Edit.InsertTextInBuffer(text); + + /// + /// Moves the caret to the first (or -th) occurrence of an + /// expression. Apex's caret search treats the argument as a regular expression, but every caller + /// passes a literal source fragment (which may contain regex metacharacters such as ')' in + /// "SomeType)"), so the input is escaped to match it literally. + /// + public void MoveToExpression(string expression, int matchIndex = 1) + => this.editor.Caret.MoveToExpression(Regex.Escape(expression), matchIndex: matchIndex); + + /// Moves the caret one character to the left. + public void MoveLeft() => this.editor.Caret.MoveLeft(); + + /// Moves the caret one character to the right. + public void MoveRight() => this.editor.Caret.MoveRight(); + + /// Gives this document's editor keyboard focus (needed before caret-driven commands). + public void Focus() => this.editor.Focus(); + + /// The text of the line the caret is currently on (untrimmed). + public string CurrentLineText => this.editor.Caret.GetCurrentLineText(); + + /// Invokes Go To Definition from the current caret position. + public void GoToDefinition() => this.editor.Caret.GoToDefinition(); + + /// + /// Replaces the entire document with and saves it to disk. Build tests + /// need this: the template auto-opens the default file, so the source must be set through the + /// open document (buffer) and saved, otherwise an out-of-band disk write is superseded by the + /// still-open (valid template) buffer and the compiler never sees the intended code. + /// + /// The template content loads into the buffer asynchronously after the document opens, so this + /// first waits for that initial content to arrive and then applies the whole-buffer replacement + /// with verify-and-retry: without this a replacement done too early is overwritten by the + /// late-arriving template content, leaving the original source on disk. The selection is deleted + /// before a direct buffer insert (which does not itself replace a selection and, unlike simulated + /// typing, avoids brace/quote auto-completion). + /// + public void ReplaceAllAndSave(string text) + { + this.editor.Focus(); + + if (!WaitUntil(() => !string.IsNullOrEmpty(this.editor.Contents))) + { + throw new InvalidOperationException("Document did not load its initial content before editing."); + } + + var replaced = WaitUntil(() => + { + this.editor.Selection.SelectAll(); + this.editor.Edit.DeleteCharNext(1); + this.editor.Edit.InsertTextInBuffer(text); + return Normalize(this.editor.Contents) == Normalize(text); + }); + + if (!replaced) + { + throw new InvalidOperationException( + $"Failed to set document content. Expected:\n{text}\nActual:\n{this.editor.Contents}"); + } + + this.window.Save(); + + if (!this.window.TryWaitForNotDirty(TimeSpan.FromSeconds(15), TimeSpan.FromMilliseconds(100))) + { + throw new InvalidOperationException("Document remained dirty after saving."); + } + } + + private static bool WaitUntil(Func condition) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(15); + do + { + if (condition()) + { + return true; + } + + Thread.Sleep(200); + } + while (DateTime.UtcNow < deadline); + + return false; + } + + private static string Normalize(string source) + => (source ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").TrimEnd('\n'); + } +}