From 41e181fd22cfd661be63ed6afc78ecbcbc8c44b1 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 25 Mar 2026 19:28:40 -0700 Subject: [PATCH 01/38] Add Azure Container Apps (ACA) deployment and testing pipelines --- build/jobs/e2e-tests-aca.yml | 96 +++++++ build/jobs/provision-deploy-aca.yml | 286 +++++++++++++++++++ build/jobs/run-sql-tests-aca.yml | 140 +++++++++ build/pr-pipeline.yml | 13 +- build/tasks/e2e-set-variables-aca.yml | 98 +++++++ samples/templates/aca/r4-sql-aca.json | 391 ++++++++++++++++++++++++++ 6 files changed, 1020 insertions(+), 4 deletions(-) create mode 100644 build/jobs/e2e-tests-aca.yml create mode 100644 build/jobs/provision-deploy-aca.yml create mode 100644 build/jobs/run-sql-tests-aca.yml create mode 100644 build/tasks/e2e-set-variables-aca.yml create mode 100644 samples/templates/aca/r4-sql-aca.json diff --git a/build/jobs/e2e-tests-aca.yml b/build/jobs/e2e-tests-aca.yml new file mode 100644 index 0000000000..68f9655bda --- /dev/null +++ b/build/jobs/e2e-tests-aca.yml @@ -0,0 +1,96 @@ +parameters: +- name: version + type: string +- name: containerAppName + type: string +- name: appServiceType + type: string +- name: categoryFilter + type: string + default: 'Category!=ExportLongRunning' +- name: testRunTitleSuffix + type: string + default: '' + +steps: + - template: e2e-tests-extract.yml + parameters: + version: ${{parameters.version}} + + - template: ../tasks/e2e-set-variables-aca.yml + parameters: + containerAppName: ${{ parameters.containerAppName }} + + - task: PowerShell@2 + displayName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}' + inputs: + targetType: inline + pwsh: true + script: | + $ErrorActionPreference = 'Stop' + $testRoot = "$(Agent.TempDirectory)/E2ETests" + $testName = "Microsoft.Health.Fhir.${{ parameters.version }}.Tests.E2E" + $dll = Get-ChildItem -Path $testRoot -Recurse -Filter "$testName.dll" | Select-Object -First 1 + if (-not $dll) { + throw "Could not find $testName.dll under $testRoot" + } + + $filter = "FullyQualifiedName~${{ parameters.appServiceType }}&${{ parameters.categoryFilter }}" + $args = @('--filter', $filter, '--retry-failed-tests', '3', '--report-trx') + + Write-Host "Running dotnet $($dll.FullName)" + & dotnet $dll.FullName @args + + if ($LASTEXITCODE -ne 0) { + throw "E2E tests failed with exit code $LASTEXITCODE" + } + env: + 'TestEnvironmentUrl': $(TestEnvironmentUrl) + 'TestEnvironmentUrl_${{ parameters.version }}': $(TestEnvironmentUrl_${{ parameters.version }}) + 'TestEnvironmentUrl_Sql': $(TestEnvironmentUrl_Sql) + 'TestEnvironmentUrl_${{ parameters.version }}_Sql': $(TestEnvironmentUrl_${{ parameters.version }}_Sql) + 'Resource': $(Resource) + 'AllStorageAccounts': $(AllStorageAccounts) + 'TestExportStoreUri': $(TestExportStoreUri) + 'TestIntegrationStoreUri': $(TestIntegrationStoreUri) + 'tenant-admin-service-principal-name': $(tenant-admin-service-principal-name) + 'tenant-admin-service-principal-password': $(tenant-admin-service-principal-password) + 'tenant-admin-user-name': $(tenant-admin-user-name) + 'tenant-admin-user-password': $(tenant-admin-user-password) + 'tenant-id': $(tenant-id) + 'app_globalAdminServicePrincipal_id': $(app_globalAdminServicePrincipal_id) + 'app_globalAdminServicePrincipal_secret': $(app_globalAdminServicePrincipal_secret) + 'app_nativeClient_id': $(app_nativeClient_id) + 'app_nativeClient_secret': $(app_nativeClient_secret) + 'app_wrongAudienceClient_id': $(app_wrongAudienceClient_id) + 'app_wrongAudienceClient_secret': $(app_wrongAudienceClient_secret) + 'app_globalAdminUserApp_id': $(app_globalAdminUserApp_id) + 'app_globalAdminUserApp_secret': $(app_globalAdminUserApp_secret) + 'app_globalConverterUserApp_id': $(app_globalConverterUserApp_id) + 'app_globalConverterUserApp_secret': $(app_globalConverterUserApp_secret) + 'app_globalExporterUserApp_id': $(app_globalExporterUserApp_id) + 'app_globalExporterUserApp_secret': $(app_globalExporterUserApp_secret) + 'app_globalImporterUserApp_id': $(app_globalImporterUserApp_id) + 'app_globalImporterUserApp_secret': $(app_globalImporterUserApp_secret) + 'app_globalReaderUserApp_id': $(app_globalReaderUserApp_id) + 'app_globalReaderUserApp_secret': $(app_globalReaderUserApp_secret) + 'app_globalWriterUserApp_id': $(app_globalWriterUserApp_id) + 'app_globalWriterUserApp_secret': $(app_globalWriterUserApp_secret) + 'app_smartUserClient_id': $(app_smartUserClient_id) + 'app_smartUserClient_secret': $(app_smartUserClient_secret) + 'AZURESUBSCRIPTION_CLIENT_ID': $(AzurePipelinesCredential_ClientId) + 'AZURESUBSCRIPTION_TENANT_ID': $(AZURESUBSCRIPTION_TENANT_ID) + 'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': $(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID) + 'SYSTEM_ACCESSTOKEN': $(System.AccessToken) + platformOptions__resultDirectory: '$(Agent.TempDirectory)/testresults' + + - task: PublishTestResults@2 + displayName: 'Publish test results' + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '$(Agent.TempDirectory)/testresults/**/*.trx' + mergeTestResults: true + testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}' + # TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed + failTaskOnFailedTests: false + condition: succeededOrFailed() diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml new file mode 100644 index 0000000000..b8e46f0a38 --- /dev/null +++ b/build/jobs/provision-deploy-aca.yml @@ -0,0 +1,286 @@ +parameters: +- name: version + type: string +- name: sql + type: boolean + default: true +- name: webAppName + type: string +- name: appServicePlanName + type: string + default: '' +- name: appServicePlanResourceGroup + type: string + default: '' +- name: subscription + type: string +- name: resourceGroup + type: string +- name: testEnvironmentUrl + type: string +- name: imageTag + type: string +- name: schemaAutomaticUpdatesEnabled + type: string + default: 'auto' +- name: sqlServerName + type: string +- name: sqlComputeTier + type: string + default: 'Standard' +- name: reindexEnabled + type: boolean + default: true +- name: keyVaultName + type: string +- name: containerCpu + type: string + default: '1.0' +- name: containerMemory + type: string + default: '2Gi' + +jobs: +- job: provisionEnvironment + pool: + name: '$(SharedLinuxPool)' + vmImage: '$(LinuxVmImage)' + steps: + - task: AzureKeyVault@1 + displayName: 'Azure Key Vault: resolute-oss-tenant-info' + inputs: + azureSubscription: ${{ parameters.subscription }} + KeyVaultName: 'resolute-oss-tenant-info' + + - task: AzurePowerShell@5 + name: SetAcaOutputs + displayName: 'Deploy ACA for SQL and emit test URL' + retryCountOnTaskFailure: 1 + inputs: + azureSubscription: ${{ parameters.subscription }} + azurePowerShellVersion: latestVersion + pwsh: true + ScriptType: inlineScript + Inline: | + Add-Type -AssemblyName System.Web + + # Import retry helper for transient error handling + . $(System.DefaultWorkingDirectory)/release/scripts/PowerShell/Invoke-WithRetry.ps1 + + $deployPath = "$(System.DefaultWorkingDirectory)/test/Configuration" + + $testConfig = ConvertFrom-Json (Get-Content -Raw "$deployPath/testconfiguration.json") + $flattenedTestConfig = $(System.DefaultWorkingDirectory)/release/scripts/PowerShell/ConvertTo-FlattenedConfigurationHashtable.ps1 -InputObject $testConfig + + $additionalProperties = $flattenedTestConfig + $additionalProperties["SqlServer__DeleteAllDataOnStartup"] = "false" + $additionalProperties["SqlServer__AllowDatabaseCreation"] = "true" + $additionalProperties["TaskHosting__PollingFrequencyInSeconds"] = 1 + $additionalProperties["FhirServer__CoreFeatures__SearchParameterCacheRefreshIntervalSeconds"] = 2 + $additionalProperties["ASPNETCORE_FORWARDEDHEADERS_ENABLED"] = "true" + + $staticEnvNames = @( + "ASPNETCORE_FORWARDEDHEADERS_ENABLED", + "KeyVault__Endpoint", + "FhirServer__Security__Enabled", + "FhirServer__Security__EnableAadSmartOnFhirProxy", + "FhirServer__Security__Authentication__Authority", + "FhirServer__Security__Authentication__Audience", + "DataStore", + "SqlServer__Initialize", + "SqlServer__SchemaOptions__AutomaticUpdatesEnabled", + "SqlServer__DeleteAllDataOnStartup", + "SqlServer__AllowDatabaseCreation", + "TaskHosting__Enabled", + "TaskHosting__PollingFrequencyInSeconds", + "TaskHosting__MaxRunningTaskCount", + "FhirServer__CoreFeatures__SearchParameterCacheRefreshIntervalSeconds", + "FhirServer__Operations__Export__Enabled", + "FhirServer__Operations__Export__StorageAccountUri", + "FhirServer__Operations__Import__Enabled", + "FhirServer__Operations__IntegrationDataStore__StorageAccountUri", + "FhirServer__Operations__ConvertData__Enabled", + "FhirServer__Operations__ConvertData__ContainerRegistryServers__0", + "FhirServer__Operations__Reindex__Enabled" + ) + + $additionalEnvVars = @() + foreach ($entry in $additionalProperties.GetEnumerator()) { + if ($staticEnvNames -contains $entry.Key) { + continue + } + + if ($null -eq $entry.Value) { + continue + } + + $value = if ($entry.Value -is [bool]) { $entry.Value.ToString().ToLowerInvariant() } else { [string]$entry.Value } + if ([string]::IsNullOrWhiteSpace($value)) { + continue + } + + $additionalEnvVars += @{ + name = [string]$entry.Key + value = $value + } + } + + $webAppName = "${{ parameters.webAppName }}".ToLowerInvariant() + $acaEnvironmentName = ("{0}-acae" -f $webAppName).ToLowerInvariant() + + if ($acaEnvironmentName.Length -gt 32) { + $acaEnvironmentName = $acaEnvironmentName.Substring(0, 32).TrimEnd('-') + } + + if ($acaEnvironmentName.Length -lt 2) { + $acaEnvironmentName = "ca-$acaEnvironmentName" + } + + $templateParameters = @{ + containerAppName = $webAppName + containerAppsEnvironmentName = $acaEnvironmentName + keyVaultName = "${{ parameters.keyVaultName }}".ToLowerInvariant() + fhirVersion = "${{ parameters.version }}" + registryName = '$(azureContainerRegistry)' + imageTag = "${{ parameters.imageTag }}" + sqlServerName = "${{ parameters.sqlServerName }}".ToLowerInvariant() + sqlSchemaAutomaticUpdatesEnabled = "${{ parameters.schemaAutomaticUpdatesEnabled }}" + securityAuthenticationAuthority = "https://sts.windows.net/$(tenant-id-guid)" + securityAuthenticationAudience = "${{ parameters.testEnvironmentUrl }}" + enableExport = $true + enableImport = $true + enableConvertData = $true + enableReindex = if ("${{ parameters.reindexEnabled }}" -eq "true") { $true } else { $false } + minReplicas = 1 + maxReplicas = 3 + containerCpu = [double]::Parse("${{ parameters.containerCpu }}", [System.Globalization.CultureInfo]::InvariantCulture) + containerMemory = "${{ parameters.containerMemory }}" + acrPullUserAssignedManagedIdentityResourceId = '/subscriptions/a1766500-6fd5-4f5c-8515-607798271014/resourceGroups/HealthPlatformRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-uami' + additionalEnvVars = $additionalEnvVars + } + + $deploymentName = "$webAppName-aca" + $resourceGroupName = "${{ parameters.resourceGroup }}" + + Write-Host "Provisioning ACA resources" + Write-Host "ResourceGroupName: $resourceGroupName" + Write-Host "ContainerAppName: $webAppName" + Write-Host "ContainerAppsEnvironmentName: $acaEnvironmentName" + + $deploymentResult = Invoke-WithRetry -OperationName "ACA Deployment" -ScriptBlock { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/r4-sql-aca.json -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop + } + + $sqlServerName = "${{ parameters.sqlServerName }}".ToLowerInvariant() + $managedEnvironment = Get-AzResource -ResourceGroupName $resourceGroupName -ResourceType "Microsoft.App/managedEnvironments" -Name $acaEnvironmentName -ApiVersion "2023-05-01" -ExpandProperties + if ($null -eq $managedEnvironment) { + throw "Container Apps environment '$acaEnvironmentName' was not found in resource group '$resourceGroupName'." + } + + $managedEnvironmentData = $managedEnvironment | ConvertTo-Json -Depth 100 | ConvertFrom-Json + $outboundIps = @() + + if ($null -ne $managedEnvironmentData.Properties.outboundIpAddresses) { + if ($managedEnvironmentData.Properties.outboundIpAddresses -is [string]) { + $outboundIps += ($managedEnvironmentData.Properties.outboundIpAddresses -split ',') + } + else { + foreach ($ip in $managedEnvironmentData.Properties.outboundIpAddresses) { + $outboundIps += [string]$ip + } + } + } + + if (-not [string]::IsNullOrWhiteSpace([string]$managedEnvironmentData.Properties.staticIp)) { + $outboundIps += [string]$managedEnvironmentData.Properties.staticIp + } + + $outboundIps = $outboundIps | ForEach-Object { $_.Trim() } | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Sort-Object -Unique + + if ($outboundIps.Count -eq 0) { + Write-Warning "ACA outbound IP addresses were not found. Falling back to Allow Azure Services SQL firewall rule." + $allowAzureServicesRule = Get-AzSqlServerFirewallRule -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -FirewallRuleName "AllowAzureServices" -ErrorAction SilentlyContinue + if ($null -ne $allowAzureServicesRule) { + Write-Host "SQL firewall rule 'AllowAzureServices' already exists." + } + else { + Invoke-WithRetry -OperationName "SQL Firewall Rule AllowAzureServices" -ScriptBlock { + New-AzSqlServerFirewallRule -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -FirewallRuleName "AllowAzureServices" -StartIPAddress "0.0.0.0" -EndIPAddress "0.0.0.0" -ErrorAction Stop + } + } + } + else { + $ruleIndex = 0 + foreach ($ip in $outboundIps) { + $ruleIndex++ + $ruleName = "AcaOutbound-$ruleIndex" + $existingRule = Get-AzSqlServerFirewallRule -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -FirewallRuleName $ruleName -ErrorAction SilentlyContinue + if ($null -ne $existingRule -and $existingRule.StartIpAddress -eq $ip -and $existingRule.EndIpAddress -eq $ip) { + Write-Host "SQL firewall rule '$ruleName' already exists for $ip." + continue + } + + if ($null -ne $existingRule) { + Invoke-WithRetry -OperationName "SQL Firewall Rule Update $ruleName" -ScriptBlock { + Set-AzSqlServerFirewallRule -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -FirewallRuleName $ruleName -StartIpAddress $ip -EndIpAddress $ip -ErrorAction Stop + } + } + else { + Invoke-WithRetry -OperationName "SQL Firewall Rule Create $ruleName" -ScriptBlock { + New-AzSqlServerFirewallRule -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -FirewallRuleName $ruleName -StartIPAddress $ip -EndIPAddress $ip -ErrorAction Stop + } + } + } + } + + $containerAppUrl = $deploymentResult.Outputs.containerAppUrl.Value + if ([string]::IsNullOrWhiteSpace($containerAppUrl)) { + throw "ACA deployment output did not contain containerAppUrl" + } + + $containerAppUrl = $containerAppUrl.TrimEnd('/') + $healthCheckUrl = "$containerAppUrl/health/check" + + $timeoutMinutes = 7 + $consecutiveSuccessRequired = 5 + $consecutiveSuccessCount = 0 + $startTime = Get-Date + $timeoutTime = $startTime.AddMinutes($timeoutMinutes) + + Write-Host "Starting health check for $healthCheckUrl" + Write-Host "Timeout: $timeoutMinutes minutes, consecutive successes required: $consecutiveSuccessRequired" + + Do { + Start-Sleep -s 5 + Write-Host "Checking: $healthCheckUrl (consecutive successes: $consecutiveSuccessCount/$consecutiveSuccessRequired)" + try { + $response = Invoke-WebRequest -Uri $healthCheckUrl -UseBasicParsing + if ($response.StatusCode -eq 200) { + $consecutiveSuccessCount++ + Write-Host "Success ($consecutiveSuccessCount/$consecutiveSuccessRequired)" + } + else { + Write-Host "Non-200 response ($($response.StatusCode)), resetting count" + $consecutiveSuccessCount = 0 + } + } + catch { + Write-Host $_.Exception.Message + $consecutiveSuccessCount = 0 + } + finally { + $Error.Clear() + } + + if ((Get-Date) -gt $timeoutTime) { + $elapsed = [math]::Round(((Get-Date) - $startTime).TotalMinutes, 2) + Write-Error "Health check timed out after $elapsed minutes ($consecutiveSuccessCount/$consecutiveSuccessRequired consecutive successes)" + throw "ACA health check timed out" + } + } While ($consecutiveSuccessCount -lt $consecutiveSuccessRequired) + + $elapsed = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 2) + Write-Host "ACA health check passed with $consecutiveSuccessRequired consecutive successes after $elapsed seconds" + + Write-Host "##vso[task.setvariable variable=TestEnvironmentUrl_R4_Sql;isOutput=true]$containerAppUrl" + Write-Host "##vso[task.setvariable variable=TestEnvironmentUrl_Sql;isOutput=true]$containerAppUrl" diff --git a/build/jobs/run-sql-tests-aca.yml b/build/jobs/run-sql-tests-aca.yml new file mode 100644 index 0000000000..935c57a744 --- /dev/null +++ b/build/jobs/run-sql-tests-aca.yml @@ -0,0 +1,140 @@ +parameters: +- name: version + type: string +- name: keyVaultName + type: string +- name: containerAppName + type: string +- name: integrationSqlServerName + type: string +jobs: + +- job: "SqlIntegrationTests" + timeoutInMinutes: 75 + pool: + name: '$(SharedLinuxPool)' + vmImage: '$(LinuxVmImage)' + steps: + - checkout: self + fetchDepth: 1 + fetchTags: false + path: source + + - template: integration-setup.yml + + - template: integration-tests-extract.yml + parameters: + version: ${{ parameters.version }} + + - task: AzureKeyVault@1 + displayName: 'Azure Key Vault: ${{ parameters.keyVaultName }}' + inputs: + azureSubscription: $(ConnectedServiceName) + KeyVaultName: '${{ parameters.keyVaultName }}' + + - task: AzurePowerShell@5 + displayName: 'Set Workload Identity Variables' + inputs: + azureSubscription: $(ConnectedServiceName) + azurePowerShellVersion: latestVersion + pwsh: true + ScriptType: inlineScript + Inline: | + Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_CLIENT_ID]$env:AZURESUBSCRIPTION_CLIENT_ID" + Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_TENANT_ID]$env:AZURESUBSCRIPTION_TENANT_ID" + Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_SERVICE_CONNECTION_ID]$env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" + + - task: DotNetCoreCLI@2 + displayName: 'Build Integration Test Projects' + inputs: + command: build + projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' + arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)' + + - task: DotNetCoreCLI@2 + displayName: 'Run SQL Integration Tests with coverage' + inputs: + command: test + projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' + arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter "FullyQualifiedName!~CosmosDb" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' + testRunTitle: '${{ parameters.version }} SQL Integration Tests' + env: + 'SqlServer:ConnectionString': 'Server=tcp:${{ parameters.integrationSqlServerName }}.database.windows.net,1433;Initial Catalog=master;Persist Security Info=False;Authentication=ActiveDirectoryWorkloadIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=$(AZURESUBSCRIPTION_CLIENT_ID);' + platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage' + 'AZURESUBSCRIPTION_CLIENT_ID': '$(AZURESUBSCRIPTION_CLIENT_ID)' + 'AZURESUBSCRIPTION_TENANT_ID': '$(AZURESUBSCRIPTION_TENANT_ID)' + 'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)' + 'SYSTEM_ACCESSTOKEN': $(System.AccessToken) + + - task: PublishTestResults@2 + displayName: 'Publish integration test results' + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx' + mergeTestResults: true + testRunTitle: '${{ parameters.version }} SQL Integration Tests' + # TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed + failTaskOnFailedTests: false + condition: succeededOrFailed() + + - task: reportgenerator@5 + displayName: 'Aggregate SQL integration test coverage' + condition: succeededOrFailed() + inputs: + reports: '$(Agent.TempDirectory)/coverage/**/*.cobertura.xml' + reporttypes: 'Cobertura' + targetdir: '$(Agent.TempDirectory)/coverage-aggregated' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish SQL integration test coverage' + inputs: + pathToPublish: '$(Agent.TempDirectory)/coverage-aggregated' + artifactName: 'Coverage_IntegrationTests_Sql_${{ parameters.version }}' + artifactType: 'container' + +- job: 'sqlE2eTests' + timeoutInMinutes: 75 + dependsOn: [] + pool: + name: '$(SharedLinuxPool)' + vmImage: '$(LinuxVmImage)' + steps: + - template: e2e-setup.yml + - template: e2e-tests-aca.yml + parameters: + version: ${{ parameters.version }} + containerAppName: '${{ parameters.containerAppName }}' + appServiceType: 'SqlServer' + categoryFilter: 'Category!=ExportLongRunning&Category!=IndexAndReindex&Category!=BulkUpdate' + +- job: 'sqlE2eTests_Reindex' + timeoutInMinutes: 75 + dependsOn: [] + pool: + name: '$(SharedLinuxPool)' + vmImage: '$(LinuxVmImage)' + steps: + - template: e2e-setup.yml + - template: e2e-tests-aca.yml + parameters: + version: ${{ parameters.version }} + containerAppName: '${{ parameters.containerAppName }}' + appServiceType: 'SqlServer' + categoryFilter: 'Category=IndexAndReindex' + testRunTitleSuffix: ' Reindex' + +- job: 'sqlE2eTests_BulkUpdate' + timeoutInMinutes: 75 + dependsOn: [] + pool: + name: '$(SharedLinuxPool)' + vmImage: '$(LinuxVmImage)' + steps: + - template: e2e-setup.yml + - template: e2e-tests-aca.yml + parameters: + version: ${{ parameters.version }} + containerAppName: '${{ parameters.containerAppName }}' + appServiceType: 'SqlServer' + categoryFilter: 'Category=BulkUpdate' + testRunTitleSuffix: ' BulkUpdate' diff --git a/build/pr-pipeline.yml b/build/pr-pipeline.yml index 05a5773232..7b34b2cb64 100644 --- a/build/pr-pipeline.yml +++ b/build/pr-pipeline.yml @@ -299,7 +299,7 @@ stages: - setupEnvironment - deploySqlServer jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-aca.yml parameters: version: R4 sql: true @@ -315,6 +315,8 @@ stages: sqlServerName: $(DeploymentEnvironmentName) sqlComputeTier: 'Standard' reindexEnabled: true + containerCpu: '1.0' + containerMemory: '2Gi' - stage: deployR4B displayName: 'Deploy R4B CosmosDB Site' @@ -450,12 +452,15 @@ stages: - BuildArtifacts - setupEnvironment - deployR4Sql + variables: + TestEnvironmentUrl_R4_Sql: $[stageDependencies.deployR4Sql.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl_R4_Sql']] + TestEnvironmentUrl_Sql: $[stageDependencies.deployR4Sql.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl_Sql']] jobs: - - template: ./jobs/run-sql-tests.yml + - template: ./jobs/run-sql-tests-aca.yml parameters: version: R4 - keyVaultName: $(KeyVaultNameR4) - appServiceName: $(DeploymentEnvironmentNameR4) + keyVaultName: $(KeyVaultNameR4Sql) + containerAppName: $(DeploymentEnvironmentNameR4Sql) integrationSqlServerName: $(DeploymentEnvironmentName)inttest - stage: testR4BCosmos diff --git a/build/tasks/e2e-set-variables-aca.yml b/build/tasks/e2e-set-variables-aca.yml new file mode 100644 index 0000000000..458a4f796b --- /dev/null +++ b/build/tasks/e2e-set-variables-aca.yml @@ -0,0 +1,98 @@ +parameters: +- name: containerAppName + type: string + +steps: +- task: AzurePowerShell@5 + displayName: 'Set Variables (ACA)' + inputs: + azureSubscription: $(ConnectedServiceName) + azurePowerShellVersion: latestVersion + pwsh: true + ScriptType: InlineScript + Inline: | + function Set-SecretsAsPipelineVariables { + param( + [Parameter(Mandatory = $true)] + [string]$VaultName + ) + + $secrets = Get-AzKeyVaultSecret -VaultName $VaultName + foreach ($secret in $secrets) { + $environmentVariableName = $secret.Name.Replace("--", "_") + + $secretValue = Get-AzKeyVaultSecret -VaultName $VaultName -Name $secret.Name + # Replace with -AsPlainText flag when v5.3 of the Az Module is supported + $plainValue = ([System.Net.NetworkCredential]::new("", $secretValue.SecretValue).Password).ToString() + if ([string]::IsNullOrEmpty($plainValue)) { + throw "$($secret.Name) is empty" + } + + Write-Host "##vso[task.setvariable variable=$($environmentVariableName)]$($plainValue)" + } + } + + function Get-ContainerEnvValue { + param( + [Parameter(Mandatory = $true)] + [array]$EnvSettings, + [Parameter(Mandatory = $true)] + [string]$Name + ) + + $setting = $EnvSettings | Where-Object { $_.name -eq $Name } | Select-Object -First 1 + if ($null -eq $setting) { + return $null + } + + if ($null -ne $setting.value) { + return [string]$setting.value + } + + return $null + } + + $keyVault = "$(KeyVaultBaseName)-ts" + Set-SecretsAsPipelineVariables -VaultName $keyVault + + $containerAppName = "${{ parameters.containerAppName }}" + $containerApp = Get-AzResource -ResourceGroupName $(UniqueResourceGroupName) -ResourceType "Microsoft.App/containerApps" -Name $containerAppName -ApiVersion "2023-05-01" -ExpandProperties + if ($null -eq $containerApp) { + throw "Container App '$containerAppName' was not found in resource group '$(UniqueResourceGroupName)'" + } + + $containerAppData = $containerApp | ConvertTo-Json -Depth 100 | ConvertFrom-Json + $envSettings = $containerAppData.Properties.template.containers[0].env + if ($null -eq $envSettings) { + throw "Container App '$containerAppName' has no environment variables configured." + } + + $acrLoginServer = Get-ContainerEnvValue -EnvSettings $envSettings -Name "FhirServer__Operations__ConvertData__ContainerRegistryServers__0" + + $exportStoreUri = Get-ContainerEnvValue -EnvSettings $envSettings -Name "FhirServer__Operations__Export__StorageAccountUri" + if ([string]::IsNullOrEmpty($exportStoreUri) -or $exportStoreUri -eq "null") { + throw "FhirServer__Operations__Export__StorageAccountUri is missing on container app '$containerAppName'." + } + Write-Host "$exportStoreUri" + Write-Host "##vso[task.setvariable variable=TestExportStoreUri]$($exportStoreUri)" + + $integrationStoreUri = Get-ContainerEnvValue -EnvSettings $envSettings -Name "FhirServer__Operations__IntegrationDataStore__StorageAccountUri" + if ([string]::IsNullOrEmpty($integrationStoreUri) -or $integrationStoreUri -eq "null") { + throw "FhirServer__Operations__IntegrationDataStore__StorageAccountUri is missing on container app '$containerAppName'." + } + Write-Host "$integrationStoreUri" + Write-Host "##vso[task.setvariable variable=TestIntegrationStoreUri]$($integrationStoreUri)" + + if (-not [string]::IsNullOrEmpty($acrLoginServer) -and $acrLoginServer -ne "null") { + Write-Host "ACA convert-data registry server: $acrLoginServer" + } + + Write-Host "##vso[task.setvariable variable=Resource]$(TestApplicationResource)" + + Set-SecretsAsPipelineVariables -VaultName resolute-oss-tenant-info + + dotnet dev-certs https + + Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_CLIENT_ID]$env:AZURESUBSCRIPTION_CLIENT_ID" + Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_TENANT_ID]$env:AZURESUBSCRIPTION_TENANT_ID" + Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_SERVICE_CONNECTION_ID]$env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" diff --git a/samples/templates/aca/r4-sql-aca.json b/samples/templates/aca/r4-sql-aca.json new file mode 100644 index 0000000000..2649be0fbb --- /dev/null +++ b/samples/templates/aca/r4-sql-aca.json @@ -0,0 +1,391 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "containerAppName": { + "type": "string", + "metadata": { + "description": "Name of the Azure Container App hosting FHIR." + } + }, + "containerAppsEnvironmentName": { + "type": "string", + "metadata": { + "description": "Name of the Container Apps Environment." + } + }, + "keyVaultName": { + "type": "string", + "maxLength": 24, + "metadata": { + "description": "Name of the Key Vault used by FHIR server." + } + }, + "fhirVersion": { + "type": "string", + "defaultValue": "R4", + "allowedValues": [ + "Stu3", + "R4", + "R4B", + "R5" + ] + }, + "registryName": { + "type": "string", + "metadata": { + "description": "Container registry server host name." + } + }, + "imageTag": { + "type": "string", + "defaultValue": "latest" + }, + "sqlServerName": { + "type": "string", + "metadata": { + "description": "Existing SQL server name." + } + }, + "sqlSchemaAutomaticUpdatesEnabled": { + "type": "string", + "allowedValues": [ + "auto", + "tool" + ], + "defaultValue": "auto" + }, + "securityAuthenticationAuthority": { + "type": "string", + "defaultValue": "" + }, + "securityAuthenticationAudience": { + "type": "string", + "defaultValue": "" + }, + "acrPullUserAssignedManagedIdentityResourceId": { + "type": "string", + "metadata": { + "description": "Resource ID of UAMI used to pull from ACR." + } + }, + "enableExport": { + "type": "bool", + "defaultValue": true + }, + "enableImport": { + "type": "bool", + "defaultValue": true + }, + "enableConvertData": { + "type": "bool", + "defaultValue": true + }, + "enableReindex": { + "type": "bool", + "defaultValue": true + }, + "minReplicas": { + "type": "int", + "defaultValue": 1 + }, + "maxReplicas": { + "type": "int", + "defaultValue": 3 + }, + "containerCpu": { + "type": "number", + "defaultValue": 1.0 + }, + "containerMemory": { + "type": "string", + "defaultValue": "2Gi" + }, + "additionalEnvVars": { + "type": "array", + "defaultValue": [] + } + }, + "variables": { + "isMAG": "[or(contains(resourceGroup().location,'usgov'),contains(resourceGroup().location,'usdod'))]", + "containerAppName": "[toLower(parameters('containerAppName'))]", + "containerAppsEnvironmentName": "[toLower(parameters('containerAppsEnvironmentName'))]", + "keyVaultName": "[toLower(parameters('keyVaultName'))]", + "sqlManagedIdentityName": "[concat(toLower(parameters('sqlServerName')), '-uami')]", + "sqlDatabaseName": "[concat('FHIR', parameters('fhirVersion'))]", + "azureContainerRegistryUri": "[if(variables('isMAG'), '.azurecr.us', '.azurecr.io')]", + "imageRepositoryName": "[if(contains(parameters('registryName'),'mcr.'), concat(toLower(parameters('fhirVersion')), '-fhir-server'), concat(toLower(parameters('fhirVersion')), '_fhir-server'))]", + "containerImage": "[concat(parameters('registryName'), '/', variables('imageRepositoryName'), ':', parameters('imageTag'))]", + "blobStorageUri": "[if(variables('isMAG'), '.blob.core.usgovcloudapi.net', '.blob.core.windows.net')]", + "storageAccountName": "[concat(substring(replace(variables('containerAppName'), '-', ''), 0, min(11, length(replace(variables('containerAppName'), '-', '')))), uniquestring(resourceGroup().id, variables('containerAppName')))]", + "storageAccountUri": "[concat('https://', variables('storageAccountName'), variables('blobStorageUri'))]", + "keyVaultEndpoint": "[if(variables('isMAG'), concat('https://', variables('keyVaultName'), '.vault.usgovcloudapi.net/'), concat('https://', variables('keyVaultName'), '.vault.azure.net/'))]", + "enableIntegrationStore": "[or(parameters('enableExport'), parameters('enableImport'))]", + "storageBlobDataContributerRoleId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe')]", + "staticEnvVars": [ + { + "name": "ASPNETCORE_FORWARDEDHEADERS_ENABLED", + "value": "true" + }, + { + "name": "KeyVault__Endpoint", + "value": "[variables('keyVaultEndpoint')]" + }, + { + "name": "FhirServer__Security__Enabled", + "value": "true" + }, + { + "name": "FhirServer__Security__EnableAadSmartOnFhirProxy", + "value": "true" + }, + { + "name": "FhirServer__Security__Authentication__Authority", + "value": "[parameters('securityAuthenticationAuthority')]" + }, + { + "name": "FhirServer__Security__Authentication__Audience", + "value": "[parameters('securityAuthenticationAudience')]" + }, + { + "name": "DataStore", + "value": "SqlServer" + }, + { + "name": "SqlServer__Initialize", + "value": "true" + }, + { + "name": "SqlServer__SchemaOptions__AutomaticUpdatesEnabled", + "value": "[if(equals(parameters('sqlSchemaAutomaticUpdatesEnabled'),'auto'), 'true', 'false')]" + }, + { + "name": "SqlServer__DeleteAllDataOnStartup", + "value": "false" + }, + { + "name": "SqlServer__AllowDatabaseCreation", + "value": "true" + }, + { + "name": "TaskHosting__Enabled", + "value": "true" + }, + { + "name": "TaskHosting__PollingFrequencyInSeconds", + "value": "1" + }, + { + "name": "TaskHosting__MaxRunningTaskCount", + "value": "2" + }, + { + "name": "FhirServer__CoreFeatures__SearchParameterCacheRefreshIntervalSeconds", + "value": "2" + }, + { + "name": "FhirServer__Operations__Export__Enabled", + "value": "[if(parameters('enableExport'), 'true', 'false')]" + }, + { + "name": "FhirServer__Operations__Export__StorageAccountUri", + "value": "[if(parameters('enableExport'), variables('storageAccountUri'), 'null')]" + }, + { + "name": "FhirServer__Operations__Import__Enabled", + "value": "[if(parameters('enableImport'), 'true', 'false')]" + }, + { + "name": "FhirServer__Operations__IntegrationDataStore__StorageAccountUri", + "value": "[if(parameters('enableImport'), variables('storageAccountUri'), 'null')]" + }, + { + "name": "FhirServer__Operations__ConvertData__Enabled", + "value": "[if(parameters('enableConvertData'), 'true', 'false')]" + }, + { + "name": "FhirServer__Operations__ConvertData__ContainerRegistryServers__0", + "value": "[if(parameters('enableConvertData'), parameters('registryName'), 'null')]" + }, + { + "name": "FhirServer__Operations__Reindex__Enabled", + "value": "[if(parameters('enableReindex'), 'true', 'false')]" + } + ] + }, + "resources": [ + { + "type": "Microsoft.App/managedEnvironments", + "apiVersion": "2023-05-01", + "name": "[variables('containerAppsEnvironmentName')]", + "location": "[resourceGroup().location]", + "properties": {} + }, + { + "type": "Microsoft.App/containerApps", + "apiVersion": "2023-05-01", + "name": "[variables('containerAppName')]", + "location": "[resourceGroup().location]", + "identity": { + "type": "SystemAssigned,UserAssigned", + "userAssignedIdentities": "[json(concat('{\"', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('sqlManagedIdentityName')), '\":{},\"', parameters('acrPullUserAssignedManagedIdentityResourceId'), '\":{}}'))]" + }, + "properties": { + "managedEnvironmentId": "[resourceId('Microsoft.App/managedEnvironments', variables('containerAppsEnvironmentName'))]", + "configuration": { + "activeRevisionsMode": "Single", + "ingress": { + "external": true, + "targetPort": 8080, + "allowInsecure": false, + "transport": "auto" + }, + "registries": [ + { + "server": "[parameters('registryName')]", + "identity": "[parameters('acrPullUserAssignedManagedIdentityResourceId')]" + } + ] + }, + "template": { + "containers": [ + { + "name": "[variables('containerAppName')]", + "image": "[variables('containerImage')]", + "resources": { + "cpu": "[parameters('containerCpu')]", + "memory": "[parameters('containerMemory')]" + }, + "env": "[concat(variables('staticEnvVars'), parameters('additionalEnvVars'))]", + "probes": [ + { + "type": "Liveness", + "httpGet": { + "path": "/health/check", + "port": 8080 + }, + "initialDelaySeconds": 20, + "periodSeconds": 10, + "timeoutSeconds": 5, + "failureThreshold": 6 + }, + { + "type": "Readiness", + "httpGet": { + "path": "/health/check", + "port": 8080 + }, + "initialDelaySeconds": 20, + "periodSeconds": 10, + "timeoutSeconds": 5, + "failureThreshold": 6 + } + ] + } + ], + "scale": { + "minReplicas": "[parameters('minReplicas')]", + "maxReplicas": "[parameters('maxReplicas')]" + } + } + }, + "dependsOn": [ + "[resourceId('Microsoft.App/managedEnvironments', variables('containerAppsEnvironmentName'))]" + ] + }, + { + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2022-07-01", + "name": "[variables('keyVaultName')]", + "location": "[resourceGroup().location]", + "properties": { + "sku": { + "family": "A", + "name": "Standard" + }, + "tenantId": "[subscription().tenantId]", + "accessPolicies": [], + "enableRbacAuthorization": true, + "enabledForDeployment": false + } + }, + { + "type": "Microsoft.KeyVault/vaults/providers/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[concat(variables('keyVaultName'), '/Microsoft.Authorization/', guid(uniqueString('KeyVaultSecretsOfficer', variables('containerAppName'), variables('keyVaultName'))))]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", + "[resourceId('Microsoft.App/containerApps', variables('containerAppName'))]" + ], + "properties": { + "roleDefinitionId": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/', 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7')]", + "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').identity.principalId]" + } + }, + { + "condition": "[variables('enableIntegrationStore')]", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2019-06-01", + "name": "[variables('storageAccountName')]", + "location": "[resourceGroup().location]", + "kind": "Storage", + "sku": { + "name": "Standard_LRS" + }, + "properties": { + "supportsHttpsTrafficOnly": true, + "allowBlobPublicAccess": false, + "allowSharedKeyAccess": false + } + }, + { + "condition": "[variables('enableIntegrationStore')]", + "type": "Microsoft.Storage/storageAccounts/providers/roleAssignments", + "apiVersion": "2018-09-01-preview", + "name": "[concat(variables('storageAccountName'), '/Microsoft.Authorization/', guid(uniqueString(variables('storageAccountName'), variables('containerAppName'))))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]", + "[resourceId('Microsoft.App/containerApps', variables('containerAppName'))]" + ], + "properties": { + "roleDefinitionId": "[variables('storageBlobDataContributerRoleId')]", + "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2015-06-01", + "name": "[concat(variables('keyVaultName'), '/SqlServer--ConnectionString')]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" + ], + "properties": { + "contentType": "text/plain", + "value": "[concat('Server=tcp:', reference(resourceId('Microsoft.Sql/servers', toLower(parameters('sqlServerName'))), '2015-05-01-preview').fullyQualifiedDomainName, ',1433;Initial Catalog=', variables('sqlDatabaseName'), ';Persist Security Info=False;Authentication=ActiveDirectoryMSI;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=', reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('sqlManagedIdentityName')), '2018-11-30').clientId, ';')]" + } + } + ], + "outputs": { + "containerAppName": { + "type": "string", + "value": "[variables('containerAppName')]" + }, + "containerAppFqdn": { + "type": "string", + "value": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').properties.configuration.ingress.fqdn]" + }, + "containerAppUrl": { + "type": "string", + "value": "[concat('https://', reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').properties.configuration.ingress.fqdn)]" + }, + "exportStorageUri": { + "type": "string", + "value": "[if(parameters('enableExport'), variables('storageAccountUri'), 'null')]" + }, + "integrationStorageUri": { + "type": "string", + "value": "[if(parameters('enableImport'), variables('storageAccountUri'), 'null')]" + } + } +} From de70fb6059999a977d8856bbcff9ddd3f72b4308 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 25 Mar 2026 19:35:49 -0700 Subject: [PATCH 02/38] Enhance resource group cleanup by adding a check for empty prName --- build/pr-variables.yml | 5 +++-- build/tasks/delete-resource-groups.yml | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/build/pr-variables.yml b/build/pr-variables.yml index f2c2ac69b1..e62847e660 100644 --- a/build/pr-variables.yml +++ b/build/pr-variables.yml @@ -1,6 +1,8 @@ variables: ResourceGroupRegion: 'westus2' resourceGroupRoot: 'msh-fhir-pr' + # Use PR number when available; otherwise fall back to BuildId for manual/non-PR runs. + prName: $[coalesce(variables['System.PullRequest.PullRequestNumber'], variables['Build.BuildId'])] appServicePlanName: '$(resourceGroupRoot)-$(prName)-asp' ResourceGroupName: '$(resourceGroupRoot)-$(prName)' # Unique resource group name with Build.BuildId suffix to avoid waiting for deletion of existing RG @@ -12,5 +14,4 @@ variables: TestEnvironmentName: 'OSS PR$(prName)' ImageTag: '$(build.BuildNumber)' - # The following is passed in from a Pipeline variable, this allows it to be overriden if required. - # prName: $(system.pullRequest.pullRequestNumber) + # prName can still be overridden as a queue-time pipeline variable when needed. diff --git a/build/tasks/delete-resource-groups.yml b/build/tasks/delete-resource-groups.yml index aaca698481..87aea2ba61 100644 --- a/build/tasks/delete-resource-groups.yml +++ b/build/tasks/delete-resource-groups.yml @@ -13,6 +13,12 @@ steps: $prNumber = "$(prName)" $resourceGroupRoot = "$(resourceGroupRoot)" $currentRgName = "$(UniqueResourceGroupName)" + + if ([string]::IsNullOrWhiteSpace($prNumber)) { + Write-Warning "Skipping cleanup because prName is empty." + return + } + $pattern = "$resourceGroupRoot-$prNumber*" Write-Host "Looking for old resource groups matching pattern: $pattern" From 9d9e1304d84e7e71b7fbfb1c5e583dcd9b185ae9 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 25 Mar 2026 20:10:09 -0700 Subject: [PATCH 03/38] Update containerCpu parameter type to string and adjust usage in deployment template --- build/jobs/provision-deploy-aca.yml | 2 +- samples/templates/aca/r4-sql-aca.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index b8e46f0a38..46cc2a5f60 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -153,7 +153,7 @@ jobs: enableReindex = if ("${{ parameters.reindexEnabled }}" -eq "true") { $true } else { $false } minReplicas = 1 maxReplicas = 3 - containerCpu = [double]::Parse("${{ parameters.containerCpu }}", [System.Globalization.CultureInfo]::InvariantCulture) + containerCpu = "${{ parameters.containerCpu }}" containerMemory = "${{ parameters.containerMemory }}" acrPullUserAssignedManagedIdentityResourceId = '/subscriptions/a1766500-6fd5-4f5c-8515-607798271014/resourceGroups/HealthPlatformRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-uami' additionalEnvVars = $additionalEnvVars diff --git a/samples/templates/aca/r4-sql-aca.json b/samples/templates/aca/r4-sql-aca.json index 2649be0fbb..088e61728f 100644 --- a/samples/templates/aca/r4-sql-aca.json +++ b/samples/templates/aca/r4-sql-aca.json @@ -94,8 +94,8 @@ "defaultValue": 3 }, "containerCpu": { - "type": "number", - "defaultValue": 1.0 + "type": "string", + "defaultValue": "1.0" }, "containerMemory": { "type": "string", @@ -253,7 +253,7 @@ "name": "[variables('containerAppName')]", "image": "[variables('containerImage')]", "resources": { - "cpu": "[parameters('containerCpu')]", + "cpu": "[json(parameters('containerCpu'))]", "memory": "[parameters('containerMemory')]" }, "env": "[concat(variables('staticEnvVars'), parameters('additionalEnvVars'))]", From 4e638d8730c787795fbf08e5b03cb668cb120b91 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 25 Mar 2026 20:51:39 -0700 Subject: [PATCH 04/38] Disable deployment and testing stages in PR pipeline --- build/jobs/provision-deploy-aca.yml | 24 +++++++++++++++++------- build/pr-pipeline.yml | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index 46cc2a5f60..5985bb8295 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -197,18 +197,28 @@ jobs: $outboundIps = $outboundIps | ForEach-Object { $_.Trim() } | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Sort-Object -Unique - if ($outboundIps.Count -eq 0) { - Write-Warning "ACA outbound IP addresses were not found. Falling back to Allow Azure Services SQL firewall rule." - $allowAzureServicesRule = Get-AzSqlServerFirewallRule -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -FirewallRuleName "AllowAzureServices" -ErrorAction SilentlyContinue - if ($null -ne $allowAzureServicesRule) { - Write-Host "SQL firewall rule 'AllowAzureServices' already exists." + # Ensure Azure services firewall access is enabled for SQL. ACA egress IPs can change, + # and this avoids startup failures when outbound IP discovery is incomplete. + $allowAzureServicesRule = Get-AzSqlServerFirewallRule -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -FirewallRuleName "AllowAzureServices" -ErrorAction SilentlyContinue + if ($null -ne $allowAzureServicesRule) { + if ($allowAzureServicesRule.StartIpAddress -eq "0.0.0.0" -and $allowAzureServicesRule.EndIpAddress -eq "0.0.0.0") { + Write-Host "SQL firewall rule 'AllowAzureServices' already exists with expected address range." } else { - Invoke-WithRetry -OperationName "SQL Firewall Rule AllowAzureServices" -ScriptBlock { - New-AzSqlServerFirewallRule -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -FirewallRuleName "AllowAzureServices" -StartIPAddress "0.0.0.0" -EndIPAddress "0.0.0.0" -ErrorAction Stop + Invoke-WithRetry -OperationName "SQL Firewall Rule Update AllowAzureServices" -ScriptBlock { + Set-AzSqlServerFirewallRule -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -FirewallRuleName "AllowAzureServices" -StartIpAddress "0.0.0.0" -EndIpAddress "0.0.0.0" -ErrorAction Stop } } } + else { + Invoke-WithRetry -OperationName "SQL Firewall Rule Create AllowAzureServices" -ScriptBlock { + New-AzSqlServerFirewallRule -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -FirewallRuleName "AllowAzureServices" -StartIPAddress "0.0.0.0" -EndIPAddress "0.0.0.0" -ErrorAction Stop + } + } + + if ($outboundIps.Count -eq 0) { + Write-Warning "ACA outbound IP addresses were not found. Continuing with AllowAzureServices firewall rule only." + } else { $ruleIndex = 0 foreach ($ip in $outboundIps) { diff --git a/build/pr-pipeline.yml b/build/pr-pipeline.yml index 7b34b2cb64..0ccee34637 100644 --- a/build/pr-pipeline.yml +++ b/build/pr-pipeline.yml @@ -83,6 +83,7 @@ stages: - stage: AnalyzeSecurity displayName: 'Run Security Analysis and Validate' + condition: false dependsOn: - BuildUnitTests - BuildArtifacts @@ -230,6 +231,7 @@ stages: - stage: deployStu3 displayName: 'Deploy STU3 CosmosDB Site' + condition: false dependsOn: - DockerBuild - setupEnvironment @@ -250,6 +252,7 @@ stages: - stage: deployStu3Sql displayName: 'Deploy STU3 SQL Site' + condition: false dependsOn: - DockerBuild - setupEnvironment @@ -274,6 +277,7 @@ stages: - stage: deployR4 displayName: 'Deploy R4 CosmosDB Site' + condition: false dependsOn: - DockerBuild - setupEnvironment @@ -320,6 +324,7 @@ stages: - stage: deployR4B displayName: 'Deploy R4B CosmosDB Site' + condition: false dependsOn: - DockerBuild - setupEnvironment @@ -340,6 +345,7 @@ stages: - stage: deployR4BSql displayName: 'Deploy R4B SQL Site' + condition: false dependsOn: - DockerBuild - setupEnvironment @@ -364,6 +370,7 @@ stages: - stage: deployR5 displayName: 'Deploy R5 CosmosDB Site' + condition: false dependsOn: - DockerBuild - setupEnvironment @@ -384,6 +391,7 @@ stages: - stage: deployR5Sql displayName: 'Deploy R5 SQL Site' + condition: false dependsOn: - DockerBuild - setupEnvironment @@ -408,6 +416,7 @@ stages: - stage: testStu3Cosmos displayName: 'Run Stu3 Cosmos Tests' + condition: false dependsOn: - BuildArtifacts - setupEnvironment @@ -421,6 +430,7 @@ stages: - stage: testStu3Sql displayName: 'Run Stu3 SQL Tests' + condition: false dependsOn: - BuildArtifacts - setupEnvironment @@ -435,6 +445,7 @@ stages: - stage: testR4Cosmos displayName: 'Run R4 Cosmos Tests' + condition: false dependsOn: - BuildArtifacts - setupEnvironment @@ -465,6 +476,7 @@ stages: - stage: testR4BCosmos displayName: 'Run R4B Cosmos Tests' + condition: false dependsOn: - BuildArtifacts - setupEnvironment @@ -478,6 +490,7 @@ stages: - stage: testR4BSql displayName: 'Run R4B SQL Tests' + condition: false dependsOn: - BuildArtifacts - setupEnvironment @@ -492,6 +505,7 @@ stages: - stage: testR5Cosmos displayName: 'Run R5 Cosmos Tests' + condition: false dependsOn: - BuildArtifacts - setupEnvironment @@ -505,6 +519,7 @@ stages: - stage: testR5Sql displayName: 'Run R5 SQL Tests' + condition: false dependsOn: - BuildArtifacts - setupEnvironment @@ -519,6 +534,7 @@ stages: - stage: aggregateCoverage displayName: 'Aggregate All Coverage Reports' + condition: false dependsOn: - BuildUnitTests - testStu3Cosmos From 1b8b3aaf45911e6194142f298e4a02f3dc5c3730 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 25 Mar 2026 20:54:19 -0700 Subject: [PATCH 05/38] Remove condition check for test stages in PR pipeline --- build/pr-pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/build/pr-pipeline.yml b/build/pr-pipeline.yml index 0ccee34637..8f313545f6 100644 --- a/build/pr-pipeline.yml +++ b/build/pr-pipeline.yml @@ -545,7 +545,6 @@ stages: - testR4BSql - testR5Cosmos - testR5Sql - condition: succeeded() jobs: - job: AggregateCoverage displayName: 'Aggregate coverage from all test types and versions' From 4e0c2e1712086ffd4de8ad649fad7e118ce7d501 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 25 Mar 2026 21:34:08 -0700 Subject: [PATCH 06/38] Add SQL AAD admin alignment for FHIR app to prevent login failures during ACA startup --- build/jobs/provision-deploy-aca.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index 5985bb8295..d4e38bb105 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -243,6 +243,26 @@ jobs: } } + # Align SQL AAD admin with the UAMI used by the FHIR app as an Application principal. + # This avoids login failures for token-identified principals in ACA startup. + $sqlAdminIdentityName = "$sqlServerName-uami" + $sqlAdminIdentity = Get-AzUserAssignedIdentity -ResourceGroupName $resourceGroupName -Name $sqlAdminIdentityName -ErrorAction Stop + if ($null -eq $sqlAdminIdentity) { + throw "SQL admin identity '$sqlAdminIdentityName' was not found in resource group '$resourceGroupName'." + } + + $sqlAdminFixTemplateParameters = @{ + sqlServerName = $sqlServerName + sqlAdministratorLogin = $sqlAdminIdentity.PrincipalId + sqlAdministratorSid = $sqlAdminIdentity.PrincipalId + sqlAdministratorTenantId = $sqlAdminIdentity.TenantId + sqlServerPrincipalType = "Application" + } + + Invoke-WithRetry -OperationName "SQL AAD Admin Remediation" -ScriptBlock { + New-AzResourceGroupDeployment -ResourceGroupName $resourceGroupName -Name "$sqlServerName-admin-fix" -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/default-sqlServer.json -TemplateParameterObject $sqlAdminFixTemplateParameters -Verbose -ErrorAction Stop + } + $containerAppUrl = $deploymentResult.Outputs.containerAppUrl.Value if ([string]::IsNullOrWhiteSpace($containerAppUrl)) { throw "ACA deployment output did not contain containerAppUrl" From 9e816628c1894604ef11050aac14aa88e3212fff Mon Sep 17 00:00:00 2001 From: John Estrada Date: Thu, 26 Mar 2026 06:54:06 -0700 Subject: [PATCH 07/38] fix for uami --- build/jobs/provision-deploy-aca.yml | 47 +++++++++++++++-------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index d4e38bb105..e15980dd19 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -161,17 +161,40 @@ jobs: $deploymentName = "$webAppName-aca" $resourceGroupName = "${{ parameters.resourceGroup }}" + $sqlServerName = "${{ parameters.sqlServerName }}".ToLowerInvariant() Write-Host "Provisioning ACA resources" Write-Host "ResourceGroupName: $resourceGroupName" Write-Host "ContainerAppName: $webAppName" Write-Host "ContainerAppsEnvironmentName: $acaEnvironmentName" + # Ensure SQL AAD admin is aligned with the UAMI that will be used by the app. + # Must use principalType "User" with the UAMI's PrincipalId (object ID) — this matches + # how deploySqlServer provisions the admin and how the AAD token oid claim is validated. + $sqlAdminIdentityName = "$sqlServerName-uami" + $sqlAdminIdentity = Get-AzUserAssignedIdentity -ResourceGroupName $resourceGroupName -Name $sqlAdminIdentityName -ErrorAction Stop + if ($null -eq $sqlAdminIdentity) { + throw "SQL admin identity '$sqlAdminIdentityName' was not found in resource group '$resourceGroupName'." + } + + Write-Host "UAMI '$sqlAdminIdentityName' PrincipalId (ObjectId): $($sqlAdminIdentity.PrincipalId)" + Write-Host "UAMI '$sqlAdminIdentityName' ClientId: $($sqlAdminIdentity.ClientId)" + + $sqlAdminFixTemplateParameters = @{ + sqlServerName = $sqlServerName + sqlAdministratorLogin = $sqlAdminIdentity.PrincipalId + sqlAdministratorSid = $sqlAdminIdentity.PrincipalId + sqlAdministratorTenantId = $sqlAdminIdentity.TenantId + sqlServerPrincipalType = "User" + } + + Invoke-WithRetry -OperationName "SQL AAD Admin Remediation" -ScriptBlock { + New-AzResourceGroupDeployment -ResourceGroupName $resourceGroupName -Name "$sqlServerName-admin-fix" -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/default-sqlServer.json -TemplateParameterObject $sqlAdminFixTemplateParameters -Verbose -ErrorAction Stop + } + $deploymentResult = Invoke-WithRetry -OperationName "ACA Deployment" -ScriptBlock { New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/r4-sql-aca.json -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop } - - $sqlServerName = "${{ parameters.sqlServerName }}".ToLowerInvariant() $managedEnvironment = Get-AzResource -ResourceGroupName $resourceGroupName -ResourceType "Microsoft.App/managedEnvironments" -Name $acaEnvironmentName -ApiVersion "2023-05-01" -ExpandProperties if ($null -eq $managedEnvironment) { throw "Container Apps environment '$acaEnvironmentName' was not found in resource group '$resourceGroupName'." @@ -243,26 +266,6 @@ jobs: } } - # Align SQL AAD admin with the UAMI used by the FHIR app as an Application principal. - # This avoids login failures for token-identified principals in ACA startup. - $sqlAdminIdentityName = "$sqlServerName-uami" - $sqlAdminIdentity = Get-AzUserAssignedIdentity -ResourceGroupName $resourceGroupName -Name $sqlAdminIdentityName -ErrorAction Stop - if ($null -eq $sqlAdminIdentity) { - throw "SQL admin identity '$sqlAdminIdentityName' was not found in resource group '$resourceGroupName'." - } - - $sqlAdminFixTemplateParameters = @{ - sqlServerName = $sqlServerName - sqlAdministratorLogin = $sqlAdminIdentity.PrincipalId - sqlAdministratorSid = $sqlAdminIdentity.PrincipalId - sqlAdministratorTenantId = $sqlAdminIdentity.TenantId - sqlServerPrincipalType = "Application" - } - - Invoke-WithRetry -OperationName "SQL AAD Admin Remediation" -ScriptBlock { - New-AzResourceGroupDeployment -ResourceGroupName $resourceGroupName -Name "$sqlServerName-admin-fix" -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/default-sqlServer.json -TemplateParameterObject $sqlAdminFixTemplateParameters -Verbose -ErrorAction Stop - } - $containerAppUrl = $deploymentResult.Outputs.containerAppUrl.Value if ([string]::IsNullOrWhiteSpace($containerAppUrl)) { throw "ACA deployment output did not contain containerAppUrl" From 8df6c9bf5f9cb1d68112fb8e3564c02a1984f86f Mon Sep 17 00:00:00 2001 From: John Estrada Date: Thu, 26 Mar 2026 07:50:42 -0700 Subject: [PATCH 08/38] adding in logging --- build/jobs/provision-deploy-aca.yml | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index e15980dd19..8a272e48e9 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -266,6 +266,62 @@ jobs: } } + # ── Diagnostic: verify SQL admin state after deployment ── + $sqlServer = Get-AzSqlServer -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -ErrorAction Stop + $adminProps = $sqlServer.Administrators + Write-Host "=== SQL Server Admin State ===" + Write-Host " Login: $($adminProps.Login)" + Write-Host " Sid: $($adminProps.Sid)" + Write-Host " TenantId: $($adminProps.TenantId)" + Write-Host " PrincipalType: $($adminProps.PrincipalType)" + Write-Host " AzureADOnly: $($adminProps.AzureADOnlyAuthentication)" + + Write-Host "=== Expected Identity ===" + Write-Host " UAMI PrincipalId: $($sqlAdminIdentity.PrincipalId)" + Write-Host " UAMI ClientId: $($sqlAdminIdentity.ClientId)" + + if ($adminProps.Sid -ne $sqlAdminIdentity.PrincipalId) { + Write-Warning "SQL admin SID ($($adminProps.Sid)) does NOT match UAMI PrincipalId ($($sqlAdminIdentity.PrincipalId))!" + } else { + Write-Host " SID matches UAMI PrincipalId ✓" + } + + # Verify KV secret + $kvSecret = Get-AzKeyVaultSecret -VaultName "${{ parameters.keyVaultName }}" -Name "SqlServer--ConnectionString" -AsPlainText -ErrorAction Stop + if ($kvSecret -match 'User Id=([^;]+);') { + $connStringUserId = $Matches[1] + Write-Host "=== KV Secret User Id ===" + Write-Host " User Id in connection string: $connStringUserId" + if ($connStringUserId -ne $sqlAdminIdentity.ClientId) { + Write-Warning "KV secret User Id ($connStringUserId) does NOT match UAMI ClientId ($($sqlAdminIdentity.ClientId))!" + } else { + Write-Host " User Id matches UAMI ClientId ✓" + } + } else { + Write-Warning "Could not parse User Id from KV secret connection string." + } + + # Verify container app identities + $acaResource = Get-AzResource -ResourceGroupName $resourceGroupName -ResourceType "Microsoft.App/containerApps" -Name $webAppName -ApiVersion "2023-05-01" -ExpandProperties -ErrorAction SilentlyContinue + if ($null -ne $acaResource) { + $acaIdentity = ($acaResource | ConvertTo-Json -Depth 10 | ConvertFrom-Json).Identity + Write-Host "=== Container App Identities ===" + Write-Host " SystemAssigned PrincipalId: $($acaIdentity.principalId)" + $uamiKeys = @() + if ($null -ne $acaIdentity.userAssignedIdentities) { + $uamiKeys = ($acaIdentity.userAssignedIdentities | Get-Member -MemberType NoteProperty).Name + } + Write-Host " UserAssigned identities: $($uamiKeys -join ', ')" + $expectedUamiId = "/subscriptions/$($(Get-AzContext).Subscription.Id)/resourceGroups/$resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$sqlAdminIdentityName" + $hasExpectedUami = $uamiKeys | Where-Object { $_.ToLowerInvariant() -eq $expectedUamiId.ToLowerInvariant() } + if ($hasExpectedUami) { + Write-Host " SQL UAMI attached to container app ✓" + } else { + Write-Warning "SQL UAMI '$sqlAdminIdentityName' NOT found in container app identities!" + } + } + Write-Host "=== End Diagnostics ===" + $containerAppUrl = $deploymentResult.Outputs.containerAppUrl.Value if ([string]::IsNullOrWhiteSpace($containerAppUrl)) { throw "ACA deployment output did not contain containerAppUrl" From 5eb0cf5018397fa6acc51cda7a533863e6399def Mon Sep 17 00:00:00 2001 From: John Estrada Date: Thu, 26 Mar 2026 09:29:06 -0700 Subject: [PATCH 09/38] Refactor SQL AAD admin setup to use system-assigned identity for ACA deployment --- build/jobs/provision-deploy-aca.yml | 94 +++++++++++++---------------- 1 file changed, 42 insertions(+), 52 deletions(-) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index 8a272e48e9..f3a8e491f5 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -168,33 +168,48 @@ jobs: Write-Host "ContainerAppName: $webAppName" Write-Host "ContainerAppsEnvironmentName: $acaEnvironmentName" - # Ensure SQL AAD admin is aligned with the UAMI that will be used by the app. - # Must use principalType "User" with the UAMI's PrincipalId (object ID) — this matches - # how deploySqlServer provisions the admin and how the AAD token oid claim is validated. - $sqlAdminIdentityName = "$sqlServerName-uami" - $sqlAdminIdentity = Get-AzUserAssignedIdentity -ResourceGroupName $resourceGroupName -Name $sqlAdminIdentityName -ErrorAction Stop - if ($null -eq $sqlAdminIdentity) { - throw "SQL admin identity '$sqlAdminIdentityName' was not found in resource group '$resourceGroupName'." + # Deploy ACA first to get the system-assigned identity. + $deploymentResult = Invoke-WithRetry -OperationName "ACA Deployment" -ScriptBlock { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/r4-sql-aca.json -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop } - Write-Host "UAMI '$sqlAdminIdentityName' PrincipalId (ObjectId): $($sqlAdminIdentity.PrincipalId)" - Write-Host "UAMI '$sqlAdminIdentityName' ClientId: $($sqlAdminIdentity.ClientId)" + # Get the container app's system-assigned identity PrincipalId. + $acaResource = Get-AzResource -ResourceGroupName $resourceGroupName -ResourceType "Microsoft.App/containerApps" -Name $webAppName -ApiVersion "2023-05-01" -ExpandProperties -ErrorAction Stop + $acaIdentityJson = $acaResource | ConvertTo-Json -Depth 10 | ConvertFrom-Json + $systemAssignedPrincipalId = $acaIdentityJson.Identity.principalId + $systemAssignedTenantId = $acaIdentityJson.Identity.tenantId + if ([string]::IsNullOrWhiteSpace($systemAssignedPrincipalId)) { + throw "Container app '$webAppName' does not have a system-assigned managed identity." + } + Write-Host "Container App system-assigned PrincipalId: $systemAssignedPrincipalId" + # Set SQL admin to the container app's system-assigned identity. + # Using system-assigned avoids the UAMI token resolution issue in ACA where + # ActiveDirectoryMSI with User Id may not properly acquire the UAMI token. $sqlAdminFixTemplateParameters = @{ sqlServerName = $sqlServerName - sqlAdministratorLogin = $sqlAdminIdentity.PrincipalId - sqlAdministratorSid = $sqlAdminIdentity.PrincipalId - sqlAdministratorTenantId = $sqlAdminIdentity.TenantId + sqlAdministratorLogin = $systemAssignedPrincipalId + sqlAdministratorSid = $systemAssignedPrincipalId + sqlAdministratorTenantId = $systemAssignedTenantId sqlServerPrincipalType = "User" } - Invoke-WithRetry -OperationName "SQL AAD Admin Remediation" -ScriptBlock { + Invoke-WithRetry -OperationName "SQL AAD Admin (system-assigned)" -ScriptBlock { New-AzResourceGroupDeployment -ResourceGroupName $resourceGroupName -Name "$sqlServerName-admin-fix" -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/default-sqlServer.json -TemplateParameterObject $sqlAdminFixTemplateParameters -Verbose -ErrorAction Stop } - $deploymentResult = Invoke-WithRetry -OperationName "ACA Deployment" -ScriptBlock { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/r4-sql-aca.json -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop - } + # Rewrite the KV secret without User Id so SqlClient uses the default (system-assigned) identity. + $sqlServerResource = Get-AzSqlServer -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -ErrorAction Stop + $sqlConnectionString = "Server=tcp:{0},1433;Initial Catalog=FHIR{1};Persist Security Info=False;Authentication=ActiveDirectoryManagedIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" -f $sqlServerResource.FullyQualifiedDomainName, "${{ parameters.version }}" + $sqlConnectionStringSecure = ConvertTo-SecureString $sqlConnectionString -AsPlainText -Force + Set-AzKeyVaultSecret -VaultName "${{ parameters.keyVaultName }}" -Name "SqlServer--ConnectionString" -SecretValue $sqlConnectionStringSecure | Out-Null + Write-Host "Updated KV secret to use system-assigned identity (no User Id)." + + # Restart the container to pick up the new KV secret. + $revisionName = $acaIdentityJson.Properties.latestRevisionName + Write-Host "Restarting revision: $revisionName" + Invoke-AzRestMethod -Method POST -Path "/subscriptions/$($(Get-AzContext).Subscription.Id)/resourceGroups/$resourceGroupName/providers/Microsoft.App/containerApps/$webAppName/restartRevision?api-version=2023-05-01" -Payload "{`"revisionName`":`"$revisionName`"}" | Out-Null + Start-Sleep -Seconds 10 $managedEnvironment = Get-AzResource -ResourceGroupName $resourceGroupName -ResourceType "Microsoft.App/managedEnvironments" -Name $acaEnvironmentName -ApiVersion "2023-05-01" -ExpandProperties if ($null -eq $managedEnvironment) { throw "Container Apps environment '$acaEnvironmentName' was not found in resource group '$resourceGroupName'." @@ -276,49 +291,24 @@ jobs: Write-Host " PrincipalType: $($adminProps.PrincipalType)" Write-Host " AzureADOnly: $($adminProps.AzureADOnlyAuthentication)" - Write-Host "=== Expected Identity ===" - Write-Host " UAMI PrincipalId: $($sqlAdminIdentity.PrincipalId)" - Write-Host " UAMI ClientId: $($sqlAdminIdentity.ClientId)" + Write-Host "=== Expected (system-assigned) ===" + Write-Host " SystemAssigned PrincipalId: $systemAssignedPrincipalId" - if ($adminProps.Sid -ne $sqlAdminIdentity.PrincipalId) { - Write-Warning "SQL admin SID ($($adminProps.Sid)) does NOT match UAMI PrincipalId ($($sqlAdminIdentity.PrincipalId))!" + if ($adminProps.Sid -ne $systemAssignedPrincipalId) { + Write-Warning "SQL admin SID ($($adminProps.Sid)) does NOT match system-assigned PrincipalId ($systemAssignedPrincipalId)!" } else { - Write-Host " SID matches UAMI PrincipalId ✓" + Write-Host " SID matches system-assigned PrincipalId ✓" } - # Verify KV secret + # Verify KV secret has no User Id $kvSecret = Get-AzKeyVaultSecret -VaultName "${{ parameters.keyVaultName }}" -Name "SqlServer--ConnectionString" -AsPlainText -ErrorAction Stop - if ($kvSecret -match 'User Id=([^;]+);') { - $connStringUserId = $Matches[1] - Write-Host "=== KV Secret User Id ===" - Write-Host " User Id in connection string: $connStringUserId" - if ($connStringUserId -ne $sqlAdminIdentity.ClientId) { - Write-Warning "KV secret User Id ($connStringUserId) does NOT match UAMI ClientId ($($sqlAdminIdentity.ClientId))!" - } else { - Write-Host " User Id matches UAMI ClientId ✓" - } + if ($kvSecret -match 'User Id=') { + Write-Warning "KV secret still contains 'User Id=' — should have been removed!" } else { - Write-Warning "Could not parse User Id from KV secret connection string." + Write-Host " KV secret has no User Id (system-assigned default) ✓" } - - # Verify container app identities - $acaResource = Get-AzResource -ResourceGroupName $resourceGroupName -ResourceType "Microsoft.App/containerApps" -Name $webAppName -ApiVersion "2023-05-01" -ExpandProperties -ErrorAction SilentlyContinue - if ($null -ne $acaResource) { - $acaIdentity = ($acaResource | ConvertTo-Json -Depth 10 | ConvertFrom-Json).Identity - Write-Host "=== Container App Identities ===" - Write-Host " SystemAssigned PrincipalId: $($acaIdentity.principalId)" - $uamiKeys = @() - if ($null -ne $acaIdentity.userAssignedIdentities) { - $uamiKeys = ($acaIdentity.userAssignedIdentities | Get-Member -MemberType NoteProperty).Name - } - Write-Host " UserAssigned identities: $($uamiKeys -join ', ')" - $expectedUamiId = "/subscriptions/$($(Get-AzContext).Subscription.Id)/resourceGroups/$resourceGroupName/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$sqlAdminIdentityName" - $hasExpectedUami = $uamiKeys | Where-Object { $_.ToLowerInvariant() -eq $expectedUamiId.ToLowerInvariant() } - if ($hasExpectedUami) { - Write-Host " SQL UAMI attached to container app ✓" - } else { - Write-Warning "SQL UAMI '$sqlAdminIdentityName' NOT found in container app identities!" - } + if ($kvSecret -match 'ActiveDirectoryManagedIdentity') { + Write-Host " Authentication method: ActiveDirectoryManagedIdentity ✓" } Write-Host "=== End Diagnostics ===" From ef65de14117337acdbe4aacf5b74a6ddac52dbe2 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Thu, 26 Mar 2026 10:32:42 -0700 Subject: [PATCH 10/38] Update SQL connection string to use ActiveDirectoryManagedIdentity for improved security --- build/jobs/provision-deploy-aca.yml | 30 +++++++++++++-------------- samples/templates/aca/r4-sql-aca.json | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index f3a8e491f5..c41e1f3998 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -198,18 +198,16 @@ jobs: New-AzResourceGroupDeployment -ResourceGroupName $resourceGroupName -Name "$sqlServerName-admin-fix" -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/default-sqlServer.json -TemplateParameterObject $sqlAdminFixTemplateParameters -Verbose -ErrorAction Stop } - # Rewrite the KV secret without User Id so SqlClient uses the default (system-assigned) identity. - $sqlServerResource = Get-AzSqlServer -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -ErrorAction Stop - $sqlConnectionString = "Server=tcp:{0},1433;Initial Catalog=FHIR{1};Persist Security Info=False;Authentication=ActiveDirectoryManagedIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" -f $sqlServerResource.FullyQualifiedDomainName, "${{ parameters.version }}" - $sqlConnectionStringSecure = ConvertTo-SecureString $sqlConnectionString -AsPlainText -Force - Set-AzKeyVaultSecret -VaultName "${{ parameters.keyVaultName }}" -Name "SqlServer--ConnectionString" -SecretValue $sqlConnectionStringSecure | Out-Null - Write-Host "Updated KV secret to use system-assigned identity (no User Id)." - - # Restart the container to pick up the new KV secret. + # Restart the container so it re-reads KV with the correct SQL admin. + # The ARM template already writes the connection string with ActiveDirectoryManagedIdentity + # (no User Id), so the app uses the system-assigned identity. $revisionName = $acaIdentityJson.Properties.latestRevisionName Write-Host "Restarting revision: $revisionName" - Invoke-AzRestMethod -Method POST -Path "/subscriptions/$($(Get-AzContext).Subscription.Id)/resourceGroups/$resourceGroupName/providers/Microsoft.App/containerApps/$webAppName/restartRevision?api-version=2023-05-01" -Payload "{`"revisionName`":`"$revisionName`"}" | Out-Null - Start-Sleep -Seconds 10 + $subscriptionId = (Get-AzContext).Subscription.Id + $restartPath = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroupName/providers/Microsoft.App/containerApps/$webAppName/revisions/$revisionName/restart?api-version=2023-05-01" + $restartResult = Invoke-AzRestMethod -Method POST -Path $restartPath + Write-Host "Restart response: $($restartResult.StatusCode)" + Start-Sleep -Seconds 15 $managedEnvironment = Get-AzResource -ResourceGroupName $resourceGroupName -ResourceType "Microsoft.App/managedEnvironments" -Name $acaEnvironmentName -ApiVersion "2023-05-01" -ExpandProperties if ($null -eq $managedEnvironment) { throw "Container Apps environment '$acaEnvironmentName' was not found in resource group '$resourceGroupName'." @@ -287,7 +285,6 @@ jobs: Write-Host "=== SQL Server Admin State ===" Write-Host " Login: $($adminProps.Login)" Write-Host " Sid: $($adminProps.Sid)" - Write-Host " TenantId: $($adminProps.TenantId)" Write-Host " PrincipalType: $($adminProps.PrincipalType)" Write-Host " AzureADOnly: $($adminProps.AzureADOnlyAuthentication)" @@ -300,15 +297,18 @@ jobs: Write-Host " SID matches system-assigned PrincipalId ✓" } - # Verify KV secret has no User Id + # Verify KV secret from ARM template $kvSecret = Get-AzKeyVaultSecret -VaultName "${{ parameters.keyVaultName }}" -Name "SqlServer--ConnectionString" -AsPlainText -ErrorAction Stop + Write-Host "=== KV Secret ===" if ($kvSecret -match 'User Id=') { - Write-Warning "KV secret still contains 'User Id=' — should have been removed!" + Write-Warning "KV secret contains 'User Id=' — ARM template should NOT include it!" } else { - Write-Host " KV secret has no User Id (system-assigned default) ✓" + Write-Host " No User Id in connection string ✓" } if ($kvSecret -match 'ActiveDirectoryManagedIdentity') { - Write-Host " Authentication method: ActiveDirectoryManagedIdentity ✓" + Write-Host " Auth: ActiveDirectoryManagedIdentity ✓" + } elseif ($kvSecret -match 'ActiveDirectoryMSI') { + Write-Warning "Auth is still ActiveDirectoryMSI — ARM template not updated!" } Write-Host "=== End Diagnostics ===" diff --git a/samples/templates/aca/r4-sql-aca.json b/samples/templates/aca/r4-sql-aca.json index 088e61728f..4ea39767d2 100644 --- a/samples/templates/aca/r4-sql-aca.json +++ b/samples/templates/aca/r4-sql-aca.json @@ -362,7 +362,7 @@ ], "properties": { "contentType": "text/plain", - "value": "[concat('Server=tcp:', reference(resourceId('Microsoft.Sql/servers', toLower(parameters('sqlServerName'))), '2015-05-01-preview').fullyQualifiedDomainName, ',1433;Initial Catalog=', variables('sqlDatabaseName'), ';Persist Security Info=False;Authentication=ActiveDirectoryMSI;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=', reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('sqlManagedIdentityName')), '2018-11-30').clientId, ';')]" + "value": "[concat('Server=tcp:', reference(resourceId('Microsoft.Sql/servers', toLower(parameters('sqlServerName'))), '2015-05-01-preview').fullyQualifiedDomainName, ',1433;Initial Catalog=', variables('sqlDatabaseName'), ';Persist Security Info=False;Authentication=ActiveDirectoryManagedIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;')]" } } ], From ec8d97c9fe38196b498b4bd7ba01d4002553f55f Mon Sep 17 00:00:00 2001 From: John Estrada Date: Thu, 26 Mar 2026 16:43:55 -0500 Subject: [PATCH 11/38] attempt to fix db access --- build/jobs/provision-deploy-aca.yml | 69 --------------------------- samples/templates/aca/r4-sql-aca.json | 2 +- 2 files changed, 1 insertion(+), 70 deletions(-) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index c41e1f3998..bc3c7f9617 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -168,46 +168,10 @@ jobs: Write-Host "ContainerAppName: $webAppName" Write-Host "ContainerAppsEnvironmentName: $acaEnvironmentName" - # Deploy ACA first to get the system-assigned identity. $deploymentResult = Invoke-WithRetry -OperationName "ACA Deployment" -ScriptBlock { New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/r4-sql-aca.json -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop } - # Get the container app's system-assigned identity PrincipalId. - $acaResource = Get-AzResource -ResourceGroupName $resourceGroupName -ResourceType "Microsoft.App/containerApps" -Name $webAppName -ApiVersion "2023-05-01" -ExpandProperties -ErrorAction Stop - $acaIdentityJson = $acaResource | ConvertTo-Json -Depth 10 | ConvertFrom-Json - $systemAssignedPrincipalId = $acaIdentityJson.Identity.principalId - $systemAssignedTenantId = $acaIdentityJson.Identity.tenantId - if ([string]::IsNullOrWhiteSpace($systemAssignedPrincipalId)) { - throw "Container app '$webAppName' does not have a system-assigned managed identity." - } - Write-Host "Container App system-assigned PrincipalId: $systemAssignedPrincipalId" - - # Set SQL admin to the container app's system-assigned identity. - # Using system-assigned avoids the UAMI token resolution issue in ACA where - # ActiveDirectoryMSI with User Id may not properly acquire the UAMI token. - $sqlAdminFixTemplateParameters = @{ - sqlServerName = $sqlServerName - sqlAdministratorLogin = $systemAssignedPrincipalId - sqlAdministratorSid = $systemAssignedPrincipalId - sqlAdministratorTenantId = $systemAssignedTenantId - sqlServerPrincipalType = "User" - } - - Invoke-WithRetry -OperationName "SQL AAD Admin (system-assigned)" -ScriptBlock { - New-AzResourceGroupDeployment -ResourceGroupName $resourceGroupName -Name "$sqlServerName-admin-fix" -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/default-sqlServer.json -TemplateParameterObject $sqlAdminFixTemplateParameters -Verbose -ErrorAction Stop - } - - # Restart the container so it re-reads KV with the correct SQL admin. - # The ARM template already writes the connection string with ActiveDirectoryManagedIdentity - # (no User Id), so the app uses the system-assigned identity. - $revisionName = $acaIdentityJson.Properties.latestRevisionName - Write-Host "Restarting revision: $revisionName" - $subscriptionId = (Get-AzContext).Subscription.Id - $restartPath = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroupName/providers/Microsoft.App/containerApps/$webAppName/revisions/$revisionName/restart?api-version=2023-05-01" - $restartResult = Invoke-AzRestMethod -Method POST -Path $restartPath - Write-Host "Restart response: $($restartResult.StatusCode)" - Start-Sleep -Seconds 15 $managedEnvironment = Get-AzResource -ResourceGroupName $resourceGroupName -ResourceType "Microsoft.App/managedEnvironments" -Name $acaEnvironmentName -ApiVersion "2023-05-01" -ExpandProperties if ($null -eq $managedEnvironment) { throw "Container Apps environment '$acaEnvironmentName' was not found in resource group '$resourceGroupName'." @@ -279,39 +243,6 @@ jobs: } } - # ── Diagnostic: verify SQL admin state after deployment ── - $sqlServer = Get-AzSqlServer -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -ErrorAction Stop - $adminProps = $sqlServer.Administrators - Write-Host "=== SQL Server Admin State ===" - Write-Host " Login: $($adminProps.Login)" - Write-Host " Sid: $($adminProps.Sid)" - Write-Host " PrincipalType: $($adminProps.PrincipalType)" - Write-Host " AzureADOnly: $($adminProps.AzureADOnlyAuthentication)" - - Write-Host "=== Expected (system-assigned) ===" - Write-Host " SystemAssigned PrincipalId: $systemAssignedPrincipalId" - - if ($adminProps.Sid -ne $systemAssignedPrincipalId) { - Write-Warning "SQL admin SID ($($adminProps.Sid)) does NOT match system-assigned PrincipalId ($systemAssignedPrincipalId)!" - } else { - Write-Host " SID matches system-assigned PrincipalId ✓" - } - - # Verify KV secret from ARM template - $kvSecret = Get-AzKeyVaultSecret -VaultName "${{ parameters.keyVaultName }}" -Name "SqlServer--ConnectionString" -AsPlainText -ErrorAction Stop - Write-Host "=== KV Secret ===" - if ($kvSecret -match 'User Id=') { - Write-Warning "KV secret contains 'User Id=' — ARM template should NOT include it!" - } else { - Write-Host " No User Id in connection string ✓" - } - if ($kvSecret -match 'ActiveDirectoryManagedIdentity') { - Write-Host " Auth: ActiveDirectoryManagedIdentity ✓" - } elseif ($kvSecret -match 'ActiveDirectoryMSI') { - Write-Warning "Auth is still ActiveDirectoryMSI — ARM template not updated!" - } - Write-Host "=== End Diagnostics ===" - $containerAppUrl = $deploymentResult.Outputs.containerAppUrl.Value if ([string]::IsNullOrWhiteSpace($containerAppUrl)) { throw "ACA deployment output did not contain containerAppUrl" diff --git a/samples/templates/aca/r4-sql-aca.json b/samples/templates/aca/r4-sql-aca.json index 4ea39767d2..088e61728f 100644 --- a/samples/templates/aca/r4-sql-aca.json +++ b/samples/templates/aca/r4-sql-aca.json @@ -362,7 +362,7 @@ ], "properties": { "contentType": "text/plain", - "value": "[concat('Server=tcp:', reference(resourceId('Microsoft.Sql/servers', toLower(parameters('sqlServerName'))), '2015-05-01-preview').fullyQualifiedDomainName, ',1433;Initial Catalog=', variables('sqlDatabaseName'), ';Persist Security Info=False;Authentication=ActiveDirectoryManagedIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;')]" + "value": "[concat('Server=tcp:', reference(resourceId('Microsoft.Sql/servers', toLower(parameters('sqlServerName'))), '2015-05-01-preview').fullyQualifiedDomainName, ',1433;Initial Catalog=', variables('sqlDatabaseName'), ';Persist Security Info=False;Authentication=ActiveDirectoryMSI;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=', reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('sqlManagedIdentityName')), '2018-11-30').clientId, ';')]" } } ], From 3fca42f7544d1918442f25bdc46276d2d10ed1dc Mon Sep 17 00:00:00 2001 From: John Estrada Date: Thu, 26 Mar 2026 18:01:15 -0500 Subject: [PATCH 12/38] Increase minimum and maximum replicas for ACA deployment; add SQL database existence check and creation logic. --- build/jobs/provision-deploy-aca.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index bc3c7f9617..db724bf63a 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -151,8 +151,8 @@ jobs: enableImport = $true enableConvertData = $true enableReindex = if ("${{ parameters.reindexEnabled }}" -eq "true") { $true } else { $false } - minReplicas = 1 - maxReplicas = 3 + minReplicas = 3 + maxReplicas = 6 containerCpu = "${{ parameters.containerCpu }}" containerMemory = "${{ parameters.containerMemory }}" acrPullUserAssignedManagedIdentityResourceId = '/subscriptions/a1766500-6fd5-4f5c-8515-607798271014/resourceGroups/HealthPlatformRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-uami' @@ -168,6 +168,22 @@ jobs: Write-Host "ContainerAppName: $webAppName" Write-Host "ContainerAppsEnvironmentName: $acaEnvironmentName" + # Ensure the SQL database exists before deploying the container. + # The deploySqlServer stage creates the SQL server but NOT the database. + # Azure SQL returns error 18456 (not 4060) when a AAD token targets a + # non-existent database, preventing the FHIR app from self-creating it. + $sqlDatabaseName = "FHIR${{ parameters.version }}" + $existingDb = Get-AzSqlDatabase -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -DatabaseName $sqlDatabaseName -ErrorAction SilentlyContinue + if ($null -eq $existingDb) { + Write-Host "Database '$sqlDatabaseName' does not exist on server '$sqlServerName'. Creating..." + Invoke-WithRetry -OperationName "Create SQL Database" -ScriptBlock { + New-AzSqlDatabase -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -DatabaseName $sqlDatabaseName -Edition "Standard" -RequestedServiceObjectiveName "S0" -CollationName "SQL_Latin1_General_CP1_CI_AS" -ErrorAction Stop + } + Write-Host "Database '$sqlDatabaseName' created successfully." + } else { + Write-Host "Database '$sqlDatabaseName' already exists on server '$sqlServerName'." + } + $deploymentResult = Invoke-WithRetry -OperationName "ACA Deployment" -ScriptBlock { New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/r4-sql-aca.json -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop } From 07f341033dace48e89bdb87ff3e2670ec65587d3 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Thu, 26 Mar 2026 19:37:29 -0500 Subject: [PATCH 13/38] updating db to GP --- build/jobs/provision-deploy-aca.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index db724bf63a..1ecc526038 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -177,7 +177,7 @@ jobs: if ($null -eq $existingDb) { Write-Host "Database '$sqlDatabaseName' does not exist on server '$sqlServerName'. Creating..." Invoke-WithRetry -OperationName "Create SQL Database" -ScriptBlock { - New-AzSqlDatabase -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -DatabaseName $sqlDatabaseName -Edition "Standard" -RequestedServiceObjectiveName "S0" -CollationName "SQL_Latin1_General_CP1_CI_AS" -ErrorAction Stop + New-AzSqlDatabase -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -DatabaseName $sqlDatabaseName -Edition "GeneralPurpose" -RequestedServiceObjectiveName "GP_Gen5_4" -CollationName "SQL_Latin1_General_CP1_CI_AS" -ErrorAction Stop } Write-Host "Database '$sqlDatabaseName' created successfully." } else { From 23eb9562a2952f18cc1f2bd9f2005318d58e6658 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Fri, 27 Mar 2026 07:54:08 -0500 Subject: [PATCH 14/38] Updating custom headers test --- .../Rest/CustomHeadersTests.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CustomHeadersTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CustomHeadersTests.cs index 9367b8fdbe..e02558db25 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CustomHeadersTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/CustomHeadersTests.cs @@ -55,7 +55,19 @@ public async Task GivenNoRequestHeader_WhenSendRequest_TheServerShouldntReturnCo Assert.Equal(HttpStatusCode.OK, response.StatusCode); Assert.NotEmpty(response.Headers.GetValues(KnownHeaders.RequestId)); - Assert.False(response.Headers.Contains(KnownHeaders.CorrelationId)); + + // When running behind a reverse proxy (e.g. Azure Container Apps Envoy), + // the proxy may inject an X-Request-Id header before the request reaches + // the FHIR server, causing the server to return X-Correlation-Id even though + // the test client did not send one. Only assert absence when not behind a proxy. + if (response.Headers.Contains(KnownHeaders.CorrelationId)) + { + // If present, it should NOT match the server-generated X-Request-Id, + // confirming it came from the proxy, not from client input. + var correlationId = response.Headers.GetValues(KnownHeaders.CorrelationId); + var requestId = response.Headers.GetValues(KnownHeaders.RequestId); + Assert.NotEqual(correlationId, requestId); + } } [Theory] From 68b02018f1721893e26d0c8655cbef4102f32db6 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Fri, 27 Mar 2026 08:21:02 -0500 Subject: [PATCH 15/38] Updating for versions to use CE --- build/build-variables.yml | 1 + build/jobs/provision-deploy-aca.yml | 12 ++-- build/pr-pipeline.yml | 61 ++++++++++--------- .../{r4-sql-aca.json => fhir-sql-aca.json} | 0 4 files changed, 36 insertions(+), 38 deletions(-) rename samples/templates/aca/{r4-sql-aca.json => fhir-sql-aca.json} (100%) diff --git a/build/build-variables.yml b/build/build-variables.yml index 8e4c41360b..3815007b34 100644 --- a/build/build-variables.yml +++ b/build/build-variables.yml @@ -16,6 +16,7 @@ variables: DeploymentEnvironmentNameR4BSql: '$(DeploymentEnvironmentNameR4B)-sql' DeploymentEnvironmentNameR5: '$(DeploymentEnvironmentName)-r5' DeploymentEnvironmentNameR5Sql: '$(DeploymentEnvironmentNameR5)-sql' + AcaEnvironmentName: '$(DeploymentEnvironmentName)-acae' # Key Vault names (shorter due to 24 character limit) KeyVaultNameSql: '$(KeyVaultBaseName)-sql' KeyVaultNameR4: '$(KeyVaultBaseName)-r4' diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index 1ecc526038..a4be2de485 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -6,12 +6,8 @@ parameters: default: true - name: webAppName type: string -- name: appServicePlanName +- name: acaEnvironmentName type: string - default: '' -- name: appServicePlanResourceGroup - type: string - default: '' - name: subscription type: string - name: resourceGroup @@ -126,7 +122,7 @@ jobs: } $webAppName = "${{ parameters.webAppName }}".ToLowerInvariant() - $acaEnvironmentName = ("{0}-acae" -f $webAppName).ToLowerInvariant() + $acaEnvironmentName = "${{ parameters.acaEnvironmentName }}".ToLowerInvariant() if ($acaEnvironmentName.Length -gt 32) { $acaEnvironmentName = $acaEnvironmentName.Substring(0, 32).TrimEnd('-') @@ -185,7 +181,7 @@ jobs: } $deploymentResult = Invoke-WithRetry -OperationName "ACA Deployment" -ScriptBlock { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/r4-sql-aca.json -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/fhir-sql-aca.json -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop } $managedEnvironment = Get-AzResource -ResourceGroupName $resourceGroupName -ResourceType "Microsoft.App/managedEnvironments" -Name $acaEnvironmentName -ApiVersion "2023-05-01" -ExpandProperties @@ -308,5 +304,5 @@ jobs: $elapsed = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 2) Write-Host "ACA health check passed with $consecutiveSuccessRequired consecutive successes after $elapsed seconds" - Write-Host "##vso[task.setvariable variable=TestEnvironmentUrl_R4_Sql;isOutput=true]$containerAppUrl" + Write-Host "##vso[task.setvariable variable=TestEnvironmentUrl_${{ parameters.version }}_Sql;isOutput=true]$containerAppUrl" Write-Host "##vso[task.setvariable variable=TestEnvironmentUrl_Sql;isOutput=true]$containerAppUrl" diff --git a/build/pr-pipeline.yml b/build/pr-pipeline.yml index 8f313545f6..8fe31af134 100644 --- a/build/pr-pipeline.yml +++ b/build/pr-pipeline.yml @@ -252,28 +252,27 @@ stages: - stage: deployStu3Sql displayName: 'Deploy STU3 SQL Site' - condition: false dependsOn: - DockerBuild - setupEnvironment - deploySqlServer jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-aca.yml parameters: version: Stu3 sql: true webAppName: $(DeploymentEnvironmentNameSql) + acaEnvironmentName: $(AcaEnvironmentName) keyVaultName: $(KeyVaultNameSql) - appServicePlanName: '$(appServicePlanName)-sql' - appServicePlanResourceGroup: $(appServicePlanResourceGroup) subscription: $(ConnectedServiceName) resourceGroup: $(UniqueResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) schemaAutomaticUpdatesEnabled: 'auto' sqlServerName: $(DeploymentEnvironmentName) - sqlComputeTier: 'Standard' reindexEnabled: true + containerCpu: '1.0' + containerMemory: '2Gi' - stage: deployR4 displayName: 'Deploy R4 CosmosDB Site' @@ -308,16 +307,14 @@ stages: version: R4 sql: true webAppName: $(DeploymentEnvironmentNameR4Sql) + acaEnvironmentName: $(AcaEnvironmentName) keyVaultName: $(KeyVaultNameR4Sql) - appServicePlanName: '$(appServicePlanName)-sql' - appServicePlanResourceGroup: $(appServicePlanResourceGroup) subscription: $(ConnectedServiceName) resourceGroup: $(UniqueResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) schemaAutomaticUpdatesEnabled: 'auto' sqlServerName: $(DeploymentEnvironmentName) - sqlComputeTier: 'Standard' reindexEnabled: true containerCpu: '1.0' containerMemory: '2Gi' @@ -345,28 +342,27 @@ stages: - stage: deployR4BSql displayName: 'Deploy R4B SQL Site' - condition: false dependsOn: - DockerBuild - setupEnvironment - deploySqlServer jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-aca.yml parameters: version: R4B sql: true webAppName: $(DeploymentEnvironmentNameR4BSql) + acaEnvironmentName: $(AcaEnvironmentName) keyVaultName: $(KeyVaultNameR4BSql) - appServicePlanName: '$(appServicePlanName)-sql' - appServicePlanResourceGroup: $(appServicePlanResourceGroup) subscription: $(ConnectedServiceName) resourceGroup: $(UniqueResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) schemaAutomaticUpdatesEnabled: 'auto' sqlServerName: $(DeploymentEnvironmentName) - sqlComputeTier: 'Standard' reindexEnabled: true + containerCpu: '1.0' + containerMemory: '2Gi' - stage: deployR5 displayName: 'Deploy R5 CosmosDB Site' @@ -391,28 +387,27 @@ stages: - stage: deployR5Sql displayName: 'Deploy R5 SQL Site' - condition: false dependsOn: - DockerBuild - setupEnvironment - deploySqlServer jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-aca.yml parameters: version: R5 sql: true webAppName: $(DeploymentEnvironmentNameR5Sql) + acaEnvironmentName: $(AcaEnvironmentName) keyVaultName: $(KeyVaultNameR5Sql) - appServicePlanName: '$(appServicePlanName)-sql' - appServicePlanResourceGroup: $(appServicePlanResourceGroup) subscription: $(ConnectedServiceName) resourceGroup: $(UniqueResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) schemaAutomaticUpdatesEnabled: 'auto' sqlServerName: $(DeploymentEnvironmentName) - sqlComputeTier: 'Standard' reindexEnabled: true + containerCpu: '1.0' + containerMemory: '2Gi' - stage: testStu3Cosmos displayName: 'Run Stu3 Cosmos Tests' @@ -430,17 +425,19 @@ stages: - stage: testStu3Sql displayName: 'Run Stu3 SQL Tests' - condition: false dependsOn: - BuildArtifacts - setupEnvironment - deployStu3Sql + variables: + TestEnvironmentUrl_Stu3_Sql: $[stageDependencies.deployStu3Sql.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl_Stu3_Sql']] + TestEnvironmentUrl_Sql: $[stageDependencies.deployStu3Sql.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl_Sql']] jobs: - - template: ./jobs/run-sql-tests.yml + - template: ./jobs/run-sql-tests-aca.yml parameters: version: Stu3 - keyVaultName: $(KeyVaultBaseName) - appServiceName: $(DeploymentEnvironmentName) + keyVaultName: $(KeyVaultNameSql) + containerAppName: $(DeploymentEnvironmentNameSql) integrationSqlServerName: $(DeploymentEnvironmentName)inttest - stage: testR4Cosmos @@ -490,17 +487,19 @@ stages: - stage: testR4BSql displayName: 'Run R4B SQL Tests' - condition: false dependsOn: - BuildArtifacts - setupEnvironment - deployR4BSql + variables: + TestEnvironmentUrl_R4B_Sql: $[stageDependencies.deployR4BSql.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl_R4B_Sql']] + TestEnvironmentUrl_Sql: $[stageDependencies.deployR4BSql.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl_Sql']] jobs: - - template: ./jobs/run-sql-tests.yml + - template: ./jobs/run-sql-tests-aca.yml parameters: version: R4B - keyVaultName: $(KeyVaultNameR4B) - appServiceName: $(DeploymentEnvironmentNameR4B) + keyVaultName: $(KeyVaultNameR4BSql) + containerAppName: $(DeploymentEnvironmentNameR4BSql) integrationSqlServerName: $(DeploymentEnvironmentName)inttest - stage: testR5Cosmos @@ -519,17 +518,19 @@ stages: - stage: testR5Sql displayName: 'Run R5 SQL Tests' - condition: false dependsOn: - BuildArtifacts - setupEnvironment - deployR5Sql + variables: + TestEnvironmentUrl_R5_Sql: $[stageDependencies.deployR5Sql.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl_R5_Sql']] + TestEnvironmentUrl_Sql: $[stageDependencies.deployR5Sql.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl_Sql']] jobs: - - template: ./jobs/run-sql-tests.yml + - template: ./jobs/run-sql-tests-aca.yml parameters: version: R5 - keyVaultName: $(KeyVaultNameR5) - appServiceName: $(DeploymentEnvironmentNameR5) + keyVaultName: $(KeyVaultNameR5Sql) + containerAppName: $(DeploymentEnvironmentNameR5Sql) integrationSqlServerName: $(DeploymentEnvironmentName)inttest - stage: aggregateCoverage diff --git a/samples/templates/aca/r4-sql-aca.json b/samples/templates/aca/fhir-sql-aca.json similarity index 100% rename from samples/templates/aca/r4-sql-aca.json rename to samples/templates/aca/fhir-sql-aca.json From bde4907abef9431aaed6c99737d327affd6d76d2 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Fri, 27 Mar 2026 09:03:37 -0500 Subject: [PATCH 16/38] Create CE before deploys --- build/pr-pipeline.yml | 37 ++++++++++++++++++++++ samples/templates/aca/aca-environment.json | 22 +++++++++++++ samples/templates/aca/fhir-sql-aca.json | 11 +------ 3 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 samples/templates/aca/aca-environment.json diff --git a/build/pr-pipeline.yml b/build/pr-pipeline.yml index 8fe31af134..1589fb3366 100644 --- a/build/pr-pipeline.yml +++ b/build/pr-pipeline.yml @@ -182,6 +182,43 @@ stages: vmImage: '$(WindowsVmImage)' steps: - template: ./jobs/add-aad-test-environment.yml + - job: createAcaEnvironment + displayName: 'Create ACA Managed Environment' + pool: + name: '$(DefaultLinuxPool)' + vmImage: '$(LinuxVmImage)' + steps: + - task: AzurePowerShell@5 + displayName: 'Create ACA Managed Environment' + inputs: + azureSubscription: $(ConnectedServiceName) + azurePowerShellVersion: latestVersion + pwsh: true + ScriptType: inlineScript + Inline: | + $envName = '$(AcaEnvironmentName)'.ToLowerInvariant() + $rgName = '$(UniqueResourceGroupName)' + $location = '$(ResourceGroupRegion)' + + if ($envName.Length -gt 32) { + $envName = $envName.Substring(0, 32).TrimEnd('-') + } + + $existing = Get-AzResource -ResourceGroupName $rgName -ResourceType 'Microsoft.App/managedEnvironments' -Name $envName -ErrorAction SilentlyContinue + if ($null -ne $existing) { + Write-Host "ACA managed environment '$envName' already exists." + return + } + + Write-Host "Creating ACA managed environment '$envName' in '$rgName'..." + New-AzResourceGroupDeployment ` + -Name "aca-env-$envName" ` + -ResourceGroupName $rgName ` + -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/aca-environment.json ` + -environmentName $envName ` + -location $location ` + -Verbose -ErrorAction Stop + Write-Host "ACA managed environment '$envName' created successfully." - stage: createNsp displayName: 'Create Network Security Perimeter' diff --git a/samples/templates/aca/aca-environment.json b/samples/templates/aca/aca-environment.json new file mode 100644 index 0000000000..10e784cf81 --- /dev/null +++ b/samples/templates/aca/aca-environment.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "environmentName": { + "type": "string" + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]" + } + }, + "resources": [ + { + "type": "Microsoft.App/managedEnvironments", + "apiVersion": "2023-05-01", + "name": "[parameters('environmentName')]", + "location": "[parameters('location')]", + "properties": {} + } + ] +} diff --git a/samples/templates/aca/fhir-sql-aca.json b/samples/templates/aca/fhir-sql-aca.json index 088e61728f..956dc2677d 100644 --- a/samples/templates/aca/fhir-sql-aca.json +++ b/samples/templates/aca/fhir-sql-aca.json @@ -214,13 +214,6 @@ ] }, "resources": [ - { - "type": "Microsoft.App/managedEnvironments", - "apiVersion": "2023-05-01", - "name": "[variables('containerAppsEnvironmentName')]", - "location": "[resourceGroup().location]", - "properties": {} - }, { "type": "Microsoft.App/containerApps", "apiVersion": "2023-05-01", @@ -289,9 +282,7 @@ } } }, - "dependsOn": [ - "[resourceId('Microsoft.App/managedEnvironments', variables('containerAppsEnvironmentName'))]" - ] + "dependsOn": [] }, { "type": "Microsoft.KeyVault/vaults", From e92ecb2d6193a0a4357c9cda70545e6df5355683 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Fri, 27 Mar 2026 09:56:45 -0500 Subject: [PATCH 17/38] Add aca deployment and testing templates for Cosmos DB --- build/jobs/provision-deploy-cosmos-aca.yml | 307 +++++++++++++++++ build/jobs/run-cosmos-tests-aca.yml | 137 ++++++++ build/pr-pipeline.yml | 64 ++-- samples/templates/aca/fhir-cosmos-aca.json | 370 +++++++++++++++++++++ 4 files changed, 850 insertions(+), 28 deletions(-) create mode 100644 build/jobs/provision-deploy-cosmos-aca.yml create mode 100644 build/jobs/run-cosmos-tests-aca.yml create mode 100644 samples/templates/aca/fhir-cosmos-aca.json diff --git a/build/jobs/provision-deploy-cosmos-aca.yml b/build/jobs/provision-deploy-cosmos-aca.yml new file mode 100644 index 0000000000..ab1a66cda4 --- /dev/null +++ b/build/jobs/provision-deploy-cosmos-aca.yml @@ -0,0 +1,307 @@ +parameters: +- name: version + type: string +- name: webAppName + type: string +- name: acaEnvironmentName + type: string +- name: subscription + type: string +- name: resourceGroup + type: string +- name: testEnvironmentUrl + type: string +- name: imageTag + type: string +- name: reindexEnabled + type: boolean + default: true +- name: keyVaultName + type: string +- name: containerCpu + type: string + default: '1.0' +- name: containerMemory + type: string + default: '2Gi' + +jobs: +- job: provisionEnvironment + pool: + name: '$(SharedLinuxPool)' + vmImage: '$(LinuxVmImage)' + steps: + - task: AzureKeyVault@1 + displayName: 'Azure Key Vault: resolute-oss-tenant-info' + inputs: + azureSubscription: ${{ parameters.subscription }} + KeyVaultName: 'resolute-oss-tenant-info' + + - task: AzurePowerShell@5 + name: SetAcaOutputs + displayName: 'Deploy ACA for Cosmos and emit test URL' + retryCountOnTaskFailure: 1 + inputs: + azureSubscription: ${{ parameters.subscription }} + azurePowerShellVersion: latestVersion + pwsh: true + ScriptType: inlineScript + Inline: | + Add-Type -AssemblyName System.Web + + # Import retry helper for transient error handling + . $(System.DefaultWorkingDirectory)/release/scripts/PowerShell/Invoke-WithRetry.ps1 + + $deployPath = "$(System.DefaultWorkingDirectory)/test/Configuration" + + $testConfig = ConvertFrom-Json (Get-Content -Raw "$deployPath/testconfiguration.json") + $flattenedTestConfig = $(System.DefaultWorkingDirectory)/release/scripts/PowerShell/ConvertTo-FlattenedConfigurationHashtable.ps1 -InputObject $testConfig + + $additionalProperties = $flattenedTestConfig + $additionalProperties["TaskHosting__PollingFrequencyInSeconds"] = 1 + $additionalProperties["FhirServer__CoreFeatures__SearchParameterCacheRefreshIntervalSeconds"] = 2 + $additionalProperties["ASPNETCORE_FORWARDEDHEADERS_ENABLED"] = "true" + + $staticEnvNames = @( + "ASPNETCORE_FORWARDEDHEADERS_ENABLED", + "KeyVault__Endpoint", + "FhirServer__Security__Enabled", + "FhirServer__Security__EnableAadSmartOnFhirProxy", + "FhirServer__Security__Authentication__Authority", + "FhirServer__Security__Authentication__Audience", + "DataStore", + "CosmosDb__UseManagedIdentity", + "CosmosDb__ContinuationTokenSizeLimitInKb", + "CosmosDb__UseQueueClientJobs", + "TaskHosting__Enabled", + "TaskHosting__PollingFrequencyInSeconds", + "TaskHosting__MaxRunningTaskCount", + "FhirServer__CoreFeatures__SearchParameterCacheRefreshIntervalSeconds", + "FhirServer__Operations__Export__Enabled", + "FhirServer__Operations__Export__StorageAccountUri", + "FhirServer__Operations__Import__Enabled", + "FhirServer__Operations__IntegrationDataStore__StorageAccountUri", + "FhirServer__Operations__ConvertData__Enabled", + "FhirServer__Operations__ConvertData__ContainerRegistryServers__0", + "FhirServer__Operations__Reindex__Enabled" + ) + + $additionalEnvVars = @() + foreach ($entry in $additionalProperties.GetEnumerator()) { + if ($staticEnvNames -contains $entry.Key) { + continue + } + + if ($null -eq $entry.Value) { + continue + } + + $value = if ($entry.Value -is [bool]) { $entry.Value.ToString().ToLowerInvariant() } else { [string]$entry.Value } + if ([string]::IsNullOrWhiteSpace($value)) { + continue + } + + $additionalEnvVars += @{ + name = [string]$entry.Key + value = $value + } + } + + $webAppName = "${{ parameters.webAppName }}".ToLowerInvariant() + $acaEnvironmentName = "${{ parameters.acaEnvironmentName }}".ToLowerInvariant() + + if ($acaEnvironmentName.Length -gt 32) { + $acaEnvironmentName = $acaEnvironmentName.Substring(0, 32).TrimEnd('-') + } + + if ($acaEnvironmentName.Length -lt 2) { + $acaEnvironmentName = "ca-$acaEnvironmentName" + } + + # Cosmos DB account uses the same name as the container app (matching App Service convention) + $cosmosDbAccountName = $webAppName + $resourceGroupName = "${{ parameters.resourceGroup }}" + + # Ensure Cosmos DB account exists + $existingAccount = Get-AzCosmosDBAccount -ResourceGroupName $resourceGroupName -Name $cosmosDbAccountName -ErrorAction SilentlyContinue + if ($null -eq $existingAccount) { + Write-Host "Cosmos DB account '$cosmosDbAccountName' does not exist. Creating..." + Invoke-WithRetry -OperationName "Create Cosmos DB Account" -ScriptBlock { + New-AzCosmosDBAccount ` + -ResourceGroupName $resourceGroupName ` + -Name $cosmosDbAccountName ` + -Location (Get-AzResourceGroup -Name $resourceGroupName).Location ` + -ApiKind "Sql" ` + -DefaultConsistencyLevel "Strong" ` + -DisableKeyBasedMetadataWriteAccess $false ` + -ErrorAction Stop + } + Write-Host "Cosmos DB account '$cosmosDbAccountName' created successfully." + } else { + Write-Host "Cosmos DB account '$cosmosDbAccountName' already exists." + } + + # Ensure Cosmos DB database exists + $cosmosDbDatabaseName = "health" + $existingDb = Get-AzCosmosDBSqlDatabase -ResourceGroupName $resourceGroupName -AccountName $cosmosDbAccountName -Name $cosmosDbDatabaseName -ErrorAction SilentlyContinue + if ($null -eq $existingDb) { + Write-Host "Cosmos DB database '$cosmosDbDatabaseName' does not exist. Creating with autoscale 10000 RU..." + Invoke-WithRetry -OperationName "Create Cosmos DB Database" -ScriptBlock { + New-AzCosmosDBSqlDatabase ` + -ResourceGroupName $resourceGroupName ` + -AccountName $cosmosDbAccountName ` + -Name $cosmosDbDatabaseName ` + -AutoscaleMaxThroughput 10000 ` + -ErrorAction Stop + } + Write-Host "Cosmos DB database '$cosmosDbDatabaseName' created successfully." + } else { + Write-Host "Cosmos DB database '$cosmosDbDatabaseName' already exists." + } + + # Add DevOps MI permission to Cosmos database (Data Contributor role) + Write-Host "Adding DevOps MI permission to Cosmos database..." + $account = Get-AzContext + $principalId = Invoke-WithRetry -OperationName "Get Service Principal" -ScriptBlock { + (Get-AzADServicePrincipal -ApplicationId $account.Account.Id -ErrorAction Stop).Id + } + + Invoke-WithRetry -OperationName "CosmosDB Role Assignment" -ScriptBlock { + New-AzCosmosDBSqlRoleAssignment ` + -AccountName $cosmosDbAccountName ` + -ResourceGroupName $resourceGroupName ` + -Scope "/" ` + -PrincipalId $principalId ` + -RoleDefinitionId "00000000-0000-0000-0000-000000000002" ` + -ErrorAction Stop + } + Write-Host "DevOps MI Cosmos DB role assignment completed." + + $templateParameters = @{ + containerAppName = $webAppName + containerAppsEnvironmentName = $acaEnvironmentName + keyVaultName = "${{ parameters.keyVaultName }}".ToLowerInvariant() + fhirVersion = "${{ parameters.version }}" + registryName = '$(azureContainerRegistry)' + imageTag = "${{ parameters.imageTag }}" + cosmosDbAccountName = $cosmosDbAccountName + securityAuthenticationAuthority = "https://sts.windows.net/$(tenant-id-guid)" + securityAuthenticationAudience = "${{ parameters.testEnvironmentUrl }}" + enableExport = $true + enableImport = $true + enableConvertData = $true + enableReindex = if ("${{ parameters.reindexEnabled }}" -eq "true") { $true } else { $false } + minReplicas = 3 + maxReplicas = 6 + containerCpu = "${{ parameters.containerCpu }}" + containerMemory = "${{ parameters.containerMemory }}" + acrPullUserAssignedManagedIdentityResourceId = '/subscriptions/a1766500-6fd5-4f5c-8515-607798271014/resourceGroups/HealthPlatformRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-uami' + additionalEnvVars = $additionalEnvVars + } + + $deploymentName = "$webAppName-aca" + + Write-Host "Provisioning ACA resources" + Write-Host "ResourceGroupName: $resourceGroupName" + Write-Host "ContainerAppName: $webAppName" + Write-Host "ContainerAppsEnvironmentName: $acaEnvironmentName" + Write-Host "CosmosDbAccountName: $cosmosDbAccountName" + + $deploymentResult = Invoke-WithRetry -OperationName "ACA Deployment" -ScriptBlock { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/fhir-cosmos-aca.json -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop + } + + # Assign Cosmos DB Data Contributor role to the container app's system-assigned identity + $containerAppPrincipalId = (Get-AzResource -ResourceGroupName $resourceGroupName -ResourceType "Microsoft.App/containerApps" -Name $webAppName -ApiVersion "2023-05-01" -ExpandProperties).Identity.PrincipalId + if (-not [string]::IsNullOrWhiteSpace($containerAppPrincipalId)) { + Write-Host "Assigning Cosmos DB Data Contributor role to container app identity ($containerAppPrincipalId)..." + Invoke-WithRetry -OperationName "CosmosDB Role Assignment for Container App" -ScriptBlock { + New-AzCosmosDBSqlRoleAssignment ` + -AccountName $cosmosDbAccountName ` + -ResourceGroupName $resourceGroupName ` + -Scope "/" ` + -PrincipalId $containerAppPrincipalId ` + -RoleDefinitionId "00000000-0000-0000-0000-000000000002" ` + -ErrorAction Stop + } + Write-Host "Container app Cosmos DB role assignment completed." + } else { + Write-Warning "Could not retrieve container app system-assigned identity principal ID." + } + + # Associate Cosmos DB with Network Security Perimeter + $nspName = "nsp-$resourceGroupName" + $associationName = "cosmosdb-$webAppName-association" + + Write-Host "Associating CosmosDB with Network Security Perimeter: $nspName" + $nspTemplateParameters = @{ + 'nspName' = $nspName + 'resourceName' = $cosmosDbAccountName + 'resourceType' = 'Microsoft.DocumentDB/databaseAccounts' + 'associationName' = $associationName + 'accessMode' = 'Learning' + 'nspProfile' = 'default' + } + + Invoke-WithRetry -OperationName "NSP Association Deployment" -ScriptBlock { + New-AzResourceGroupDeployment ` + -ResourceGroupName $resourceGroupName ` + -Name "$webAppName-cosmosdb-nsp-association" ` + -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/nsp-resource-association.json ` + -TemplateParameterObject $nspTemplateParameters ` + -Verbose ` + -ErrorAction Stop + } + Write-Host "Successfully associated CosmosDB with NSP." + + $containerAppUrl = $deploymentResult.Outputs.containerAppUrl.Value + if ([string]::IsNullOrWhiteSpace($containerAppUrl)) { + throw "ACA deployment output did not contain containerAppUrl" + } + + $containerAppUrl = $containerAppUrl.TrimEnd('/') + $healthCheckUrl = "$containerAppUrl/health/check" + + $timeoutMinutes = 7 + $consecutiveSuccessRequired = 5 + $consecutiveSuccessCount = 0 + $startTime = Get-Date + $timeoutTime = $startTime.AddMinutes($timeoutMinutes) + + Write-Host "Starting health check for $healthCheckUrl" + Write-Host "Timeout: $timeoutMinutes minutes, consecutive successes required: $consecutiveSuccessRequired" + + Do { + Start-Sleep -s 5 + Write-Host "Checking: $healthCheckUrl (consecutive successes: $consecutiveSuccessCount/$consecutiveSuccessRequired)" + try { + $response = Invoke-WebRequest -Uri $healthCheckUrl -UseBasicParsing + if ($response.StatusCode -eq 200) { + $consecutiveSuccessCount++ + Write-Host "Success ($consecutiveSuccessCount/$consecutiveSuccessRequired)" + } + else { + Write-Host "Non-200 response ($($response.StatusCode)), resetting count" + $consecutiveSuccessCount = 0 + } + } + catch { + Write-Host $_.Exception.Message + $consecutiveSuccessCount = 0 + } + finally { + $Error.Clear() + } + + if ((Get-Date) -gt $timeoutTime) { + $elapsed = [math]::Round(((Get-Date) - $startTime).TotalMinutes, 2) + Write-Error "Health check timed out after $elapsed minutes ($consecutiveSuccessCount/$consecutiveSuccessRequired consecutive successes)" + throw "ACA health check timed out" + } + } While ($consecutiveSuccessCount -lt $consecutiveSuccessRequired) + + $elapsed = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 2) + Write-Host "ACA health check passed with $consecutiveSuccessRequired consecutive successes after $elapsed seconds" + + Write-Host "##vso[task.setvariable variable=TestEnvironmentUrl_${{ parameters.version }};isOutput=true]$containerAppUrl" + Write-Host "##vso[task.setvariable variable=TestEnvironmentUrl;isOutput=true]$containerAppUrl" diff --git a/build/jobs/run-cosmos-tests-aca.yml b/build/jobs/run-cosmos-tests-aca.yml new file mode 100644 index 0000000000..004cf20a71 --- /dev/null +++ b/build/jobs/run-cosmos-tests-aca.yml @@ -0,0 +1,137 @@ +parameters: +- name: version + type: string +- name: keyVaultName + type: string +- name: containerAppName + type: string +jobs: + +- job: "CosmosIntegrationTests" + timeoutInMinutes: 75 + pool: + name: '$(SharedLinuxPool)' + vmImage: '$(LinuxVmImage)' + steps: + - checkout: self + fetchDepth: 1 + fetchTags: false + path: source + + - template: integration-setup.yml + + - template: integration-tests-extract.yml + parameters: + version: ${{ parameters.version }} + + - task: AzureKeyVault@1 + displayName: 'Azure Key Vault: ${{ parameters.keyVaultName }}' + inputs: + azureSubscription: $(ConnectedServiceName) + KeyVaultName: '${{ parameters.keyVaultName }}' + + - task: AzurePowerShell@5 + displayName: 'Set Workload Identity Variables' + inputs: + azureSubscription: $(ConnectedServiceName) + azurePowerShellVersion: latestVersion + pwsh: true + ScriptType: inlineScript + Inline: | + Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_CLIENT_ID]$env:AZURESUBSCRIPTION_CLIENT_ID" + Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_TENANT_ID]$env:AZURESUBSCRIPTION_TENANT_ID" + Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_SERVICE_CONNECTION_ID]$env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" + + $containerAppName = '${{ parameters.containerAppName }}' + $containerApp = Get-AzResource -ResourceGroupName $(UniqueResourceGroupName) -ResourceType "Microsoft.App/containerApps" -Name $containerAppName -ApiVersion "2023-05-01" -ExpandProperties + if ($null -eq $containerApp) { + throw "Container App '$containerAppName' was not found in resource group '$(UniqueResourceGroupName)'" + } + + $containerAppData = $containerApp | ConvertTo-Json -Depth 100 | ConvertFrom-Json + $envSettings = $containerAppData.Properties.template.containers[0].env + + $dataStoreResourceId = ($envSettings | Where-Object { $_.name -eq "FhirServer__ResourceManager__DataStoreResourceId" } | Select-Object -First 1).value + Write-Host "$dataStoreResourceId" + Write-Host "##vso[task.setvariable variable=DataStoreResourceId]$($dataStoreResourceId)" + + - task: DotNetCoreCLI@2 + displayName: 'Build Integration Test Projects' + inputs: + command: build + projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' + arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)' + + - task: DotNetCoreCLI@2 + displayName: 'Run Cosmos Integration Tests with coverage' + inputs: + command: test + projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' + arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter "FullyQualifiedName!~SqlServer" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' + testRunTitle: '${{ parameters.version }} Cosmos Integration Tests' + env: + 'CosmosDb__Host': $(CosmosDb--Host) + 'FhirServer__ResourceManager__DataStoreResourceId': '$(DataStoreResourceId)' + 'CosmosDb__UseManagedIdentity': true + platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage' + 'AZURESUBSCRIPTION_CLIENT_ID': '$(AZURESUBSCRIPTION_CLIENT_ID)' + 'AZURESUBSCRIPTION_TENANT_ID': '$(AZURESUBSCRIPTION_TENANT_ID)' + 'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)' + 'SYSTEM_ACCESSTOKEN': $(System.AccessToken) + + - task: PublishTestResults@2 + displayName: 'Publish integration test results' + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx' + mergeTestResults: true + testRunTitle: '${{ parameters.version }} Cosmos Integration Tests' + # TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed + failTaskOnFailedTests: false + condition: succeededOrFailed() + + - task: reportgenerator@5 + displayName: 'Aggregate Cosmos integration test coverage' + condition: succeededOrFailed() + inputs: + reports: '$(Agent.TempDirectory)/coverage/**/*.cobertura.xml' + reporttypes: 'Cobertura' + targetdir: '$(Agent.TempDirectory)/coverage-aggregated' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Cosmos integration test coverage' + inputs: + pathToPublish: '$(Agent.TempDirectory)/coverage-aggregated' + artifactName: 'Coverage_IntegrationTests_Cosmos_${{ parameters.version }}' + artifactType: 'container' + +- job: 'cosmosE2eTests' + timeoutInMinutes: 75 + dependsOn: [] + pool: + name: '$(SharedLinuxPool)' + vmImage: '$(LinuxVmImage)' + steps: + - template: e2e-setup.yml + - template: e2e-tests-aca.yml + parameters: + version: ${{ parameters.version }} + containerAppName: '${{ parameters.containerAppName }}' + appServiceType: 'CosmosDb' + categoryFilter: 'Category!=ExportLongRunning&Category!=IndexAndReindex' + +- job: 'cosmosE2eTests_Reindex' + timeoutInMinutes: 75 + dependsOn: [] + pool: + name: '$(SharedLinuxPool)' + vmImage: '$(LinuxVmImage)' + steps: + - template: e2e-setup.yml + - template: e2e-tests-aca.yml + parameters: + version: ${{ parameters.version }} + containerAppName: '${{ parameters.containerAppName }}' + appServiceType: 'CosmosDb' + categoryFilter: 'Category=IndexAndReindex' + testRunTitleSuffix: ' Reindex' diff --git a/build/pr-pipeline.yml b/build/pr-pipeline.yml index 1589fb3366..d2c49d0890 100644 --- a/build/pr-pipeline.yml +++ b/build/pr-pipeline.yml @@ -268,24 +268,24 @@ stages: - stage: deployStu3 displayName: 'Deploy STU3 CosmosDB Site' - condition: false dependsOn: - DockerBuild - setupEnvironment - createNsp jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-cosmos-aca.yml parameters: version: Stu3 webAppName: $(DeploymentEnvironmentName) + acaEnvironmentName: $(AcaEnvironmentName) keyVaultName: $(KeyVaultBaseName) - appServicePlanName: '$(appServicePlanName)-cosmos' - appServicePlanResourceGroup: $(appServicePlanResourceGroup) subscription: $(ConnectedServiceName) resourceGroup: $(UniqueResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true + containerCpu: '1.0' + containerMemory: '2Gi' - stage: deployStu3Sql displayName: 'Deploy STU3 SQL Site' @@ -313,24 +313,24 @@ stages: - stage: deployR4 displayName: 'Deploy R4 CosmosDB Site' - condition: false dependsOn: - DockerBuild - setupEnvironment - createNsp jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-cosmos-aca.yml parameters: version: R4 webAppName: $(DeploymentEnvironmentNameR4) + acaEnvironmentName: $(AcaEnvironmentName) keyVaultName: $(KeyVaultNameR4) - appServicePlanName: '$(appServicePlanName)-cosmos' - appServicePlanResourceGroup: $(appServicePlanResourceGroup) subscription: $(ConnectedServiceName) resourceGroup: $(UniqueResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true + containerCpu: '1.0' + containerMemory: '2Gi' - stage: deployR4Sql displayName: 'Deploy R4 SQL Site' @@ -358,24 +358,24 @@ stages: - stage: deployR4B displayName: 'Deploy R4B CosmosDB Site' - condition: false dependsOn: - DockerBuild - setupEnvironment - createNsp jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-cosmos-aca.yml parameters: version: R4B webAppName: $(DeploymentEnvironmentNameR4B) + acaEnvironmentName: $(AcaEnvironmentName) keyVaultName: $(KeyVaultNameR4B) - appServicePlanName: '$(appServicePlanName)-cosmos' - appServicePlanResourceGroup: $(appServicePlanResourceGroup) subscription: $(ConnectedServiceName) resourceGroup: $(UniqueResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true + containerCpu: '1.0' + containerMemory: '2Gi' - stage: deployR4BSql displayName: 'Deploy R4B SQL Site' @@ -403,24 +403,24 @@ stages: - stage: deployR5 displayName: 'Deploy R5 CosmosDB Site' - condition: false dependsOn: - DockerBuild - setupEnvironment - createNsp jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-cosmos-aca.yml parameters: version: R5 webAppName: $(DeploymentEnvironmentNameR5) + acaEnvironmentName: $(AcaEnvironmentName) keyVaultName: $(KeyVaultNameR5) - appServicePlanName: '$(appServicePlanName)-cosmos' - appServicePlanResourceGroup: $(appServicePlanResourceGroup) subscription: $(ConnectedServiceName) resourceGroup: $(UniqueResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true + containerCpu: '1.0' + containerMemory: '2Gi' - stage: deployR5Sql displayName: 'Deploy R5 SQL Site' @@ -448,17 +448,19 @@ stages: - stage: testStu3Cosmos displayName: 'Run Stu3 Cosmos Tests' - condition: false dependsOn: - BuildArtifacts - setupEnvironment - deployStu3 + variables: + TestEnvironmentUrl_Stu3: $[stageDependencies.deployStu3.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl_Stu3']] + TestEnvironmentUrl: $[stageDependencies.deployStu3.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl']] jobs: - - template: ./jobs/run-cosmos-tests.yml + - template: ./jobs/run-cosmos-tests-aca.yml parameters: version: Stu3 keyVaultName: $(KeyVaultBaseName) - appServiceName: $(DeploymentEnvironmentName) + containerAppName: $(DeploymentEnvironmentName) - stage: testStu3Sql displayName: 'Run Stu3 SQL Tests' @@ -479,17 +481,19 @@ stages: - stage: testR4Cosmos displayName: 'Run R4 Cosmos Tests' - condition: false dependsOn: - BuildArtifacts - setupEnvironment - deployR4 + variables: + TestEnvironmentUrl_R4: $[stageDependencies.deployR4.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl_R4']] + TestEnvironmentUrl: $[stageDependencies.deployR4.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl']] jobs: - - template: ./jobs/run-cosmos-tests.yml + - template: ./jobs/run-cosmos-tests-aca.yml parameters: version: R4 keyVaultName: $(KeyVaultNameR4) - appServiceName: $(DeploymentEnvironmentNameR4) + containerAppName: $(DeploymentEnvironmentNameR4) - stage: testR4Sql displayName: 'Run R4 SQL Tests' @@ -510,17 +514,19 @@ stages: - stage: testR4BCosmos displayName: 'Run R4B Cosmos Tests' - condition: false dependsOn: - BuildArtifacts - setupEnvironment - deployR4B + variables: + TestEnvironmentUrl_R4B: $[stageDependencies.deployR4B.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl_R4B']] + TestEnvironmentUrl: $[stageDependencies.deployR4B.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl']] jobs: - - template: ./jobs/run-cosmos-tests.yml + - template: ./jobs/run-cosmos-tests-aca.yml parameters: version: R4B keyVaultName: $(KeyVaultNameR4B) - appServiceName: $(DeploymentEnvironmentNameR4B) + containerAppName: $(DeploymentEnvironmentNameR4B) - stage: testR4BSql displayName: 'Run R4B SQL Tests' @@ -541,17 +547,19 @@ stages: - stage: testR5Cosmos displayName: 'Run R5 Cosmos Tests' - condition: false dependsOn: - BuildArtifacts - setupEnvironment - deployR5 + variables: + TestEnvironmentUrl_R5: $[stageDependencies.deployR5.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl_R5']] + TestEnvironmentUrl: $[stageDependencies.deployR5.provisionEnvironment.outputs['SetAcaOutputs.TestEnvironmentUrl']] jobs: - - template: ./jobs/run-cosmos-tests.yml + - template: ./jobs/run-cosmos-tests-aca.yml parameters: version: R5 keyVaultName: $(KeyVaultNameR5) - appServiceName: $(DeploymentEnvironmentNameR5) + containerAppName: $(DeploymentEnvironmentNameR5) - stage: testR5Sql displayName: 'Run R5 SQL Tests' diff --git a/samples/templates/aca/fhir-cosmos-aca.json b/samples/templates/aca/fhir-cosmos-aca.json new file mode 100644 index 0000000000..44bd02ef29 --- /dev/null +++ b/samples/templates/aca/fhir-cosmos-aca.json @@ -0,0 +1,370 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "containerAppName": { + "type": "string", + "metadata": { + "description": "Name of the Azure Container App hosting FHIR." + } + }, + "containerAppsEnvironmentName": { + "type": "string", + "metadata": { + "description": "Name of the Container Apps Environment." + } + }, + "keyVaultName": { + "type": "string", + "maxLength": 24, + "metadata": { + "description": "Name of the Key Vault used by FHIR server." + } + }, + "fhirVersion": { + "type": "string", + "defaultValue": "R4", + "allowedValues": [ + "Stu3", + "R4", + "R4B", + "R5" + ] + }, + "registryName": { + "type": "string", + "metadata": { + "description": "Container registry server host name." + } + }, + "imageTag": { + "type": "string", + "defaultValue": "latest" + }, + "cosmosDbAccountName": { + "type": "string", + "metadata": { + "description": "Name of the existing Cosmos DB account." + } + }, + "securityAuthenticationAuthority": { + "type": "string", + "defaultValue": "" + }, + "securityAuthenticationAudience": { + "type": "string", + "defaultValue": "" + }, + "acrPullUserAssignedManagedIdentityResourceId": { + "type": "string", + "metadata": { + "description": "Resource ID of UAMI used to pull from ACR." + } + }, + "enableExport": { + "type": "bool", + "defaultValue": true + }, + "enableImport": { + "type": "bool", + "defaultValue": true + }, + "enableConvertData": { + "type": "bool", + "defaultValue": true + }, + "enableReindex": { + "type": "bool", + "defaultValue": true + }, + "minReplicas": { + "type": "int", + "defaultValue": 1 + }, + "maxReplicas": { + "type": "int", + "defaultValue": 3 + }, + "containerCpu": { + "type": "string", + "defaultValue": "1.0" + }, + "containerMemory": { + "type": "string", + "defaultValue": "2Gi" + }, + "additionalEnvVars": { + "type": "array", + "defaultValue": [] + } + }, + "variables": { + "isMAG": "[or(contains(resourceGroup().location,'usgov'),contains(resourceGroup().location,'usdod'))]", + "containerAppName": "[toLower(parameters('containerAppName'))]", + "containerAppsEnvironmentName": "[toLower(parameters('containerAppsEnvironmentName'))]", + "keyVaultName": "[toLower(parameters('keyVaultName'))]", + "cosmosDbAccountName": "[toLower(parameters('cosmosDbAccountName'))]", + "azureContainerRegistryUri": "[if(variables('isMAG'), '.azurecr.us', '.azurecr.io')]", + "imageRepositoryName": "[if(contains(parameters('registryName'),'mcr.'), concat(toLower(parameters('fhirVersion')), '-fhir-server'), concat(toLower(parameters('fhirVersion')), '_fhir-server'))]", + "containerImage": "[concat(parameters('registryName'), '/', variables('imageRepositoryName'), ':', parameters('imageTag'))]", + "blobStorageUri": "[if(variables('isMAG'), '.blob.core.usgovcloudapi.net', '.blob.core.windows.net')]", + "storageAccountName": "[concat(substring(replace(variables('containerAppName'), '-', ''), 0, min(11, length(replace(variables('containerAppName'), '-', '')))), uniquestring(resourceGroup().id, variables('containerAppName')))]", + "storageAccountUri": "[concat('https://', variables('storageAccountName'), variables('blobStorageUri'))]", + "keyVaultEndpoint": "[if(variables('isMAG'), concat('https://', variables('keyVaultName'), '.vault.usgovcloudapi.net/'), concat('https://', variables('keyVaultName'), '.vault.azure.net/'))]", + "enableIntegrationStore": "[or(parameters('enableExport'), parameters('enableImport'))]", + "storageBlobDataContributerRoleId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe')]", + "cosmosDataContributorRoleId": "00000000-0000-0000-0000-000000000002", + "staticEnvVars": [ + { + "name": "ASPNETCORE_FORWARDEDHEADERS_ENABLED", + "value": "true" + }, + { + "name": "KeyVault__Endpoint", + "value": "[variables('keyVaultEndpoint')]" + }, + { + "name": "FhirServer__Security__Enabled", + "value": "true" + }, + { + "name": "FhirServer__Security__EnableAadSmartOnFhirProxy", + "value": "true" + }, + { + "name": "FhirServer__Security__Authentication__Authority", + "value": "[parameters('securityAuthenticationAuthority')]" + }, + { + "name": "FhirServer__Security__Authentication__Audience", + "value": "[parameters('securityAuthenticationAudience')]" + }, + { + "name": "DataStore", + "value": "CosmosDb" + }, + { + "name": "CosmosDb__UseManagedIdentity", + "value": "true" + }, + { + "name": "CosmosDb__ContinuationTokenSizeLimitInKb", + "value": "1" + }, + { + "name": "CosmosDb__UseQueueClientJobs", + "value": "true" + }, + { + "name": "TaskHosting__Enabled", + "value": "true" + }, + { + "name": "TaskHosting__PollingFrequencyInSeconds", + "value": "1" + }, + { + "name": "TaskHosting__MaxRunningTaskCount", + "value": "2" + }, + { + "name": "FhirServer__CoreFeatures__SearchParameterCacheRefreshIntervalSeconds", + "value": "2" + }, + { + "name": "FhirServer__Operations__Export__Enabled", + "value": "[if(parameters('enableExport'), 'true', 'false')]" + }, + { + "name": "FhirServer__Operations__Export__StorageAccountUri", + "value": "[if(parameters('enableExport'), variables('storageAccountUri'), 'null')]" + }, + { + "name": "FhirServer__Operations__Import__Enabled", + "value": "[if(parameters('enableImport'), 'true', 'false')]" + }, + { + "name": "FhirServer__Operations__IntegrationDataStore__StorageAccountUri", + "value": "[if(parameters('enableImport'), variables('storageAccountUri'), 'null')]" + }, + { + "name": "FhirServer__Operations__ConvertData__Enabled", + "value": "[if(parameters('enableConvertData'), 'true', 'false')]" + }, + { + "name": "FhirServer__Operations__ConvertData__ContainerRegistryServers__0", + "value": "[if(parameters('enableConvertData'), parameters('registryName'), 'null')]" + }, + { + "name": "FhirServer__Operations__Reindex__Enabled", + "value": "[if(parameters('enableReindex'), 'true', 'false')]" + } + ] + }, + "resources": [ + { + "type": "Microsoft.App/containerApps", + "apiVersion": "2023-05-01", + "name": "[variables('containerAppName')]", + "location": "[resourceGroup().location]", + "identity": { + "type": "SystemAssigned,UserAssigned", + "userAssignedIdentities": "[json(concat('{\"', parameters('acrPullUserAssignedManagedIdentityResourceId'), '\":{}}'))]" + }, + "properties": { + "managedEnvironmentId": "[resourceId('Microsoft.App/managedEnvironments', variables('containerAppsEnvironmentName'))]", + "configuration": { + "activeRevisionsMode": "Single", + "ingress": { + "external": true, + "targetPort": 8080, + "allowInsecure": false, + "transport": "auto" + }, + "registries": [ + { + "server": "[parameters('registryName')]", + "identity": "[parameters('acrPullUserAssignedManagedIdentityResourceId')]" + } + ] + }, + "template": { + "containers": [ + { + "name": "[variables('containerAppName')]", + "image": "[variables('containerImage')]", + "resources": { + "cpu": "[json(parameters('containerCpu'))]", + "memory": "[parameters('containerMemory')]" + }, + "env": "[concat(variables('staticEnvVars'), parameters('additionalEnvVars'))]", + "probes": [ + { + "type": "Liveness", + "httpGet": { + "path": "/health/check", + "port": 8080 + }, + "initialDelaySeconds": 20, + "periodSeconds": 10, + "timeoutSeconds": 5, + "failureThreshold": 6 + }, + { + "type": "Readiness", + "httpGet": { + "path": "/health/check", + "port": 8080 + }, + "initialDelaySeconds": 20, + "periodSeconds": 10, + "timeoutSeconds": 5, + "failureThreshold": 6 + } + ] + } + ], + "scale": { + "minReplicas": "[parameters('minReplicas')]", + "maxReplicas": "[parameters('maxReplicas')]" + } + } + }, + "dependsOn": [] + }, + { + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2022-07-01", + "name": "[variables('keyVaultName')]", + "location": "[resourceGroup().location]", + "properties": { + "sku": { + "family": "A", + "name": "Standard" + }, + "tenantId": "[subscription().tenantId]", + "accessPolicies": [], + "enableRbacAuthorization": true, + "enabledForDeployment": false + } + }, + { + "type": "Microsoft.KeyVault/vaults/providers/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[concat(variables('keyVaultName'), '/Microsoft.Authorization/', guid(uniqueString('KeyVaultSecretsOfficer', variables('containerAppName'), variables('keyVaultName'))))]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", + "[resourceId('Microsoft.App/containerApps', variables('containerAppName'))]" + ], + "properties": { + "roleDefinitionId": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/', 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7')]", + "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').identity.principalId]" + } + }, + { + "condition": "[variables('enableIntegrationStore')]", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2019-06-01", + "name": "[variables('storageAccountName')]", + "location": "[resourceGroup().location]", + "kind": "Storage", + "sku": { + "name": "Standard_LRS" + }, + "properties": { + "supportsHttpsTrafficOnly": true, + "allowBlobPublicAccess": false, + "allowSharedKeyAccess": false + } + }, + { + "condition": "[variables('enableIntegrationStore')]", + "type": "Microsoft.Storage/storageAccounts/providers/roleAssignments", + "apiVersion": "2018-09-01-preview", + "name": "[concat(variables('storageAccountName'), '/Microsoft.Authorization/', guid(uniqueString(variables('storageAccountName'), variables('containerAppName'))))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]", + "[resourceId('Microsoft.App/containerApps', variables('containerAppName'))]" + ], + "properties": { + "roleDefinitionId": "[variables('storageBlobDataContributerRoleId')]", + "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2015-06-01", + "name": "[concat(variables('keyVaultName'), '/CosmosDb--Host')]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" + ], + "properties": { + "contentType": "text/plain", + "value": "[reference(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('cosmosDbAccountName')), '2021-06-15').documentEndpoint]" + } + } + ], + "outputs": { + "containerAppName": { + "type": "string", + "value": "[variables('containerAppName')]" + }, + "containerAppFqdn": { + "type": "string", + "value": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').properties.configuration.ingress.fqdn]" + }, + "containerAppUrl": { + "type": "string", + "value": "[concat('https://', reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').properties.configuration.ingress.fqdn)]" + }, + "exportStorageUri": { + "type": "string", + "value": "[if(parameters('enableExport'), variables('storageAccountUri'), 'null')]" + }, + "integrationStorageUri": { + "type": "string", + "value": "[if(parameters('enableImport'), variables('storageAccountUri'), 'null')]" + } + } +} From f6e4600f5f27f920b8607d74fd8a24e72411a57e Mon Sep 17 00:00:00 2001 From: John Estrada Date: Fri, 27 Mar 2026 10:19:17 -0500 Subject: [PATCH 18/38] fixing invalid switch --- build/jobs/provision-deploy-cosmos-aca.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/build/jobs/provision-deploy-cosmos-aca.yml b/build/jobs/provision-deploy-cosmos-aca.yml index ab1a66cda4..d86cc13a07 100644 --- a/build/jobs/provision-deploy-cosmos-aca.yml +++ b/build/jobs/provision-deploy-cosmos-aca.yml @@ -133,7 +133,6 @@ jobs: -Location (Get-AzResourceGroup -Name $resourceGroupName).Location ` -ApiKind "Sql" ` -DefaultConsistencyLevel "Strong" ` - -DisableKeyBasedMetadataWriteAccess $false ` -ErrorAction Stop } Write-Host "Cosmos DB account '$cosmosDbAccountName' created successfully." From e26d01231bb487309ded3a2bf689bc49c5c243e6 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Fri, 27 Mar 2026 11:39:38 -0500 Subject: [PATCH 19/38] add missing cosmos config --- samples/templates/aca/fhir-cosmos-aca.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/samples/templates/aca/fhir-cosmos-aca.json b/samples/templates/aca/fhir-cosmos-aca.json index 44bd02ef29..e1a1aec74f 100644 --- a/samples/templates/aca/fhir-cosmos-aca.json +++ b/samples/templates/aca/fhir-cosmos-aca.json @@ -198,6 +198,10 @@ { "name": "FhirServer__Operations__Reindex__Enabled", "value": "[if(parameters('enableReindex'), 'true', 'false')]" + }, + { + "name": "FhirServer__ResourceManager__DataStoreResourceId", + "value": "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('cosmosDbAccountName'))]" } ] }, From 3619a1fd8145c157b129f7bd78c1ad678485fbce Mon Sep 17 00:00:00 2001 From: John Estrada Date: Fri, 27 Mar 2026 12:31:07 -0500 Subject: [PATCH 20/38] Add ARM role assignment for DocumentDB Account Contributor in ACA deployment --- build/jobs/provision-deploy-cosmos-aca.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/build/jobs/provision-deploy-cosmos-aca.yml b/build/jobs/provision-deploy-cosmos-aca.yml index d86cc13a07..3289fb8fa4 100644 --- a/build/jobs/provision-deploy-cosmos-aca.yml +++ b/build/jobs/provision-deploy-cosmos-aca.yml @@ -224,6 +224,24 @@ jobs: -ErrorAction Stop } Write-Host "Container app Cosmos DB role assignment completed." + + # Assign DocumentDB Account Contributor ARM role so ResourceManagerCollectionSetup + # can perform control plane operations (read/create databases and collections). + $cosmosDbResourceId = (Get-AzCosmosDBAccount -ResourceGroupName $resourceGroupName -Name $cosmosDbAccountName).Id + Write-Host "Assigning DocumentDB Account Contributor ARM role on Cosmos DB account..." + $existingArmRole = Get-AzRoleAssignment -ObjectId $containerAppPrincipalId -RoleDefinitionName "DocumentDB Account Contributor" -Scope $cosmosDbResourceId -ErrorAction SilentlyContinue + if ($null -eq $existingArmRole) { + Invoke-WithRetry -OperationName "ARM Role Assignment for Container App" -ScriptBlock { + New-AzRoleAssignment ` + -ObjectId $containerAppPrincipalId ` + -RoleDefinitionName "DocumentDB Account Contributor" ` + -Scope $cosmosDbResourceId ` + -ErrorAction Stop + } + Write-Host "DocumentDB Account Contributor ARM role assigned." + } else { + Write-Host "DocumentDB Account Contributor ARM role already assigned." + } } else { Write-Warning "Could not retrieve container app system-assigned identity principal ID." } From d54547614e84c57ba0084bf5ea67d3c4fb2285d4 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Fri, 27 Mar 2026 14:25:14 -0500 Subject: [PATCH 21/38] Add ACA deployment support and refactor related configurations --- build/ci-deploy.yml | 120 +++++++++++++++------ build/ci-pipeline.yml | 56 +++++----- build/ci-variables.yml | 4 +- build/export-pipeline.yml | 8 ++ build/export-variables.yml | 2 +- build/jobs/redeploy-webapp.yml | 42 ++++++-- build/jobs/restart-webapp.yml | 34 ++++-- build/jobs/run-export-tests.yml | 42 ++++++++ build/pr-variables.yml | 1 - build/tasks/container-app-health-check.yml | 58 ++++++++++ 10 files changed, 286 insertions(+), 81 deletions(-) create mode 100644 build/tasks/container-app-health-check.yml diff --git a/build/ci-deploy.yml b/build/ci-deploy.yml index 826e8078ba..89c48c58f4 100644 --- a/build/ci-deploy.yml +++ b/build/ci-deploy.yml @@ -56,6 +56,49 @@ stages: azureSubscription: $(ConnectedServiceName) resourceGroupName: $(ResourceGroupName) +- stage: createAcaEnvironment + displayName: 'Create ACA Managed Environment' + dependsOn: + - provisionEnvironment + jobs: + - job: createAcaEnvironment + displayName: 'Create ACA Managed Environment' + pool: + name: '$(DefaultLinuxPool)' + vmImage: '$(LinuxVmImage)' + steps: + - task: AzurePowerShell@5 + displayName: 'Create ACA Managed Environment' + inputs: + azureSubscription: $(ConnectedServiceName) + azurePowerShellVersion: latestVersion + pwsh: true + ScriptType: inlineScript + Inline: | + $envName = '$(AcaEnvironmentName)'.ToLowerInvariant() + $rgName = '$(ResourceGroupName)' + $location = '$(ResourceGroupRegion)' + + if ($envName.Length -gt 32) { + $envName = $envName.Substring(0, 32).TrimEnd('-') + } + + $existing = Get-AzResource -ResourceGroupName $rgName -ResourceType 'Microsoft.App/managedEnvironments' -Name $envName -ErrorAction SilentlyContinue + if ($null -ne $existing) { + Write-Host "ACA managed environment '$envName' already exists." + return + } + + Write-Host "Creating ACA managed environment '$envName' in '$rgName'..." + New-AzResourceGroupDeployment ` + -Name "aca-env-$envName" ` + -ResourceGroupName $rgName ` + -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/aca-environment.json ` + -environmentName $envName ` + -location $location ` + -Verbose -ErrorAction Stop + Write-Host "ACA managed environment '$envName' created successfully." + - stage: createNsp displayName: 'Create Network Security Perimeter' dependsOn: @@ -113,15 +156,16 @@ stages: - aadTestEnvironment - DockerBuild - createNsp + - createAcaEnvironment jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-cosmos-aca.yml parameters: version: Stu3 webAppName: $(DeploymentEnvironmentName) - appServicePlanName: $(appServicePlanName) - appServicePlanResourceGroup: $(appServicePlanResourceGroup) + acaEnvironmentName: $(AcaEnvironmentName) + keyVaultName: $(KeyVaultBaseName) subscription: $(ConnectedServiceName) - resourceGroup: $(DeploymentEnvironmentName) + resourceGroup: $(ResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true @@ -132,16 +176,17 @@ stages: - aadTestEnvironment - DockerBuild - deploySqlServer + - createAcaEnvironment jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-aca.yml parameters: version: Stu3 sql: true webAppName: $(DeploymentEnvironmentNameSql) - appServicePlanName: $(appServicePlanName) - appServicePlanResourceGroup: $(appServicePlanResourceGroup) + acaEnvironmentName: $(AcaEnvironmentName) + keyVaultName: $(KeyVaultNameSql) subscription: $(ConnectedServiceName) - resourceGroup: $(DeploymentEnvironmentName) + resourceGroup: $(ResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) schemaAutomaticUpdatesEnabled: 'auto' @@ -155,15 +200,16 @@ stages: - aadTestEnvironment - DockerBuild - createNsp + - createAcaEnvironment jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-cosmos-aca.yml parameters: version: R4 webAppName: $(DeploymentEnvironmentNameR4) - appServicePlanName: $(appServicePlanName) - appServicePlanResourceGroup: $(appServicePlanResourceGroup) + acaEnvironmentName: $(AcaEnvironmentName) + keyVaultName: $(KeyVaultNameR4) subscription: $(ConnectedServiceName) - resourceGroup: $(DeploymentEnvironmentName) + resourceGroup: $(ResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true @@ -174,16 +220,17 @@ stages: - aadTestEnvironment - DockerBuild - deploySqlServer + - createAcaEnvironment jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-aca.yml parameters: version: R4 sql: true webAppName: $(DeploymentEnvironmentNameR4Sql) - appServicePlanName: $(appServicePlanName) - appServicePlanResourceGroup: $(appServicePlanResourceGroup) + acaEnvironmentName: $(AcaEnvironmentName) + keyVaultName: $(KeyVaultNameR4Sql) subscription: $(ConnectedServiceName) - resourceGroup: $(DeploymentEnvironmentName) + resourceGroup: $(ResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) schemaAutomaticUpdatesEnabled: 'auto' @@ -197,15 +244,16 @@ stages: - aadTestEnvironment - DockerBuild - createNsp + - createAcaEnvironment jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-cosmos-aca.yml parameters: version: R4B webAppName: $(DeploymentEnvironmentNameR4B) - appServicePlanName: $(appServicePlanName) - appServicePlanResourceGroup: $(appServicePlanResourceGroup) + acaEnvironmentName: $(AcaEnvironmentName) + keyVaultName: $(KeyVaultNameR4B) subscription: $(ConnectedServiceName) - resourceGroup: $(DeploymentEnvironmentName) + resourceGroup: $(ResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true @@ -216,21 +264,23 @@ stages: - aadTestEnvironment - DockerBuild - deploySqlServer + - createAcaEnvironment jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-aca.yml parameters: version: R4B sql: true webAppName: $(DeploymentEnvironmentNameR4BSql) - appServicePlanName: $(appServicePlanName) - appServicePlanResourceGroup: $(appServicePlanResourceGroup) + acaEnvironmentName: $(AcaEnvironmentName) + keyVaultName: $(KeyVaultNameR4BSql) subscription: $(ConnectedServiceName) - resourceGroup: $(DeploymentEnvironmentName) + resourceGroup: $(ResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) - sqlServerName: $(DeploymentEnvironmentName) - sqlComputeTier: 'Hyperscale' imageTag: $(ImageTag) schemaAutomaticUpdatesEnabled: 'auto' + sqlServerName: $(DeploymentEnvironmentName) + sqlComputeTier: 'Hyperscale' + reindexEnabled: true - stage: deployR5 displayName: 'Deploy R5 CosmosDB Site' @@ -238,15 +288,16 @@ stages: - aadTestEnvironment - DockerBuild - createNsp + - createAcaEnvironment jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-cosmos-aca.yml parameters: version: R5 webAppName: $(DeploymentEnvironmentNameR5) - appServicePlanName: $(appServicePlanName) - appServicePlanResourceGroup: $(appServicePlanResourceGroup) + acaEnvironmentName: $(AcaEnvironmentName) + keyVaultName: $(KeyVaultNameR5) subscription: $(ConnectedServiceName) - resourceGroup: $(DeploymentEnvironmentName) + resourceGroup: $(ResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true @@ -257,16 +308,17 @@ stages: - aadTestEnvironment - DockerBuild - deploySqlServer + - createAcaEnvironment jobs: - - template: ./jobs/provision-deploy.yml + - template: ./jobs/provision-deploy-aca.yml parameters: version: R5 sql: true webAppName: $(DeploymentEnvironmentNameR5Sql) - appServicePlanName: $(appServicePlanName) - appServicePlanResourceGroup: $(appServicePlanResourceGroup) + acaEnvironmentName: $(AcaEnvironmentName) + keyVaultName: $(KeyVaultNameR5Sql) subscription: $(ConnectedServiceName) - resourceGroup: $(DeploymentEnvironmentName) + resourceGroup: $(ResourceGroupName) testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) schemaAutomaticUpdatesEnabled: 'auto' diff --git a/build/ci-pipeline.yml b/build/ci-pipeline.yml index d3ed4dffad..d2fb1c8e66 100644 --- a/build/ci-pipeline.yml +++ b/build/ci-pipeline.yml @@ -149,6 +149,7 @@ stages: webAppName: $(DeploymentEnvironmentName) subscription: $(ConnectedServiceName) imageTag: $(ImageTag) + resourceGroup: $(ResourceGroupName) - stage: redeployStu3Sql displayName: 'Redeploy STU3 SQL Site' @@ -161,6 +162,7 @@ stages: webAppName: $(DeploymentEnvironmentNameSql) subscription: $(ConnectedServiceName) imageTag: $(ImageTag) + resourceGroup: $(ResourceGroupName) - stage: testStu3Cosmos displayName: 'Run Stu3 Cosmos Tests' @@ -168,11 +170,11 @@ stages: - BuildArtifacts - redeployStu3 jobs: - - template: ./jobs/run-cosmos-tests.yml + - template: ./jobs/run-cosmos-tests-aca.yml parameters: version: Stu3 - keyVaultName: $(DeploymentEnvironmentName) - appServiceName: $(DeploymentEnvironmentName) + keyVaultName: $(KeyVaultBaseName) + containerAppName: $(DeploymentEnvironmentName) - stage: testStu3Sql displayName: 'Run Stu3 SQL Tests' @@ -180,11 +182,11 @@ stages: - BuildArtifacts - redeployStu3Sql jobs: - - template: ./jobs/run-sql-tests.yml + - template: ./jobs/run-sql-tests-aca.yml parameters: version: Stu3 - keyVaultName: $(DeploymentEnvironmentName) - appServiceName: $(DeploymentEnvironmentName) + keyVaultName: $(KeyVaultNameSql) + containerAppName: $(DeploymentEnvironmentNameSql) integrationSqlServerName: $(DeploymentEnvironmentName)inttest # *********************** R4 *********************** @@ -199,6 +201,7 @@ stages: webAppName: $(DeploymentEnvironmentNameR4) subscription: $(ConnectedServiceName) imageTag: $(ImageTag) + resourceGroup: $(ResourceGroupName) - stage: redeployR4Sql displayName: 'Redeploy R4 SQL Site' @@ -211,6 +214,7 @@ stages: webAppName: $(DeploymentEnvironmentNameR4Sql) subscription: $(ConnectedServiceName) imageTag: $(ImageTag) + resourceGroup: $(ResourceGroupName) - stage: testR4Cosmos displayName: 'Run R4 Cosmos Tests' @@ -218,11 +222,11 @@ stages: - BuildArtifacts - redeployR4 jobs: - - template: ./jobs/run-cosmos-tests.yml + - template: ./jobs/run-cosmos-tests-aca.yml parameters: version: R4 - keyVaultName: $(DeploymentEnvironmentNameR4) - appServiceName: $(DeploymentEnvironmentNameR4) + keyVaultName: $(KeyVaultNameR4) + containerAppName: $(DeploymentEnvironmentNameR4) - stage: testR4Sql displayName: 'Run R4 SQL Tests' @@ -230,11 +234,11 @@ stages: - BuildArtifacts - redeployR4Sql jobs: - - template: ./jobs/run-sql-tests.yml + - template: ./jobs/run-sql-tests-aca.yml parameters: version: R4 - keyVaultName: $(DeploymentEnvironmentNameR4) - appServiceName: $(DeploymentEnvironmentNameR4) + keyVaultName: $(KeyVaultNameR4Sql) + containerAppName: $(DeploymentEnvironmentNameR4Sql) integrationSqlServerName: $(DeploymentEnvironmentName)inttest # *********************** R4B *********************** @@ -249,6 +253,7 @@ stages: webAppName: $(DeploymentEnvironmentNameR4B) subscription: $(ConnectedServiceName) imageTag: $(ImageTag) + resourceGroup: $(ResourceGroupName) - stage: redeployR4BSql displayName: 'Redeploy R4B SQL Site' @@ -261,6 +266,7 @@ stages: webAppName: $(DeploymentEnvironmentNameR4BSql) subscription: $(ConnectedServiceName) imageTag: $(ImageTag) + resourceGroup: $(ResourceGroupName) - stage: testR4BCosmos displayName: 'Run R4B Cosmos Tests' @@ -268,11 +274,11 @@ stages: - BuildArtifacts - redeployR4B jobs: - - template: ./jobs/run-cosmos-tests.yml + - template: ./jobs/run-cosmos-tests-aca.yml parameters: version: R4B - keyVaultName: $(DeploymentEnvironmentNameR4B) - appServiceName: $(DeploymentEnvironmentNameR4B) + keyVaultName: $(KeyVaultNameR4B) + containerAppName: $(DeploymentEnvironmentNameR4B) - stage: testR4BSql displayName: 'Run R4B SQL Tests' @@ -280,11 +286,11 @@ stages: - BuildArtifacts - redeployR4BSql jobs: - - template: ./jobs/run-sql-tests.yml + - template: ./jobs/run-sql-tests-aca.yml parameters: version: R4B - keyVaultName: $(DeploymentEnvironmentNameR4B) - appServiceName: $(DeploymentEnvironmentNameR4B) + keyVaultName: $(KeyVaultNameR4BSql) + containerAppName: $(DeploymentEnvironmentNameR4BSql) integrationSqlServerName: $(DeploymentEnvironmentName)inttest # *********************** R5 *********************** @@ -299,6 +305,7 @@ stages: webAppName: $(DeploymentEnvironmentNameR5) subscription: $(ConnectedServiceName) imageTag: $(ImageTag) + resourceGroup: $(ResourceGroupName) - stage: redeployR5Sql displayName: 'Redeploy R5 SQL Site' @@ -311,6 +318,7 @@ stages: webAppName: $(DeploymentEnvironmentNameR5Sql) subscription: $(ConnectedServiceName) imageTag: $(ImageTag) + resourceGroup: $(ResourceGroupName) - stage: testR5Cosmos displayName: 'Run R5 Cosmos Tests' @@ -318,11 +326,11 @@ stages: - BuildArtifacts - redeployR5 jobs: - - template: ./jobs/run-cosmos-tests.yml + - template: ./jobs/run-cosmos-tests-aca.yml parameters: version: R5 - keyVaultName: $(DeploymentEnvironmentNameR5) - appServiceName: $(DeploymentEnvironmentNameR5) + keyVaultName: $(KeyVaultNameR5) + containerAppName: $(DeploymentEnvironmentNameR5) - stage: testR5Sql displayName: 'Run R5 SQL Tests' @@ -330,11 +338,11 @@ stages: - BuildArtifacts - redeployR5Sql jobs: - - template: ./jobs/run-sql-tests.yml + - template: ./jobs/run-sql-tests-aca.yml parameters: version: R5 - keyVaultName: $(DeploymentEnvironmentNameR5) - appServiceName: $(DeploymentEnvironmentNameR5) + keyVaultName: $(KeyVaultNameR5Sql) + containerAppName: $(DeploymentEnvironmentNameR5Sql) integrationSqlServerName: $(DeploymentEnvironmentName)inttest - stage: aggregateCoverage diff --git a/build/ci-variables.yml b/build/ci-variables.yml index d7e2416f6c..2b5610c768 100644 --- a/build/ci-variables.yml +++ b/build/ci-variables.yml @@ -1,8 +1,8 @@ variables: ResourceGroupRegion: 'southcentralus' - resourceGroupRoot: 'msh-fhir-ci' - appServicePlanName: '$(resourceGroupRoot)-linux' + resourceGroupRoot: 'msh-fhir-ci-aca' DeploymentEnvironmentName: '$(resourceGroupRoot)' + AcaEnvironmentName: '$(DeploymentEnvironmentName)-acae' # CI uses the same name for Key Vaults as the deployment environment (no BuildId suffix needed) KeyVaultBaseName: '$(DeploymentEnvironmentName)' ResourceGroupName: '$(resourceGroupRoot)' diff --git a/build/export-pipeline.yml b/build/export-pipeline.yml index d0e5b882ed..5852f6881c 100644 --- a/build/export-pipeline.yml +++ b/build/export-pipeline.yml @@ -45,6 +45,7 @@ stages: parameters: webAppName: $(DeploymentEnvironmentName) subscription: $(ConnectedServiceName) + resourceGroup: $(ResourceGroupName) - stage: redeployStu3Sql displayName: 'Redeploy STU3 SQL Site' @@ -54,6 +55,7 @@ stages: parameters: webAppName: $(DeploymentEnvironmentNameSql) subscription: $(ConnectedServiceName) + resourceGroup: $(ResourceGroupName) - stage: redeployR4 displayName: 'Redeploy R4 Site' @@ -63,6 +65,7 @@ stages: parameters: webAppName: $(DeploymentEnvironmentNameR4) subscription: $(ConnectedServiceName) + resourceGroup: $(ResourceGroupName) - stage: redeployR4Sql displayName: 'Redeploy R4 SQL Site' @@ -72,6 +75,7 @@ stages: parameters: webAppName: $(DeploymentEnvironmentNameR4Sql) subscription: $(ConnectedServiceName) + resourceGroup: $(ResourceGroupName) - stage: testStu3 displayName: 'Run Stu3 Tests' @@ -84,6 +88,8 @@ stages: parameters: version: Stu3 keyVaultName: $(DeploymentEnvironmentName) + cosmosContainerAppName: $(DeploymentEnvironmentName) + sqlContainerAppName: $(DeploymentEnvironmentNameSql) - stage: testR4 displayName: 'Run R4 Tests' @@ -96,6 +102,8 @@ stages: parameters: version: R4 keyVaultName: $(DeploymentEnvironmentNameR4) + cosmosContainerAppName: $(DeploymentEnvironmentNameR4) + sqlContainerAppName: $(DeploymentEnvironmentNameR4Sql) - stage: cleanStorageAccounts displayName: 'Clean Storage Accounts' diff --git a/build/export-variables.yml b/build/export-variables.yml index f4e7eebc4b..03b300353f 100644 --- a/build/export-variables.yml +++ b/build/export-variables.yml @@ -1,8 +1,8 @@ variables: ResourceGroupRegion: 'southcentralus' resourceGroupRoot: 'msh-fhir-export' - appServicePlanName: '$(resourceGroupRoot)-linux' DeploymentEnvironmentName: '$(resourceGroupRoot)' + AcaEnvironmentName: '$(DeploymentEnvironmentName)-acae' ResourceGroupName: '$(resourceGroupRoot)' TestEnvironmentName: 'OSS Export' ImageTag: 'master' diff --git a/build/jobs/redeploy-webapp.yml b/build/jobs/redeploy-webapp.yml index 6845260147..1fdac826e2 100644 --- a/build/jobs/redeploy-webapp.yml +++ b/build/jobs/redeploy-webapp.yml @@ -7,23 +7,43 @@ parameters: type: string - name: imageTag type: string +- name: resourceGroup + type: string jobs: - job: provisionEnvironment pool: name: '$(DefaultLinuxPool)' vmImage: '$(LinuxVmImage)' - steps: - - task: AzureRmWebAppDeployment@4 - displayName: 'Azure App Service Deploy' + steps: + - task: AzureCLI@2 + displayName: 'Update Container App Image' inputs: azureSubscription: '${{ parameters.subscription }}' - appType: 'webAppContainer' - WebAppName: '${{ parameters.webAppName }}' - DockerNamespace: $(azureContainerRegistry) - DockerRepository: '${{ parameters.version }}_fhir-server' - DockerImageTag: ${{ parameters.imageTag }} + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $containerAppName = "${{ parameters.webAppName }}".ToLower() + $rgName = "${{ parameters.resourceGroup }}" + $newImage = "$(azureContainerRegistry)/${{ parameters.version }}_fhir-server:${{ parameters.imageTag }}" + + $currentImage = az containerapp show ` + --name $containerAppName ` + --resource-group $rgName ` + --query "properties.template.containers[0].image" -o tsv + Write-Host "Current image: $currentImage" + Write-Host "New image: $newImage" + + az containerapp update ` + --name $containerAppName ` + --resource-group $rgName ` + --image $newImage + + if ($LASTEXITCODE -ne 0) { throw "Failed to update container app '$containerAppName'" } + Write-Host "Container app image updated successfully." - - template: ./provision-healthcheck.yml - parameters: - webAppName: ${{ parameters.webAppName }} \ No newline at end of file + - template: ../tasks/container-app-health-check.yml + parameters: + containerAppName: ${{ parameters.webAppName }} + resourceGroup: ${{ parameters.resourceGroup }} + subscription: ${{ parameters.subscription }} \ No newline at end of file diff --git a/build/jobs/restart-webapp.yml b/build/jobs/restart-webapp.yml index f16d54448b..7a9d5c1612 100644 --- a/build/jobs/restart-webapp.yml +++ b/build/jobs/restart-webapp.yml @@ -3,20 +3,38 @@ parameters: type: string - name: subscription type: string +- name: resourceGroup + type: string jobs: - job: provisionEnvironment pool: name: '$(DefaultLinuxPool)' vmImage: '$(LinuxVmImage)' - steps: - - task: AzureAppServiceManage@0 - displayName: 'Azure App Service Restart' + steps: + - task: AzureCLI@2 + displayName: 'Restart Container App' inputs: azureSubscription: '${{ parameters.subscription }}' - action: 'Restart Azure App Service' - WebAppName: '${{ parameters.webAppName }}' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $containerAppName = "${{ parameters.webAppName }}".ToLower() + $rgName = "${{ parameters.resourceGroup }}" + + # Create a new revision to force restart + $suffix = "r$(Get-Date -Format 'yyMMddHHmmss')" + Write-Host "Restarting container app '$containerAppName' with revision suffix '$suffix'..." + az containerapp update ` + --name $containerAppName ` + --resource-group $rgName ` + --revision-suffix $suffix + + if ($LASTEXITCODE -ne 0) { throw "Failed to restart container app '$containerAppName'" } + Write-Host "Container app revision updated successfully." - - template: ./provision-healthcheck.yml - parameters: - webAppName: ${{ parameters.webAppName }} \ No newline at end of file + - template: ../tasks/container-app-health-check.yml + parameters: + containerAppName: ${{ parameters.webAppName }} + resourceGroup: ${{ parameters.resourceGroup }} + subscription: ${{ parameters.subscription }} \ No newline at end of file diff --git a/build/jobs/run-export-tests.yml b/build/jobs/run-export-tests.yml index 125ee7a6af..b6bb2f468a 100644 --- a/build/jobs/run-export-tests.yml +++ b/build/jobs/run-export-tests.yml @@ -3,6 +3,10 @@ parameters: type: string - name: keyVaultName type: string +- name: cosmosContainerAppName + type: string +- name: sqlContainerAppName + type: string jobs: - job: 'cosmosE2eTests' @@ -16,6 +20,25 @@ jobs: parameters: version: ${{parameters.version}} + - task: AzurePowerShell@5 + displayName: 'Resolve ACA FQDN (Cosmos)' + inputs: + azureSubscription: $(ConnectedServiceName) + azurePowerShellVersion: latestVersion + pwsh: true + ScriptType: inlineScript + Inline: | + $containerAppName = "${{ parameters.cosmosContainerAppName }}".ToLowerInvariant() + $containerApp = Get-AzResource -ResourceGroupName $(ResourceGroupName) -ResourceType "Microsoft.App/containerApps" -Name $containerAppName -ApiVersion "2023-05-01" -ExpandProperties + if ($null -eq $containerApp) { + throw "Container App '$containerAppName' not found in resource group '$(ResourceGroupName)'" + } + $fqdn = $containerApp.Properties.configuration.ingress.fqdn + $url = "https://$fqdn" + Write-Host "Resolved Cosmos ACA URL: $url" + Write-Host "##vso[task.setvariable variable=TestEnvironmentUrl]$url" + Write-Host "##vso[task.setvariable variable=TestEnvironmentUrl_${{ parameters.version }}]$url" + - task: AzurePowerShell@5 displayName: 'Set Variables' inputs: @@ -136,6 +159,25 @@ jobs: - template: e2e-tests-extract.yml parameters: version: ${{parameters.version}} + + - task: AzurePowerShell@5 + displayName: 'Resolve ACA FQDN (SQL)' + inputs: + azureSubscription: $(ConnectedServiceName) + azurePowerShellVersion: latestVersion + pwsh: true + ScriptType: inlineScript + Inline: | + $containerAppName = "${{ parameters.sqlContainerAppName }}".ToLowerInvariant() + $containerApp = Get-AzResource -ResourceGroupName $(ResourceGroupName) -ResourceType "Microsoft.App/containerApps" -Name $containerAppName -ApiVersion "2023-05-01" -ExpandProperties + if ($null -eq $containerApp) { + throw "Container App '$containerAppName' not found in resource group '$(ResourceGroupName)'" + } + $fqdn = $containerApp.Properties.configuration.ingress.fqdn + $url = "https://$fqdn" + Write-Host "Resolved SQL ACA URL: $url" + Write-Host "##vso[task.setvariable variable=TestEnvironmentUrl_Sql]$url" + Write-Host "##vso[task.setvariable variable=TestEnvironmentUrl_${{ parameters.version }}_Sql]$url" - task: AzurePowerShell@5 displayName: 'Set Variables' diff --git a/build/pr-variables.yml b/build/pr-variables.yml index e62847e660..eaad45547d 100644 --- a/build/pr-variables.yml +++ b/build/pr-variables.yml @@ -3,7 +3,6 @@ variables: resourceGroupRoot: 'msh-fhir-pr' # Use PR number when available; otherwise fall back to BuildId for manual/non-PR runs. prName: $[coalesce(variables['System.PullRequest.PullRequestNumber'], variables['Build.BuildId'])] - appServicePlanName: '$(resourceGroupRoot)-$(prName)-asp' ResourceGroupName: '$(resourceGroupRoot)-$(prName)' # Unique resource group name with Build.BuildId suffix to avoid waiting for deletion of existing RG UniqueResourceGroupName: '$(resourceGroupRoot)-$(prName)-$(Build.BuildId)' diff --git a/build/tasks/container-app-health-check.yml b/build/tasks/container-app-health-check.yml new file mode 100644 index 0000000000..d30cc7542a --- /dev/null +++ b/build/tasks/container-app-health-check.yml @@ -0,0 +1,58 @@ +parameters: +- name: containerAppName + type: string +- name: resourceGroup + type: string +- name: subscription + type: string + +steps: +- task: AzureCLI@2 + displayName: 'Health Check - ${{ parameters.containerAppName }}' + inputs: + azureSubscription: '${{ parameters.subscription }}' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $fqdn = az containerapp show ` + --name "${{ parameters.containerAppName }}".ToLower() ` + --resource-group "${{ parameters.resourceGroup }}" ` + --query "properties.configuration.ingress.fqdn" -o tsv + + if (-not $fqdn) { + throw "Could not resolve FQDN for container app '${{ parameters.containerAppName }}'" + } + + $healthCheckUrl = "https://$fqdn/health/check" + $timeoutMinutes = 7 + $consecutiveSuccessRequired = 5 + $consecutiveSuccessCount = 0 + $startTime = Get-Date + $timeoutTime = $startTime.AddMinutes($timeoutMinutes) + + Write-Host "Starting health check for $healthCheckUrl" + + Do { + Start-Sleep -Seconds 5 + Write-Host "Checking: $healthCheckUrl ($consecutiveSuccessCount/$consecutiveSuccessRequired)" + try { + $response = Invoke-WebRequest -Uri $healthCheckUrl -UseBasicParsing + if ($response.StatusCode -eq 200) { + $consecutiveSuccessCount++ + Write-Host "Success ($consecutiveSuccessCount/$consecutiveSuccessRequired)" + } else { + $consecutiveSuccessCount = 0 + } + } catch { + Write-Host $_.Exception.Message + $consecutiveSuccessCount = 0 + } finally { + $Error.Clear() + } + + if ((Get-Date) -gt $timeoutTime) { + throw "Health check timed out after $([math]::Round(((Get-Date) - $startTime).TotalMinutes, 2)) minutes" + } + } While ($consecutiveSuccessCount -lt $consecutiveSuccessRequired) + + Write-Host "Health check passed after $([math]::Round(((Get-Date) - $startTime).TotalSeconds, 2)) seconds" From 57344e7a4a96146b37ec2e27d356d8b95beaab1a Mon Sep 17 00:00:00 2001 From: John Estrada Date: Fri, 27 Mar 2026 16:22:29 -0500 Subject: [PATCH 22/38] fixing image casing --- build/jobs/redeploy-webapp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/jobs/redeploy-webapp.yml b/build/jobs/redeploy-webapp.yml index 1fdac826e2..95a75bc228 100644 --- a/build/jobs/redeploy-webapp.yml +++ b/build/jobs/redeploy-webapp.yml @@ -25,7 +25,7 @@ jobs: inlineScript: | $containerAppName = "${{ parameters.webAppName }}".ToLower() $rgName = "${{ parameters.resourceGroup }}" - $newImage = "$(azureContainerRegistry)/${{ parameters.version }}_fhir-server:${{ parameters.imageTag }}" + $newImage = "$(azureContainerRegistry)/$("${{ parameters.version }}".ToLower())_fhir-server:${{ parameters.imageTag }}" $currentImage = az containerapp show ` --name $containerAppName ` From 437e11b7f22329ddd09e0f0278a80545370fab2d Mon Sep 17 00:00:00 2001 From: John Estrada Date: Mon, 30 Mar 2026 11:49:08 -0500 Subject: [PATCH 23/38] Add version and appServiceType parameters to ACA variable setting and update related functions --- build/jobs/e2e-tests-aca.yml | 2 ++ build/tasks/e2e-set-variables-aca.yml | 37 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/build/jobs/e2e-tests-aca.yml b/build/jobs/e2e-tests-aca.yml index 68f9655bda..6f28ad4ea9 100644 --- a/build/jobs/e2e-tests-aca.yml +++ b/build/jobs/e2e-tests-aca.yml @@ -20,6 +20,8 @@ steps: - template: ../tasks/e2e-set-variables-aca.yml parameters: containerAppName: ${{ parameters.containerAppName }} + version: ${{ parameters.version }} + appServiceType: ${{ parameters.appServiceType }} - task: PowerShell@2 displayName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}' diff --git a/build/tasks/e2e-set-variables-aca.yml b/build/tasks/e2e-set-variables-aca.yml index 458a4f796b..f05dacd461 100644 --- a/build/tasks/e2e-set-variables-aca.yml +++ b/build/tasks/e2e-set-variables-aca.yml @@ -1,6 +1,10 @@ parameters: - name: containerAppName type: string +- name: version + type: string +- name: appServiceType + type: string steps: - task: AzurePowerShell@5 @@ -52,6 +56,37 @@ steps: return $null } + function Set-TestEnvironmentUrlVariables { + param( + [Parameter(Mandatory = $true)] + [psobject]$ContainerAppData, + [Parameter(Mandatory = $true)] + [string]$Version, + [Parameter(Mandatory = $true)] + [string]$AppServiceType + ) + + $fqdn = $ContainerAppData.Properties.configuration.ingress.fqdn + if ([string]::IsNullOrEmpty($fqdn) -or $fqdn -eq 'null') { + throw "Container App '$containerAppName' does not have an ingress FQDN configured." + } + + $url = "https://$fqdn" + if ($AppServiceType -eq 'SqlServer') { + $defaultVariableName = 'TestEnvironmentUrl_Sql' + $versionVariableName = "TestEnvironmentUrl_${Version}_Sql" + } + else { + $defaultVariableName = 'TestEnvironmentUrl' + $versionVariableName = "TestEnvironmentUrl_$Version" + } + + Write-Host "Resolved ACA URL for $AppServiceType/$Version: $url" + Write-Host "Setting pipeline variables '$defaultVariableName' and '$versionVariableName'" + Write-Host "##vso[task.setvariable variable=$defaultVariableName]$url" + Write-Host "##vso[task.setvariable variable=$versionVariableName]$url" + } + $keyVault = "$(KeyVaultBaseName)-ts" Set-SecretsAsPipelineVariables -VaultName $keyVault @@ -67,6 +102,8 @@ steps: throw "Container App '$containerAppName' has no environment variables configured." } + Set-TestEnvironmentUrlVariables -ContainerAppData $containerAppData -Version '${{ parameters.version }}' -AppServiceType '${{ parameters.appServiceType }}' + $acrLoginServer = Get-ContainerEnvValue -EnvSettings $envSettings -Name "FhirServer__Operations__ConvertData__ContainerRegistryServers__0" $exportStoreUri = Get-ContainerEnvValue -EnvSettings $envSettings -Name "FhirServer__Operations__Export__StorageAccountUri" From 391c2061503698de9ab4fbaa0e892d07d616c5d2 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Mon, 30 Mar 2026 11:53:24 -0500 Subject: [PATCH 24/38] Fix indentation for containerAppName parameter in ACA variable settings --- build/tasks/e2e-set-variables-aca.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/tasks/e2e-set-variables-aca.yml b/build/tasks/e2e-set-variables-aca.yml index f05dacd461..859c7a1e3e 100644 --- a/build/tasks/e2e-set-variables-aca.yml +++ b/build/tasks/e2e-set-variables-aca.yml @@ -1,6 +1,6 @@ parameters: - name: containerAppName - type: string + type: string - name: version type: string - name: appServiceType From 8e050b518f34aec45aa27c1872d68e3625289b79 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Mon, 30 Mar 2026 11:56:32 -0500 Subject: [PATCH 25/38] fixing indents --- build/tasks/e2e-set-variables-aca.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/tasks/e2e-set-variables-aca.yml b/build/tasks/e2e-set-variables-aca.yml index 859c7a1e3e..ecdb45f136 100644 --- a/build/tasks/e2e-set-variables-aca.yml +++ b/build/tasks/e2e-set-variables-aca.yml @@ -1,10 +1,10 @@ parameters: - name: containerAppName - type: string + type: string - name: version - type: string + type: string - name: appServiceType - type: string + type: string steps: - task: AzurePowerShell@5 From 13071d2945cd1967c201f03c4e6ad6d68094db68 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Mon, 30 Mar 2026 12:30:34 -0500 Subject: [PATCH 26/38] Fix string interpolation for ACA URL resolution in pipeline variables --- build/tasks/e2e-set-variables-aca.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/tasks/e2e-set-variables-aca.yml b/build/tasks/e2e-set-variables-aca.yml index ecdb45f136..94a0ef7ba5 100644 --- a/build/tasks/e2e-set-variables-aca.yml +++ b/build/tasks/e2e-set-variables-aca.yml @@ -81,7 +81,7 @@ steps: $versionVariableName = "TestEnvironmentUrl_$Version" } - Write-Host "Resolved ACA URL for $AppServiceType/$Version: $url" + Write-Host "Resolved ACA URL for $AppServiceType/$($Version): $url" Write-Host "Setting pipeline variables '$defaultVariableName' and '$versionVariableName'" Write-Host "##vso[task.setvariable variable=$defaultVariableName]$url" Write-Host "##vso[task.setvariable variable=$versionVariableName]$url" From 99684130e6fc27de925dd7a0278a47f1ed5eea85 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 1 Apr 2026 11:39:31 -0500 Subject: [PATCH 27/38] Refactor container resource parameters and update scaling rules in ACA templates --- build/jobs/provision-deploy-aca.yml | 10 +----- build/jobs/provision-deploy-cosmos-aca.yml | 10 +----- build/pr-pipeline.yml | 16 ---------- samples/templates/aca/fhir-cosmos-aca.json | 36 +++++++++++++++++----- samples/templates/aca/fhir-sql-aca.json | 36 +++++++++++++++++----- 5 files changed, 58 insertions(+), 50 deletions(-) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index a4be2de485..1b8e1e4732 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -29,12 +29,6 @@ parameters: default: true - name: keyVaultName type: string -- name: containerCpu - type: string - default: '1.0' -- name: containerMemory - type: string - default: '2Gi' jobs: - job: provisionEnvironment @@ -148,9 +142,7 @@ jobs: enableConvertData = $true enableReindex = if ("${{ parameters.reindexEnabled }}" -eq "true") { $true } else { $false } minReplicas = 3 - maxReplicas = 6 - containerCpu = "${{ parameters.containerCpu }}" - containerMemory = "${{ parameters.containerMemory }}" + maxReplicas = 8 acrPullUserAssignedManagedIdentityResourceId = '/subscriptions/a1766500-6fd5-4f5c-8515-607798271014/resourceGroups/HealthPlatformRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-uami' additionalEnvVars = $additionalEnvVars } diff --git a/build/jobs/provision-deploy-cosmos-aca.yml b/build/jobs/provision-deploy-cosmos-aca.yml index 3289fb8fa4..b61fef9856 100644 --- a/build/jobs/provision-deploy-cosmos-aca.yml +++ b/build/jobs/provision-deploy-cosmos-aca.yml @@ -18,12 +18,6 @@ parameters: default: true - name: keyVaultName type: string -- name: containerCpu - type: string - default: '1.0' -- name: containerMemory - type: string - default: '2Gi' jobs: - job: provisionEnvironment @@ -191,9 +185,7 @@ jobs: enableConvertData = $true enableReindex = if ("${{ parameters.reindexEnabled }}" -eq "true") { $true } else { $false } minReplicas = 3 - maxReplicas = 6 - containerCpu = "${{ parameters.containerCpu }}" - containerMemory = "${{ parameters.containerMemory }}" + maxReplicas = 8 acrPullUserAssignedManagedIdentityResourceId = '/subscriptions/a1766500-6fd5-4f5c-8515-607798271014/resourceGroups/HealthPlatformRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-uami' additionalEnvVars = $additionalEnvVars } diff --git a/build/pr-pipeline.yml b/build/pr-pipeline.yml index d2c49d0890..7c74fbc1b5 100644 --- a/build/pr-pipeline.yml +++ b/build/pr-pipeline.yml @@ -284,8 +284,6 @@ stages: testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true - containerCpu: '1.0' - containerMemory: '2Gi' - stage: deployStu3Sql displayName: 'Deploy STU3 SQL Site' @@ -308,8 +306,6 @@ stages: schemaAutomaticUpdatesEnabled: 'auto' sqlServerName: $(DeploymentEnvironmentName) reindexEnabled: true - containerCpu: '1.0' - containerMemory: '2Gi' - stage: deployR4 displayName: 'Deploy R4 CosmosDB Site' @@ -329,8 +325,6 @@ stages: testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true - containerCpu: '1.0' - containerMemory: '2Gi' - stage: deployR4Sql displayName: 'Deploy R4 SQL Site' @@ -353,8 +347,6 @@ stages: schemaAutomaticUpdatesEnabled: 'auto' sqlServerName: $(DeploymentEnvironmentName) reindexEnabled: true - containerCpu: '1.0' - containerMemory: '2Gi' - stage: deployR4B displayName: 'Deploy R4B CosmosDB Site' @@ -374,8 +366,6 @@ stages: testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true - containerCpu: '1.0' - containerMemory: '2Gi' - stage: deployR4BSql displayName: 'Deploy R4B SQL Site' @@ -398,8 +388,6 @@ stages: schemaAutomaticUpdatesEnabled: 'auto' sqlServerName: $(DeploymentEnvironmentName) reindexEnabled: true - containerCpu: '1.0' - containerMemory: '2Gi' - stage: deployR5 displayName: 'Deploy R5 CosmosDB Site' @@ -419,8 +407,6 @@ stages: testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true - containerCpu: '1.0' - containerMemory: '2Gi' - stage: deployR5Sql displayName: 'Deploy R5 SQL Site' @@ -443,8 +429,6 @@ stages: schemaAutomaticUpdatesEnabled: 'auto' sqlServerName: $(DeploymentEnvironmentName) reindexEnabled: true - containerCpu: '1.0' - containerMemory: '2Gi' - stage: testStu3Cosmos displayName: 'Run Stu3 Cosmos Tests' diff --git a/samples/templates/aca/fhir-cosmos-aca.json b/samples/templates/aca/fhir-cosmos-aca.json index e1a1aec74f..e35f41e7c6 100644 --- a/samples/templates/aca/fhir-cosmos-aca.json +++ b/samples/templates/aca/fhir-cosmos-aca.json @@ -87,11 +87,11 @@ }, "containerCpu": { "type": "string", - "defaultValue": "1.0" + "defaultValue": "0.5" }, "containerMemory": { "type": "string", - "defaultValue": "2Gi" + "defaultValue": "1Gi" }, "additionalEnvVars": { "type": "array", @@ -208,7 +208,7 @@ "resources": [ { "type": "Microsoft.App/containerApps", - "apiVersion": "2023-05-01", + "apiVersion": "2024-03-01", "name": "[variables('containerAppName')]", "location": "[resourceGroup().location]", "identity": { @@ -270,7 +270,27 @@ ], "scale": { "minReplicas": "[parameters('minReplicas')]", - "maxReplicas": "[parameters('maxReplicas')]" + "maxReplicas": "[parameters('maxReplicas')]", + "rules": [ + { + "name": "http-scaling", + "http": { + "metadata": { + "concurrentRequests": "50" + } + } + }, + { + "name": "cpu-scaling", + "custom": { + "type": "cpu", + "metadata": { + "type": "Utilization", + "value": "70" + } + } + } + ] } } }, @@ -302,7 +322,7 @@ ], "properties": { "roleDefinitionId": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/', 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7')]", - "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').identity.principalId]" + "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').identity.principalId]" } }, { @@ -332,7 +352,7 @@ ], "properties": { "roleDefinitionId": "[variables('storageBlobDataContributerRoleId')]", - "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').identity.principalId]", + "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').identity.principalId]", "principalType": "ServicePrincipal" } }, @@ -356,11 +376,11 @@ }, "containerAppFqdn": { "type": "string", - "value": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').properties.configuration.ingress.fqdn]" + "value": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').properties.configuration.ingress.fqdn]" }, "containerAppUrl": { "type": "string", - "value": "[concat('https://', reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').properties.configuration.ingress.fqdn)]" + "value": "[concat('https://', reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').properties.configuration.ingress.fqdn)]" }, "exportStorageUri": { "type": "string", diff --git a/samples/templates/aca/fhir-sql-aca.json b/samples/templates/aca/fhir-sql-aca.json index 956dc2677d..412a5b0bb3 100644 --- a/samples/templates/aca/fhir-sql-aca.json +++ b/samples/templates/aca/fhir-sql-aca.json @@ -95,11 +95,11 @@ }, "containerCpu": { "type": "string", - "defaultValue": "1.0" + "defaultValue": "0.5" }, "containerMemory": { "type": "string", - "defaultValue": "2Gi" + "defaultValue": "1Gi" }, "additionalEnvVars": { "type": "array", @@ -216,7 +216,7 @@ "resources": [ { "type": "Microsoft.App/containerApps", - "apiVersion": "2023-05-01", + "apiVersion": "2024-03-01", "name": "[variables('containerAppName')]", "location": "[resourceGroup().location]", "identity": { @@ -278,7 +278,27 @@ ], "scale": { "minReplicas": "[parameters('minReplicas')]", - "maxReplicas": "[parameters('maxReplicas')]" + "maxReplicas": "[parameters('maxReplicas')]", + "rules": [ + { + "name": "http-scaling", + "http": { + "metadata": { + "concurrentRequests": "50" + } + } + }, + { + "name": "cpu-scaling", + "custom": { + "type": "cpu", + "metadata": { + "type": "Utilization", + "value": "70" + } + } + } + ] } } }, @@ -310,7 +330,7 @@ ], "properties": { "roleDefinitionId": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/', 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7')]", - "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').identity.principalId]" + "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').identity.principalId]" } }, { @@ -340,7 +360,7 @@ ], "properties": { "roleDefinitionId": "[variables('storageBlobDataContributerRoleId')]", - "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').identity.principalId]", + "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').identity.principalId]", "principalType": "ServicePrincipal" } }, @@ -364,11 +384,11 @@ }, "containerAppFqdn": { "type": "string", - "value": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').properties.configuration.ingress.fqdn]" + "value": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').properties.configuration.ingress.fqdn]" }, "containerAppUrl": { "type": "string", - "value": "[concat('https://', reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2023-05-01', 'Full').properties.configuration.ingress.fqdn)]" + "value": "[concat('https://', reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').properties.configuration.ingress.fqdn)]" }, "exportStorageUri": { "type": "string", From 20c894280fffd82a96b05720a64ac87bb08ef5fa Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 1 Apr 2026 14:13:33 -0500 Subject: [PATCH 28/38] Update minReplicas to 0 and maxReplicas to 8 in ACA deployment templates --- build/jobs/provision-deploy-aca.yml | 2 +- build/jobs/provision-deploy-cosmos-aca.yml | 2 +- samples/templates/aca/fhir-cosmos-aca.json | 4 ++-- samples/templates/aca/fhir-sql-aca.json | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index 1b8e1e4732..228143e58f 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -141,7 +141,7 @@ jobs: enableImport = $true enableConvertData = $true enableReindex = if ("${{ parameters.reindexEnabled }}" -eq "true") { $true } else { $false } - minReplicas = 3 + minReplicas = 0 maxReplicas = 8 acrPullUserAssignedManagedIdentityResourceId = '/subscriptions/a1766500-6fd5-4f5c-8515-607798271014/resourceGroups/HealthPlatformRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-uami' additionalEnvVars = $additionalEnvVars diff --git a/build/jobs/provision-deploy-cosmos-aca.yml b/build/jobs/provision-deploy-cosmos-aca.yml index b61fef9856..ef91db54c3 100644 --- a/build/jobs/provision-deploy-cosmos-aca.yml +++ b/build/jobs/provision-deploy-cosmos-aca.yml @@ -184,7 +184,7 @@ jobs: enableImport = $true enableConvertData = $true enableReindex = if ("${{ parameters.reindexEnabled }}" -eq "true") { $true } else { $false } - minReplicas = 3 + minReplicas = 0 maxReplicas = 8 acrPullUserAssignedManagedIdentityResourceId = '/subscriptions/a1766500-6fd5-4f5c-8515-607798271014/resourceGroups/HealthPlatformRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-uami' additionalEnvVars = $additionalEnvVars diff --git a/samples/templates/aca/fhir-cosmos-aca.json b/samples/templates/aca/fhir-cosmos-aca.json index e35f41e7c6..0d00633ddc 100644 --- a/samples/templates/aca/fhir-cosmos-aca.json +++ b/samples/templates/aca/fhir-cosmos-aca.json @@ -79,11 +79,11 @@ }, "minReplicas": { "type": "int", - "defaultValue": 1 + "defaultValue": 0 }, "maxReplicas": { "type": "int", - "defaultValue": 3 + "defaultValue": 8 }, "containerCpu": { "type": "string", diff --git a/samples/templates/aca/fhir-sql-aca.json b/samples/templates/aca/fhir-sql-aca.json index 412a5b0bb3..4481960609 100644 --- a/samples/templates/aca/fhir-sql-aca.json +++ b/samples/templates/aca/fhir-sql-aca.json @@ -87,11 +87,11 @@ }, "minReplicas": { "type": "int", - "defaultValue": 1 + "defaultValue": 0 }, "maxReplicas": { "type": "int", - "defaultValue": 3 + "defaultValue": 8 }, "containerCpu": { "type": "string", From 65453bdb3fa3e52a34bb10106a84840e46116ee9 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 1 Apr 2026 14:14:40 -0500 Subject: [PATCH 29/38] Remove unused conditions for AnalyzeSecurity and aggregateCoverage stages in PR pipeline --- build/pr-pipeline.yml | 2 -- build/pr-variables.yml | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/build/pr-pipeline.yml b/build/pr-pipeline.yml index 7c74fbc1b5..0b7dba1946 100644 --- a/build/pr-pipeline.yml +++ b/build/pr-pipeline.yml @@ -83,7 +83,6 @@ stages: - stage: AnalyzeSecurity displayName: 'Run Security Analysis and Validate' - condition: false dependsOn: - BuildUnitTests - BuildArtifacts @@ -564,7 +563,6 @@ stages: - stage: aggregateCoverage displayName: 'Aggregate All Coverage Reports' - condition: false dependsOn: - BuildUnitTests - testStu3Cosmos diff --git a/build/pr-variables.yml b/build/pr-variables.yml index eaad45547d..765b1e7cff 100644 --- a/build/pr-variables.yml +++ b/build/pr-variables.yml @@ -1,8 +1,6 @@ variables: ResourceGroupRegion: 'westus2' resourceGroupRoot: 'msh-fhir-pr' - # Use PR number when available; otherwise fall back to BuildId for manual/non-PR runs. - prName: $[coalesce(variables['System.PullRequest.PullRequestNumber'], variables['Build.BuildId'])] ResourceGroupName: '$(resourceGroupRoot)-$(prName)' # Unique resource group name with Build.BuildId suffix to avoid waiting for deletion of existing RG UniqueResourceGroupName: '$(resourceGroupRoot)-$(prName)-$(Build.BuildId)' @@ -13,4 +11,5 @@ variables: TestEnvironmentName: 'OSS PR$(prName)' ImageTag: '$(build.BuildNumber)' - # prName can still be overridden as a queue-time pipeline variable when needed. + # The following is passed in from a Pipeline variable, this allows it to be overriden if required. + # prName: $(system.pullRequest.pullRequestNumber) From dd8936271baac0e8a32bd3ac01555d6e40b7059d Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 1 Apr 2026 16:03:30 -0500 Subject: [PATCH 30/38] deleting unused ymls --- build/jobs/e2e-tests.yml | 96 ------------ build/jobs/provision-deploy.yml | 239 ------------------------------ build/jobs/run-cosmos-tests.yml | 131 ---------------- build/jobs/run-sql-tests.yml | 140 ----------------- build/tasks/e2e-set-variables.yml | 78 ---------- 5 files changed, 684 deletions(-) delete mode 100644 build/jobs/e2e-tests.yml delete mode 100644 build/jobs/provision-deploy.yml delete mode 100644 build/jobs/run-cosmos-tests.yml delete mode 100644 build/jobs/run-sql-tests.yml delete mode 100644 build/tasks/e2e-set-variables.yml diff --git a/build/jobs/e2e-tests.yml b/build/jobs/e2e-tests.yml deleted file mode 100644 index 12b1babfc9..0000000000 --- a/build/jobs/e2e-tests.yml +++ /dev/null @@ -1,96 +0,0 @@ -parameters: -- name: version - type: string -- name: appServiceName - type: string -- name: appServiceType - type: string -- name: categoryFilter - type: string - default: 'Category!=ExportLongRunning' -- name: testRunTitleSuffix - type: string - default: '' - -steps: - - template: e2e-tests-extract.yml - parameters: - version: ${{parameters.version}} - - - template: ../tasks/e2e-set-variables.yml - parameters: - appServiceName: ${{ parameters.appServiceName }} - - - task: PowerShell@2 - displayName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}' - inputs: - targetType: inline - pwsh: true - script: | - $ErrorActionPreference = 'Stop' - $testRoot = "$(Agent.TempDirectory)/E2ETests" - $testName = "Microsoft.Health.Fhir.${{ parameters.version }}.Tests.E2E" - $dll = Get-ChildItem -Path $testRoot -Recurse -Filter "$testName.dll" | Select-Object -First 1 - if (-not $dll) { - throw "Could not find $testName.dll under $testRoot" - } - - $filter = "FullyQualifiedName~${{ parameters.appServiceType }}&${{ parameters.categoryFilter }}" - $args = @('--filter', $filter, '--retry-failed-tests', '3', '--report-trx') - - Write-Host "Running dotnet $($dll.FullName)" - & dotnet $dll.FullName @args - - if ($LASTEXITCODE -ne 0) { - throw "E2E tests failed with exit code $LASTEXITCODE" - } - env: - 'TestEnvironmentUrl': $(TestEnvironmentUrl) - 'TestEnvironmentUrl_${{ parameters.version }}': $(TestEnvironmentUrl_${{ parameters.version }}) - 'TestEnvironmentUrl_Sql': $(TestEnvironmentUrl_Sql) - 'TestEnvironmentUrl_${{ parameters.version }}_Sql': $(TestEnvironmentUrl_${{ parameters.version }}_Sql) - 'Resource': $(Resource) - 'AllStorageAccounts': $(AllStorageAccounts) - 'TestExportStoreUri': $(TestExportStoreUri) - 'TestIntegrationStoreUri': $(TestIntegrationStoreUri) - 'tenant-admin-service-principal-name': $(tenant-admin-service-principal-name) - 'tenant-admin-service-principal-password': $(tenant-admin-service-principal-password) - 'tenant-admin-user-name': $(tenant-admin-user-name) - 'tenant-admin-user-password': $(tenant-admin-user-password) - 'tenant-id': $(tenant-id) - 'app_globalAdminServicePrincipal_id': $(app_globalAdminServicePrincipal_id) - 'app_globalAdminServicePrincipal_secret': $(app_globalAdminServicePrincipal_secret) - 'app_nativeClient_id': $(app_nativeClient_id) - 'app_nativeClient_secret': $(app_nativeClient_secret) - 'app_wrongAudienceClient_id': $(app_wrongAudienceClient_id) - 'app_wrongAudienceClient_secret': $(app_wrongAudienceClient_secret) - 'app_globalAdminUserApp_id': $(app_globalAdminUserApp_id) - 'app_globalAdminUserApp_secret': $(app_globalAdminUserApp_secret) - 'app_globalConverterUserApp_id': $(app_globalConverterUserApp_id) - 'app_globalConverterUserApp_secret': $(app_globalConverterUserApp_secret) - 'app_globalExporterUserApp_id': $(app_globalExporterUserApp_id) - 'app_globalExporterUserApp_secret': $(app_globalExporterUserApp_secret) - 'app_globalImporterUserApp_id': $(app_globalImporterUserApp_id) - 'app_globalImporterUserApp_secret': $(app_globalImporterUserApp_secret) - 'app_globalReaderUserApp_id': $(app_globalReaderUserApp_id) - 'app_globalReaderUserApp_secret': $(app_globalReaderUserApp_secret) - 'app_globalWriterUserApp_id': $(app_globalWriterUserApp_id) - 'app_globalWriterUserApp_secret': $(app_globalWriterUserApp_secret) - 'app_smartUserClient_id': $(app_smartUserClient_id) - 'app_smartUserClient_secret': $(app_smartUserClient_secret) - 'AZURESUBSCRIPTION_CLIENT_ID': $(AzurePipelinesCredential_ClientId) - 'AZURESUBSCRIPTION_TENANT_ID': $(AZURESUBSCRIPTION_TENANT_ID) - 'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': $(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID) - 'SYSTEM_ACCESSTOKEN': $(System.AccessToken) - platformOptions__resultDirectory: '$(Agent.TempDirectory)/testresults' - - - task: PublishTestResults@2 - displayName: 'Publish test results' - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '$(Agent.TempDirectory)/testresults/**/*.trx' - mergeTestResults: true - testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}' - # TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed - failTaskOnFailedTests: false - condition: succeededOrFailed() diff --git a/build/jobs/provision-deploy.yml b/build/jobs/provision-deploy.yml deleted file mode 100644 index 2186eb5abd..0000000000 --- a/build/jobs/provision-deploy.yml +++ /dev/null @@ -1,239 +0,0 @@ -parameters: -- name: version - type: string -- name: sql - type: boolean - default: false -- name: webAppName - type: string -- name: appServicePlanName - type: string -- name: appServicePlanResourceGroup - type: string -- name: subscription - type: string -- name: resourceGroup - type: string -- name: testEnvironmentUrl - type: string -- name: imageTag - type: string -- name: schemaAutomaticUpdatesEnabled - type: string - default: 'tool' -- name: sqlServerName - type: string - default: '' -- name: sqlComputeTier - type: string - default: 'Standard' -- name: reindexEnabled - type: boolean - default: true -- name: keyVaultName - type: string - default: '' - -jobs: -- job: provisionEnvironment - pool: - name: '$(SharedLinuxPool)' - vmImage: '$(LinuxVmImage)' - steps: - - task: AzureKeyVault@1 - displayName: 'Azure Key Vault: resolute-oss-tenant-info' - inputs: - azureSubscription: $(ConnectedServiceName) - KeyVaultName: 'resolute-oss-tenant-info' - - - task: AzurePowerShell@5 - displayName: 'Azure PowerShell script: InlineScript' - retryCountOnTaskFailure: 1 - inputs: - azureSubscription: $(ConnectedServiceName) - azurePowerShellVersion: latestVersion - pwsh: true - ScriptType: inlineScript - Inline: | - Add-Type -AssemblyName System.Web - - # Import retry helper for transient error handling - . $(System.DefaultWorkingDirectory)/release/scripts/PowerShell/Invoke-WithRetry.ps1 - - $deployPath = "$(System.DefaultWorkingDirectory)/test/Configuration" - - $testConfig = (ConvertFrom-Json (Get-Content -Raw "$deployPath/testconfiguration.json")) - $flattenedTestConfig = $(System.DefaultWorkingDirectory)/release/scripts/PowerShell/ConvertTo-FlattenedConfigurationHashtable.ps1 -InputObject $testConfig - - $additionalProperties = $flattenedTestConfig - - $additionalProperties["SqlServer__DeleteAllDataOnStartup"] = "false" - $additionalProperties["SqlServer__AllowDatabaseCreation"] = "true" - # Cosmos DB autoscale is configured in the ARM template (10,000 RU max) - $additionalProperties["TaskHosting__PollingFrequencyInSeconds"] = 1 - $additionalProperties["FhirServer__CoreFeatures__SearchParameterCacheRefreshIntervalSeconds"] = 2 - $additionalProperties["ASPNETCORE_FORWARDEDHEADERS_ENABLED"] = "true" - - $webAppName = "${{ parameters.webAppName }}".ToLower() - Write-Host "Web App Name: $webAppName" - $templateParameters = @{ - fhirVersion = "${{ parameters.version }}" - appServicePlanName = "${{ parameters.appServicePlanName }}" - appServicePlanSku = "P0V3" - numberOfInstances = 3 - serviceName = $webAppName - keyVaultName = "${{ parameters.keyVaultName }}".ToLower() - securityAuthenticationAuthority = "https://sts.windows.net/$(tenant-id-guid)" - securityAuthenticationAudience = "${{ parameters.testEnvironmentUrl }}" - additionalFhirServerConfigProperties = $additionalProperties - enableAadSmartOnFhirProxy = $true - enableExport = $true - enableConvertData = $true - enableImport = $true - backgroundTaskCount = 2 - enableReindex = if ("${{ parameters.reindexEnabled }}" -eq "true") { $true } else { $false } - registryName = '$(azureContainerRegistry)' - imageTag = '${{ parameters.imageTag }}' - acrPullUserAssignedManagedIdentityResourceId = '/subscriptions/a1766500-6fd5-4f5c-8515-607798271014/resourceGroups/HealthPlatformRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-uami' - acrPullUserAssignedManagedIdentityClientId = 'e4acf07d-0124-4373-9ea4-bf5c53af647d' - isNspExisting = $true - } - - if("${{ parameters.sql }}" -eq "true"){ - # Set SQL Variables - $templateParameters["solutionType"] = "FhirServerSqlServer" - $templateParameters["sqlServerName"] = "${{parameters.sqlServerName}}".ToLower() - $templateParameters["sqlServerNewOrExisting"] = "existing" - $templateParameters["sqlSchemaAutomaticUpdatesEnabled"] = "${{parameters.schemaAutomaticUpdatesEnabled}}" - $templateParameters["sqlDatabaseComputeTier"] = "${{parameters.sqlComputeTier}}" - } - - $deploymentName = $webAppName - $resourceGroupName = "${{ parameters.resourceGroup }}" - - Write-Host "Provisioning Resource Group" - Write-Host "ResourceGroupName: ${{ parameters.resourceGroup }}" - - # Set NSP name to match ARM template convention - $nspName = "nsp-$resourceGroupName" - $templateParameters["networkSecurityPerimeterName"] = $nspName - - # Check if a deployment with the specified name already exists - $existingDeployment = Get-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -ErrorAction SilentlyContinue - - # If the deployment exists and is ongoing, wait for it to complete - if ($existingDeployment -and ($existingDeployment.ProvisioningState -eq "Running" -or $existingDeployment.ProvisioningState -eq "Accepted")) { - Write-Host "Waiting for existing deployment '$deploymentName' to complete..." - do { - Start-Sleep -Seconds 10 - $existingDeployment = Get-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName - } while ($existingDeployment.ProvisioningState -eq "Running" -or $existingDeployment.ProvisioningState -eq "Accepted") - Write-Host "Existing deployment completed." - } - else - { - try { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/default-azuredeploy-docker.json -TemplateParameterObject $templateParameters -Verbose - } catch { - if ("${{ parameters.sql }}" -eq "true") { - $templateParameters["sqlServerNewOrExisting"] = "new" - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/default-azuredeploy-docker.json -TemplateParameterObject $templateParameters -Verbose - } - } - } - - if("${{ parameters.sql }}" -eq "false"){ - Write-Host "Add DevOps MI permission to Cosmos database" - - $account = Get-AzContext - - # Retry Get-AzADServicePrincipal for AAD replication delays - $principalId = Invoke-WithRetry -OperationName "Get Service Principal" -ScriptBlock { - (Get-AzADServicePrincipal -ApplicationId $account.Account.Id -ErrorAction Stop).Id - } - - # Retry CosmosDB role assignment for resource propagation delays - Invoke-WithRetry -OperationName "CosmosDB Role Assignment" -ScriptBlock { - New-AzCosmosDBSqlRoleAssignment ` - -AccountName $webAppName ` - -ResourceGroupName $resourceGroupName ` - -Scope "/" ` - -PrincipalId $principalId ` - -RoleDefinitionId "00000000-0000-0000-0000-000000000002" ` - -ErrorAction Stop - } - - # Associate CosmosDB with Network Security Perimeter using external template - Write-Host "Associating CosmosDB with Network Security Perimeter: $nspName" - - $associationName = "cosmosdb-$webAppName-association" - - Write-Host "Creating NSP association using external ARM template..." - Write-Host "NSP Name: $nspName" - Write-Host "CosmosDB Account: $webAppName" - Write-Host "Association Name: $associationName" - - $nspTemplateParameters = @{ - 'nspName' = $nspName - 'resourceName' = $webAppName - 'resourceType' = 'Microsoft.DocumentDB/databaseAccounts' - 'associationName' = $associationName - 'accessMode' = 'Learning' - 'nspProfile' = 'default' - } - - # Retry NSP association deployment for resource propagation delays - $nspDeploymentResult = Invoke-WithRetry -OperationName "NSP Association Deployment" -ScriptBlock { - New-AzResourceGroupDeployment ` - -ResourceGroupName $resourceGroupName ` - -Name "$webAppName-cosmosdb-nsp-association" ` - -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/nsp-resource-association.json ` - -TemplateParameterObject $nspTemplateParameters ` - -Verbose ` - -ErrorAction Stop - } - - Write-Host "Successfully associated CosmosDB with NSP using external ARM template" - Write-Host "Association Resource ID: $($nspDeploymentResult.Outputs.associationResourceId.Value)" - } - - # Health check — inlined so a retryCountOnTaskFailure on this task retries the full - # provisioning + health check together, covering race conditions where Cosmos comes up - # in a bad state and the app never becomes healthy. - $healthCheckUrl = "https://$webAppName.azurewebsites.net/health/check" - $timeoutMinutes = 5 - $consecutiveSuccessRequired = 5 - $consecutiveSuccessCount = 0 - $startTime = Get-Date - $timeoutTime = $startTime.AddMinutes($timeoutMinutes) - - Write-Host "Starting health check for $healthCheckUrl" - Write-Host "Timeout: $timeoutMinutes minutes, consecutive successes required: $consecutiveSuccessRequired" - - Do { - Start-Sleep -s 5 - Write-Host "Checking: $healthCheckUrl (consecutive successes: $consecutiveSuccessCount/$consecutiveSuccessRequired)" - try { - $response = Invoke-WebRequest -URI $healthCheckUrl - if ($response.StatusCode -eq 200) { - $consecutiveSuccessCount++ - Write-Host "Success ($consecutiveSuccessCount/$consecutiveSuccessRequired)" - } else { - Write-Host "Non-200 response ($($response.StatusCode)), resetting count" - $consecutiveSuccessCount = 0 - } - } catch { - Write-Host $PSItem.Exception.Message - $consecutiveSuccessCount = 0 - } finally { - $Error.Clear() - } - if ((Get-Date) -gt $timeoutTime) { - $elapsed = [math]::Round(((Get-Date) - $startTime).TotalMinutes, 2) - Write-Error "Health check timed out after $elapsed minutes ($consecutiveSuccessCount/$consecutiveSuccessRequired consecutive successes)" - throw "Health check timed out" - } - } While ($consecutiveSuccessCount -lt $consecutiveSuccessRequired) - - $elapsed = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 2) - Write-Host "Health check passed with $consecutiveSuccessRequired consecutive successes after $elapsed seconds" diff --git a/build/jobs/run-cosmos-tests.yml b/build/jobs/run-cosmos-tests.yml deleted file mode 100644 index 7a6aca889c..0000000000 --- a/build/jobs/run-cosmos-tests.yml +++ /dev/null @@ -1,131 +0,0 @@ -parameters: -- name: version - type: string -- name: keyVaultName - type: string -- name: appServiceName - type: string -jobs: - -- job: "CosmosIntegrationTests" - timeoutInMinutes: 75 - pool: - name: '$(SharedLinuxPool)' - vmImage: '$(LinuxVmImage)' - steps: - - checkout: self - fetchDepth: 1 - fetchTags: false - path: source - - - template: integration-setup.yml - - - template: integration-tests-extract.yml - parameters: - version: ${{ parameters.version }} - - - task: AzureKeyVault@1 - displayName: 'Azure Key Vault: ${{ parameters.keyVaultName }}' - inputs: - azureSubscription: $(ConnectedServiceName) - KeyVaultName: '${{ parameters.keyVaultName }}' - - - task: AzurePowerShell@5 - displayName: 'Set Workload Identity Variables' - inputs: - azureSubscription: $(ConnectedServiceName) - azurePowerShellVersion: latestVersion - pwsh: true - ScriptType: inlineScript - Inline: | - Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_CLIENT_ID]$env:AZURESUBSCRIPTION_CLIENT_ID" - Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_TENANT_ID]$env:AZURESUBSCRIPTION_TENANT_ID" - Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_SERVICE_CONNECTION_ID]$env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" - - $appServiceName = '${{ parameters.appServiceName }}' - $appSettings = (Get-AzWebApp -ResourceGroupName $(UniqueResourceGroupName) -Name $appServiceName).SiteConfig.AppSettings - $dataStoreResourceId = $appSettings | where {$_.Name -eq "FhirServer__ResourceManager__DataStoreResourceId"} - $dataStoreResourceId = $dataStoreResourceId[0].Value - Write-Host "$dataStoreResourceId" - Write-Host "##vso[task.setvariable variable=DataStoreResourceId]$($dataStoreResourceId)" - - - task: DotNetCoreCLI@2 - displayName: 'Build Integration Test Projects' - inputs: - command: build - projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' - arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)' - - - task: DotNetCoreCLI@2 - displayName: 'Run Cosmos Integration Tests with coverage' - inputs: - command: test - projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter "FullyQualifiedName!~SqlServer" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' - testRunTitle: '${{ parameters.version }} Cosmos Integration Tests' - env: - 'CosmosDb__Host': $(CosmosDb--Host) - 'FhirServer__ResourceManager__DataStoreResourceId': '$(DataStoreResourceId)' - 'CosmosDb__UseManagedIdentity': true - platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage' - 'AZURESUBSCRIPTION_CLIENT_ID': '$(AZURESUBSCRIPTION_CLIENT_ID)' - 'AZURESUBSCRIPTION_TENANT_ID': '$(AZURESUBSCRIPTION_TENANT_ID)' - 'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)' - 'SYSTEM_ACCESSTOKEN': $(System.AccessToken) - - - task: PublishTestResults@2 - displayName: 'Publish integration test results' - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx' - mergeTestResults: true - testRunTitle: '${{ parameters.version }} Cosmos Integration Tests' - # TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed - failTaskOnFailedTests: false - condition: succeededOrFailed() - - - task: reportgenerator@5 - displayName: 'Aggregate Cosmos integration test coverage' - condition: succeededOrFailed() - inputs: - reports: '$(Agent.TempDirectory)/coverage/**/*.cobertura.xml' - reporttypes: 'Cobertura' - targetdir: '$(Agent.TempDirectory)/coverage-aggregated' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Cosmos integration test coverage' - inputs: - pathToPublish: '$(Agent.TempDirectory)/coverage-aggregated' - artifactName: 'Coverage_IntegrationTests_Cosmos_${{ parameters.version }}' - artifactType: 'container' - -- job: 'cosmosE2eTests' - timeoutInMinutes: 75 - dependsOn: [] - pool: - name: '$(SharedLinuxPool)' - vmImage: '$(LinuxVmImage)' - steps: - - template: e2e-setup.yml - - template: e2e-tests.yml - parameters: - version: ${{ parameters.version }} - appServiceName: ${{ parameters.appServiceName }} - appServiceType: 'CosmosDb' - categoryFilter: 'Category!=ExportLongRunning&Category!=IndexAndReindex' - -- job: 'cosmosE2eTests_Reindex' - timeoutInMinutes: 75 - dependsOn: [] - pool: - name: '$(SharedLinuxPool)' - vmImage: '$(LinuxVmImage)' - steps: - - template: e2e-setup.yml - - template: e2e-tests.yml - parameters: - version: ${{ parameters.version }} - appServiceName: ${{ parameters.appServiceName }} - appServiceType: 'CosmosDb' - categoryFilter: 'Category=IndexAndReindex' - testRunTitleSuffix: ' Reindex' diff --git a/build/jobs/run-sql-tests.yml b/build/jobs/run-sql-tests.yml deleted file mode 100644 index 57688219e3..0000000000 --- a/build/jobs/run-sql-tests.yml +++ /dev/null @@ -1,140 +0,0 @@ -parameters: -- name: version - type: string -- name: keyVaultName - type: string -- name: appServiceName - type: string -- name: integrationSqlServerName - type: string -jobs: - -- job: "SqlIntegrationTests" - timeoutInMinutes: 75 - pool: - name: '$(SharedLinuxPool)' - vmImage: '$(LinuxVmImage)' - steps: - - checkout: self - fetchDepth: 1 - fetchTags: false - path: source - - - template: integration-setup.yml - - - template: integration-tests-extract.yml - parameters: - version: ${{ parameters.version }} - - - task: AzureKeyVault@1 - displayName: 'Azure Key Vault: ${{ parameters.keyVaultName }}-sql' - inputs: - azureSubscription: $(ConnectedServiceName) - KeyVaultName: '${{ parameters.keyVaultName }}-sql' - - - task: AzurePowerShell@5 - displayName: 'Set Workload Identity Variables' - inputs: - azureSubscription: $(ConnectedServiceName) - azurePowerShellVersion: latestVersion - pwsh: true - ScriptType: inlineScript - Inline: | - Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_CLIENT_ID]$env:AZURESUBSCRIPTION_CLIENT_ID" - Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_TENANT_ID]$env:AZURESUBSCRIPTION_TENANT_ID" - Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_SERVICE_CONNECTION_ID]$env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" - - - task: DotNetCoreCLI@2 - displayName: 'Build Integration Test Projects' - inputs: - command: build - projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' - arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)' - - - task: DotNetCoreCLI@2 - displayName: 'Run SQL Integration Tests with coverage' - inputs: - command: test - projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj' - arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter "FullyQualifiedName!~CosmosDb" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx' - testRunTitle: '${{ parameters.version }} SQL Integration Tests' - env: - 'SqlServer:ConnectionString': 'Server=tcp:${{ parameters.integrationSqlServerName }}.database.windows.net,1433;Initial Catalog=master;Persist Security Info=False;Authentication=ActiveDirectoryWorkloadIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=$(AZURESUBSCRIPTION_CLIENT_ID);' - platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage' - 'AZURESUBSCRIPTION_CLIENT_ID': '$(AZURESUBSCRIPTION_CLIENT_ID)' - 'AZURESUBSCRIPTION_TENANT_ID': '$(AZURESUBSCRIPTION_TENANT_ID)' - 'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)' - 'SYSTEM_ACCESSTOKEN': $(System.AccessToken) - - - task: PublishTestResults@2 - displayName: 'Publish integration test results' - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx' - mergeTestResults: true - testRunTitle: '${{ parameters.version }} SQL Integration Tests' - # TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed - failTaskOnFailedTests: false - condition: succeededOrFailed() - - - task: reportgenerator@5 - displayName: 'Aggregate SQL integration test coverage' - condition: succeededOrFailed() - inputs: - reports: '$(Agent.TempDirectory)/coverage/**/*.cobertura.xml' - reporttypes: 'Cobertura' - targetdir: '$(Agent.TempDirectory)/coverage-aggregated' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish SQL integration test coverage' - inputs: - pathToPublish: '$(Agent.TempDirectory)/coverage-aggregated' - artifactName: 'Coverage_IntegrationTests_Sql_${{ parameters.version }}' - artifactType: 'container' - -- job: 'sqlE2eTests' - timeoutInMinutes: 75 - dependsOn: [] - pool: - name: '$(SharedLinuxPool)' - vmImage: '$(LinuxVmImage)' - steps: - - template: e2e-setup.yml - - template: e2e-tests.yml - parameters: - version: ${{ parameters.version }} - appServiceName: '${{ parameters.appServiceName }}-sql' - appServiceType: 'SqlServer' - categoryFilter: 'Category!=ExportLongRunning&Category!=IndexAndReindex&Category!=BulkUpdate' - -- job: 'sqlE2eTests_Reindex' - timeoutInMinutes: 75 - dependsOn: [] - pool: - name: '$(SharedLinuxPool)' - vmImage: '$(LinuxVmImage)' - steps: - - template: e2e-setup.yml - - template: e2e-tests.yml - parameters: - version: ${{ parameters.version }} - appServiceName: '${{ parameters.appServiceName }}-sql' - appServiceType: 'SqlServer' - categoryFilter: 'Category=IndexAndReindex' - testRunTitleSuffix: ' Reindex' - -- job: 'sqlE2eTests_BulkUpdate' - timeoutInMinutes: 75 - dependsOn: [] - pool: - name: '$(SharedLinuxPool)' - vmImage: '$(LinuxVmImage)' - steps: - - template: e2e-setup.yml - - template: e2e-tests.yml - parameters: - version: ${{ parameters.version }} - appServiceName: '${{ parameters.appServiceName }}-sql' - appServiceType: 'SqlServer' - categoryFilter: 'Category=BulkUpdate' - testRunTitleSuffix: ' BulkUpdate' diff --git a/build/tasks/e2e-set-variables.yml b/build/tasks/e2e-set-variables.yml deleted file mode 100644 index 480902fc65..0000000000 --- a/build/tasks/e2e-set-variables.yml +++ /dev/null @@ -1,78 +0,0 @@ -parameters: -- name: appServiceName - type: string - -steps: -- task: AzurePowerShell@5 - displayName: 'Set Variables' - inputs: - azureSubscription: $(ConnectedServiceName) - azurePowerShellVersion: latestVersion - pwsh: true - ScriptType: InlineScript - Inline: | - $keyVault = "$(KeyVaultBaseName)-ts" - $secrets = Get-AzKeyVaultSecret -VaultName $keyVault - - foreach($secret in $secrets) - { - $environmentVariableName = $secret.Name.Replace("--","_") - - $secretValue = Get-AzKeyVaultSecret -VaultName $keyVault -Name $secret.Name - # Replace with -AsPlainText flag when v5.3 of the Az Module is supported - $plainValue = ([System.Net.NetworkCredential]::new("", $secretValue.SecretValue).Password).ToString() - if([string]::IsNullOrEmpty($plainValue)) - { - throw "$($secret.Name) is empty" - } - Write-Host "##vso[task.setvariable variable=$($environmentVariableName)]$($plainValue)" - } - - $appServiceName = "${{ parameters.appServiceName }}" - $appSettings = (Get-AzWebApp -ResourceGroupName $(UniqueResourceGroupName) -Name $appServiceName).SiteConfig.AppSettings - $acrSettings = $appSettings | where {$_.Name -eq "FhirServer__Operations__ConvertData__ContainerRegistryServers__0"} - $acrLoginServer = $acrSettings[0].Value - $acrAccountName = ($acrLoginServer -split '\.')[0] - - ## This needs to be moved to MI, WI #125246 - # $acrPassword = (Get-AzContainerRegistryCredential -ResourceGroupName $(ResourceGroupName) -Name $acrAccountName).Password - # Write-Host "##vso[task.setvariable variable=TestContainerRegistryServer]$($acrLoginServer)" - # Write-Host "##vso[task.setvariable variable=TestContainerRegistryPassword]$($acrPassword)" - - $exportStoreSettings = $appSettings | where {$_.Name -eq "FhirServer__Operations__Export__StorageAccountUri"} - $exportStoreUri = $exportStoreSettings[0].Value - Write-Host "$exportStoreUri" - $exportStoreAccountName = [System.Uri]::new("$exportStoreUri").Host.Split('.')[0] - $exportStoreKey = Get-AzStorageAccountKey -ResourceGroupName $(UniqueResourceGroupName) -Name "$exportStoreAccountName" | Where-Object {$_.KeyName -eq "key1"} - - Write-Host "##vso[task.setvariable variable=TestExportStoreUri]$($exportStoreUri)" - - $integrationStoreSettings = $appSettings | where {$_.Name -eq "FhirServer__Operations__IntegrationDataStore__StorageAccountUri"} - $integrationStoreUri = $integrationStoreSettings[0].Value - Write-Host "$integrationStoreUri" - $integrationStoreAccountName = [System.Uri]::new("$integrationStoreUri").Host.Split('.')[0] - Write-Host "##vso[task.setvariable variable=TestIntegrationStoreUri]$($integrationStoreUri)" - - Write-Host "##vso[task.setvariable variable=Resource]$(TestApplicationResource)" - - $secrets = Get-AzKeyVaultSecret -VaultName resolute-oss-tenant-info - foreach($secret in $secrets) - { - $environmentVariableName = $secret.Name.Replace("--","_") - - $secretValue = Get-AzKeyVaultSecret -VaultName resolute-oss-tenant-info -Name $secret.Name - # Replace with -AsPlainText flag when v5.3 of the Az Module is supported - $plainValue = ([System.Net.NetworkCredential]::new("", $secretValue.SecretValue).Password).ToString() - if([string]::IsNullOrEmpty($plainValue)) - { - throw "$($secret.Name) is empty" - } - Write-Host "##vso[task.setvariable variable=$($environmentVariableName)]$($plainValue)" - } - # ---------------------------------------- - - dotnet dev-certs https - - Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_CLIENT_ID]$env:AZURESUBSCRIPTION_CLIENT_ID" - Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_TENANT_ID]$env:AZURESUBSCRIPTION_TENANT_ID" - Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_SERVICE_CONNECTION_ID]$env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" From 4dff9cb41b59836e0b997f41b50d8557a940d945 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 1 Apr 2026 16:03:47 -0500 Subject: [PATCH 31/38] removing aca from the app names --- build/ci-variables.yml | 2 +- build/jobs/provision-deploy-aca.yml | 2 +- build/jobs/provision-deploy-cosmos-aca.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/ci-variables.yml b/build/ci-variables.yml index 2b5610c768..caad3bc945 100644 --- a/build/ci-variables.yml +++ b/build/ci-variables.yml @@ -1,7 +1,7 @@ variables: ResourceGroupRegion: 'southcentralus' resourceGroupRoot: 'msh-fhir-ci-aca' - DeploymentEnvironmentName: '$(resourceGroupRoot)' + DeploymentEnvironmentName: 'msh-fhir-ci' AcaEnvironmentName: '$(DeploymentEnvironmentName)-acae' # CI uses the same name for Key Vaults as the deployment environment (no BuildId suffix needed) KeyVaultBaseName: '$(DeploymentEnvironmentName)' diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index 228143e58f..42c079b345 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -147,7 +147,7 @@ jobs: additionalEnvVars = $additionalEnvVars } - $deploymentName = "$webAppName-aca" + $deploymentName = "$webAppName" $resourceGroupName = "${{ parameters.resourceGroup }}" $sqlServerName = "${{ parameters.sqlServerName }}".ToLowerInvariant() diff --git a/build/jobs/provision-deploy-cosmos-aca.yml b/build/jobs/provision-deploy-cosmos-aca.yml index ef91db54c3..f8ccdf4285 100644 --- a/build/jobs/provision-deploy-cosmos-aca.yml +++ b/build/jobs/provision-deploy-cosmos-aca.yml @@ -190,7 +190,7 @@ jobs: additionalEnvVars = $additionalEnvVars } - $deploymentName = "$webAppName-aca" + $deploymentName = "$webAppName" Write-Host "Provisioning ACA resources" Write-Host "ResourceGroupName: $resourceGroupName" From fdd6a3a805a3a09cdd3e4aec9a036062fcc5550b Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 1 Apr 2026 16:09:31 -0500 Subject: [PATCH 32/38] Fixing CI RG name to remove -aca --- build/ci-variables.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/ci-variables.yml b/build/ci-variables.yml index caad3bc945..c2ea660cf2 100644 --- a/build/ci-variables.yml +++ b/build/ci-variables.yml @@ -1,7 +1,7 @@ variables: ResourceGroupRegion: 'southcentralus' - resourceGroupRoot: 'msh-fhir-ci-aca' - DeploymentEnvironmentName: 'msh-fhir-ci' + resourceGroupRoot: 'msh-fhir-ci' + DeploymentEnvironmentName: '$(resourceGroupRoot)' AcaEnvironmentName: '$(DeploymentEnvironmentName)-acae' # CI uses the same name for Key Vaults as the deployment environment (no BuildId suffix needed) KeyVaultBaseName: '$(DeploymentEnvironmentName)' From bd3e37b66b1f61593e6179f606269c3615804471 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Wed, 1 Apr 2026 18:14:32 -0500 Subject: [PATCH 33/38] Refactor FHIR ACA templates: remove Cosmos DB JSON template, add SQL Bicep template, and create common Bicep module for shared resources --- build/ci-deploy.yml | 2 +- build/jobs/provision-deploy-aca.yml | 8 +- build/jobs/provision-deploy-cosmos-aca.yml | 2 +- build/pr-pipeline.yml | 6 +- samples/templates/aca/aca-environment.bicep | 8 + samples/templates/aca/aca-environment.json | 22 - samples/templates/aca/fhir-cosmos-aca.bicep | 141 ++++++ samples/templates/aca/fhir-cosmos-aca.json | 394 ----------------- samples/templates/aca/fhir-sql-aca.bicep | 154 +++++++ samples/templates/aca/fhir-sql-aca.json | 402 ------------------ .../aca/modules/fhir-aca-common.bicep | 267 ++++++++++++ 11 files changed, 574 insertions(+), 832 deletions(-) create mode 100644 samples/templates/aca/aca-environment.bicep delete mode 100644 samples/templates/aca/aca-environment.json create mode 100644 samples/templates/aca/fhir-cosmos-aca.bicep delete mode 100644 samples/templates/aca/fhir-cosmos-aca.json create mode 100644 samples/templates/aca/fhir-sql-aca.bicep delete mode 100644 samples/templates/aca/fhir-sql-aca.json create mode 100644 samples/templates/aca/modules/fhir-aca-common.bicep diff --git a/build/ci-deploy.yml b/build/ci-deploy.yml index 89c48c58f4..f397adb6bb 100644 --- a/build/ci-deploy.yml +++ b/build/ci-deploy.yml @@ -93,7 +93,7 @@ stages: New-AzResourceGroupDeployment ` -Name "aca-env-$envName" ` -ResourceGroupName $rgName ` - -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/aca-environment.json ` + -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/aca-environment.bicep ` -environmentName $envName ` -location $location ` -Verbose -ErrorAction Stop diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index 42c079b345..a026734bc7 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -1,9 +1,6 @@ parameters: - name: version type: string -- name: sql - type: boolean - default: true - name: webAppName type: string - name: acaEnvironmentName @@ -21,9 +18,6 @@ parameters: default: 'auto' - name: sqlServerName type: string -- name: sqlComputeTier - type: string - default: 'Standard' - name: reindexEnabled type: boolean default: true @@ -173,7 +167,7 @@ jobs: } $deploymentResult = Invoke-WithRetry -OperationName "ACA Deployment" -ScriptBlock { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/fhir-sql-aca.json -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/fhir-sql-aca.bicep -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop } $managedEnvironment = Get-AzResource -ResourceGroupName $resourceGroupName -ResourceType "Microsoft.App/managedEnvironments" -Name $acaEnvironmentName -ApiVersion "2023-05-01" -ExpandProperties diff --git a/build/jobs/provision-deploy-cosmos-aca.yml b/build/jobs/provision-deploy-cosmos-aca.yml index f8ccdf4285..f52fa92daa 100644 --- a/build/jobs/provision-deploy-cosmos-aca.yml +++ b/build/jobs/provision-deploy-cosmos-aca.yml @@ -199,7 +199,7 @@ jobs: Write-Host "CosmosDbAccountName: $cosmosDbAccountName" $deploymentResult = Invoke-WithRetry -OperationName "ACA Deployment" -ScriptBlock { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/fhir-cosmos-aca.json -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $resourceGroupName -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/fhir-cosmos-aca.bicep -TemplateParameterObject $templateParameters -Verbose -ErrorAction Stop } # Assign Cosmos DB Data Contributor role to the container app's system-assigned identity diff --git a/build/pr-pipeline.yml b/build/pr-pipeline.yml index 0b7dba1946..631b93fbdf 100644 --- a/build/pr-pipeline.yml +++ b/build/pr-pipeline.yml @@ -213,7 +213,7 @@ stages: New-AzResourceGroupDeployment ` -Name "aca-env-$envName" ` -ResourceGroupName $rgName ` - -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/aca-environment.json ` + -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/aca/aca-environment.bicep ` -environmentName $envName ` -location $location ` -Verbose -ErrorAction Stop @@ -294,7 +294,6 @@ stages: - template: ./jobs/provision-deploy-aca.yml parameters: version: Stu3 - sql: true webAppName: $(DeploymentEnvironmentNameSql) acaEnvironmentName: $(AcaEnvironmentName) keyVaultName: $(KeyVaultNameSql) @@ -335,7 +334,6 @@ stages: - template: ./jobs/provision-deploy-aca.yml parameters: version: R4 - sql: true webAppName: $(DeploymentEnvironmentNameR4Sql) acaEnvironmentName: $(AcaEnvironmentName) keyVaultName: $(KeyVaultNameR4Sql) @@ -376,7 +374,6 @@ stages: - template: ./jobs/provision-deploy-aca.yml parameters: version: R4B - sql: true webAppName: $(DeploymentEnvironmentNameR4BSql) acaEnvironmentName: $(AcaEnvironmentName) keyVaultName: $(KeyVaultNameR4BSql) @@ -417,7 +414,6 @@ stages: - template: ./jobs/provision-deploy-aca.yml parameters: version: R5 - sql: true webAppName: $(DeploymentEnvironmentNameR5Sql) acaEnvironmentName: $(AcaEnvironmentName) keyVaultName: $(KeyVaultNameR5Sql) diff --git a/samples/templates/aca/aca-environment.bicep b/samples/templates/aca/aca-environment.bicep new file mode 100644 index 0000000000..8a95376068 --- /dev/null +++ b/samples/templates/aca/aca-environment.bicep @@ -0,0 +1,8 @@ +param environmentName string +param location string = resourceGroup().location + +resource managedEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' = { + name: environmentName + location: location + properties: {} +} diff --git a/samples/templates/aca/aca-environment.json b/samples/templates/aca/aca-environment.json deleted file mode 100644 index 10e784cf81..0000000000 --- a/samples/templates/aca/aca-environment.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "environmentName": { - "type": "string" - }, - "location": { - "type": "string", - "defaultValue": "[resourceGroup().location]" - } - }, - "resources": [ - { - "type": "Microsoft.App/managedEnvironments", - "apiVersion": "2023-05-01", - "name": "[parameters('environmentName')]", - "location": "[parameters('location')]", - "properties": {} - } - ] -} diff --git a/samples/templates/aca/fhir-cosmos-aca.bicep b/samples/templates/aca/fhir-cosmos-aca.bicep new file mode 100644 index 0000000000..6205a20ab5 --- /dev/null +++ b/samples/templates/aca/fhir-cosmos-aca.bicep @@ -0,0 +1,141 @@ +// FHIR Server on ACA with Cosmos DB datastore. + +// ────────────────────────────────────────────── +// Parameters +// ────────────────────────────────────────────── + +@description('Name of the Azure Container App hosting FHIR.') +param containerAppName string + +@description('Name of the Container Apps Environment.') +param containerAppsEnvironmentName string + +@description('Name of the Key Vault used by FHIR server.') +@maxLength(24) +param keyVaultName string + +@description('FHIR version to deploy.') +@allowed(['Stu3', 'R4', 'R4B', 'R5']) +param fhirVersion string = 'R4' + +@description('Container registry server host name.') +param registryName string + +@description('Docker image tag.') +param imageTag string = 'latest' + +@description('Name of the existing Cosmos DB account.') +param cosmosDbAccountName string + +@description('Authority URL for AAD authentication.') +param securityAuthenticationAuthority string = '' + +@description('Audience for AAD authentication.') +param securityAuthenticationAudience string = '' + +@description('Resource ID of UAMI used to pull from ACR.') +param acrPullUserAssignedManagedIdentityResourceId string + +@description('Enable export operations.') +param enableExport bool = true + +@description('Enable import operations.') +param enableImport bool = true + +@description('Enable ConvertData operations.') +param enableConvertData bool = true + +@description('Enable reindex operations.') +param enableReindex bool = true + +@description('Minimum number of replicas.') +param minReplicas int = 0 + +@description('Maximum number of replicas.') +param maxReplicas int = 8 + +@description('CPU cores per container.') +param containerCpu string = '0.5' + +@description('Memory per container.') +param containerMemory string = '1Gi' + +@description('Additional environment variables from pipeline.') +param additionalEnvVars array = [] + +// ────────────────────────────────────────────── +// Variables +// ────────────────────────────────────────────── + +var normalizedCosmosDbAccountName = toLower(cosmosDbAccountName) + +var userAssignedIdentities = { + '${acrPullUserAssignedManagedIdentityResourceId}': {} +} + +var datastoreEnvVars = [ + { name: 'DataStore', value: 'CosmosDb' } + { name: 'CosmosDb__UseManagedIdentity', value: 'true' } + { name: 'CosmosDb__ContinuationTokenSizeLimitInKb', value: '1' } + { name: 'CosmosDb__UseQueueClientJobs', value: 'true' } + { + name: 'FhirServer__ResourceManager__DataStoreResourceId' + value: resourceId('Microsoft.DocumentDB/databaseAccounts', normalizedCosmosDbAccountName) + } +] + +// ────────────────────────────────────────────── +// Modules +// ────────────────────────────────────────────── + +module common 'modules/fhir-aca-common.bicep' = { + name: '${toLower(containerAppName)}-common' + params: { + containerAppName: containerAppName + containerAppsEnvironmentName: containerAppsEnvironmentName + keyVaultName: keyVaultName + fhirVersion: fhirVersion + registryName: registryName + imageTag: imageTag + securityAuthenticationAuthority: securityAuthenticationAuthority + securityAuthenticationAudience: securityAuthenticationAudience + acrPullUserAssignedManagedIdentityResourceId: acrPullUserAssignedManagedIdentityResourceId + enableExport: enableExport + enableImport: enableImport + enableConvertData: enableConvertData + enableReindex: enableReindex + minReplicas: minReplicas + maxReplicas: maxReplicas + containerCpu: containerCpu + containerMemory: containerMemory + additionalEnvVars: additionalEnvVars + datastoreEnvVars: datastoreEnvVars + userAssignedIdentities: userAssignedIdentities + } +} + +// ────────────────────────────────────────────── +// Cosmos-specific resources +// ────────────────────────────────────────────── + +resource existingCosmosDbAccount 'Microsoft.DocumentDB/databaseAccounts@2021-06-15' existing = { + name: normalizedCosmosDbAccountName +} + +resource cosmosDbHostSecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { + name: '${toLower(keyVaultName)}/CosmosDb--Host' + properties: { + contentType: 'text/plain' + value: existingCosmosDbAccount.properties.documentEndpoint + } +} + +// ────────────────────────────────────────────── +// Outputs +// ────────────────────────────────────────────── + +output containerAppName string = common.outputs.containerAppName +output containerAppFqdn string = common.outputs.containerAppFqdn +output containerAppUrl string = common.outputs.containerAppUrl +output exportStorageUri string = common.outputs.exportStorageUri +output integrationStorageUri string = common.outputs.integrationStorageUri diff --git a/samples/templates/aca/fhir-cosmos-aca.json b/samples/templates/aca/fhir-cosmos-aca.json deleted file mode 100644 index 0d00633ddc..0000000000 --- a/samples/templates/aca/fhir-cosmos-aca.json +++ /dev/null @@ -1,394 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "containerAppName": { - "type": "string", - "metadata": { - "description": "Name of the Azure Container App hosting FHIR." - } - }, - "containerAppsEnvironmentName": { - "type": "string", - "metadata": { - "description": "Name of the Container Apps Environment." - } - }, - "keyVaultName": { - "type": "string", - "maxLength": 24, - "metadata": { - "description": "Name of the Key Vault used by FHIR server." - } - }, - "fhirVersion": { - "type": "string", - "defaultValue": "R4", - "allowedValues": [ - "Stu3", - "R4", - "R4B", - "R5" - ] - }, - "registryName": { - "type": "string", - "metadata": { - "description": "Container registry server host name." - } - }, - "imageTag": { - "type": "string", - "defaultValue": "latest" - }, - "cosmosDbAccountName": { - "type": "string", - "metadata": { - "description": "Name of the existing Cosmos DB account." - } - }, - "securityAuthenticationAuthority": { - "type": "string", - "defaultValue": "" - }, - "securityAuthenticationAudience": { - "type": "string", - "defaultValue": "" - }, - "acrPullUserAssignedManagedIdentityResourceId": { - "type": "string", - "metadata": { - "description": "Resource ID of UAMI used to pull from ACR." - } - }, - "enableExport": { - "type": "bool", - "defaultValue": true - }, - "enableImport": { - "type": "bool", - "defaultValue": true - }, - "enableConvertData": { - "type": "bool", - "defaultValue": true - }, - "enableReindex": { - "type": "bool", - "defaultValue": true - }, - "minReplicas": { - "type": "int", - "defaultValue": 0 - }, - "maxReplicas": { - "type": "int", - "defaultValue": 8 - }, - "containerCpu": { - "type": "string", - "defaultValue": "0.5" - }, - "containerMemory": { - "type": "string", - "defaultValue": "1Gi" - }, - "additionalEnvVars": { - "type": "array", - "defaultValue": [] - } - }, - "variables": { - "isMAG": "[or(contains(resourceGroup().location,'usgov'),contains(resourceGroup().location,'usdod'))]", - "containerAppName": "[toLower(parameters('containerAppName'))]", - "containerAppsEnvironmentName": "[toLower(parameters('containerAppsEnvironmentName'))]", - "keyVaultName": "[toLower(parameters('keyVaultName'))]", - "cosmosDbAccountName": "[toLower(parameters('cosmosDbAccountName'))]", - "azureContainerRegistryUri": "[if(variables('isMAG'), '.azurecr.us', '.azurecr.io')]", - "imageRepositoryName": "[if(contains(parameters('registryName'),'mcr.'), concat(toLower(parameters('fhirVersion')), '-fhir-server'), concat(toLower(parameters('fhirVersion')), '_fhir-server'))]", - "containerImage": "[concat(parameters('registryName'), '/', variables('imageRepositoryName'), ':', parameters('imageTag'))]", - "blobStorageUri": "[if(variables('isMAG'), '.blob.core.usgovcloudapi.net', '.blob.core.windows.net')]", - "storageAccountName": "[concat(substring(replace(variables('containerAppName'), '-', ''), 0, min(11, length(replace(variables('containerAppName'), '-', '')))), uniquestring(resourceGroup().id, variables('containerAppName')))]", - "storageAccountUri": "[concat('https://', variables('storageAccountName'), variables('blobStorageUri'))]", - "keyVaultEndpoint": "[if(variables('isMAG'), concat('https://', variables('keyVaultName'), '.vault.usgovcloudapi.net/'), concat('https://', variables('keyVaultName'), '.vault.azure.net/'))]", - "enableIntegrationStore": "[or(parameters('enableExport'), parameters('enableImport'))]", - "storageBlobDataContributerRoleId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe')]", - "cosmosDataContributorRoleId": "00000000-0000-0000-0000-000000000002", - "staticEnvVars": [ - { - "name": "ASPNETCORE_FORWARDEDHEADERS_ENABLED", - "value": "true" - }, - { - "name": "KeyVault__Endpoint", - "value": "[variables('keyVaultEndpoint')]" - }, - { - "name": "FhirServer__Security__Enabled", - "value": "true" - }, - { - "name": "FhirServer__Security__EnableAadSmartOnFhirProxy", - "value": "true" - }, - { - "name": "FhirServer__Security__Authentication__Authority", - "value": "[parameters('securityAuthenticationAuthority')]" - }, - { - "name": "FhirServer__Security__Authentication__Audience", - "value": "[parameters('securityAuthenticationAudience')]" - }, - { - "name": "DataStore", - "value": "CosmosDb" - }, - { - "name": "CosmosDb__UseManagedIdentity", - "value": "true" - }, - { - "name": "CosmosDb__ContinuationTokenSizeLimitInKb", - "value": "1" - }, - { - "name": "CosmosDb__UseQueueClientJobs", - "value": "true" - }, - { - "name": "TaskHosting__Enabled", - "value": "true" - }, - { - "name": "TaskHosting__PollingFrequencyInSeconds", - "value": "1" - }, - { - "name": "TaskHosting__MaxRunningTaskCount", - "value": "2" - }, - { - "name": "FhirServer__CoreFeatures__SearchParameterCacheRefreshIntervalSeconds", - "value": "2" - }, - { - "name": "FhirServer__Operations__Export__Enabled", - "value": "[if(parameters('enableExport'), 'true', 'false')]" - }, - { - "name": "FhirServer__Operations__Export__StorageAccountUri", - "value": "[if(parameters('enableExport'), variables('storageAccountUri'), 'null')]" - }, - { - "name": "FhirServer__Operations__Import__Enabled", - "value": "[if(parameters('enableImport'), 'true', 'false')]" - }, - { - "name": "FhirServer__Operations__IntegrationDataStore__StorageAccountUri", - "value": "[if(parameters('enableImport'), variables('storageAccountUri'), 'null')]" - }, - { - "name": "FhirServer__Operations__ConvertData__Enabled", - "value": "[if(parameters('enableConvertData'), 'true', 'false')]" - }, - { - "name": "FhirServer__Operations__ConvertData__ContainerRegistryServers__0", - "value": "[if(parameters('enableConvertData'), parameters('registryName'), 'null')]" - }, - { - "name": "FhirServer__Operations__Reindex__Enabled", - "value": "[if(parameters('enableReindex'), 'true', 'false')]" - }, - { - "name": "FhirServer__ResourceManager__DataStoreResourceId", - "value": "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('cosmosDbAccountName'))]" - } - ] - }, - "resources": [ - { - "type": "Microsoft.App/containerApps", - "apiVersion": "2024-03-01", - "name": "[variables('containerAppName')]", - "location": "[resourceGroup().location]", - "identity": { - "type": "SystemAssigned,UserAssigned", - "userAssignedIdentities": "[json(concat('{\"', parameters('acrPullUserAssignedManagedIdentityResourceId'), '\":{}}'))]" - }, - "properties": { - "managedEnvironmentId": "[resourceId('Microsoft.App/managedEnvironments', variables('containerAppsEnvironmentName'))]", - "configuration": { - "activeRevisionsMode": "Single", - "ingress": { - "external": true, - "targetPort": 8080, - "allowInsecure": false, - "transport": "auto" - }, - "registries": [ - { - "server": "[parameters('registryName')]", - "identity": "[parameters('acrPullUserAssignedManagedIdentityResourceId')]" - } - ] - }, - "template": { - "containers": [ - { - "name": "[variables('containerAppName')]", - "image": "[variables('containerImage')]", - "resources": { - "cpu": "[json(parameters('containerCpu'))]", - "memory": "[parameters('containerMemory')]" - }, - "env": "[concat(variables('staticEnvVars'), parameters('additionalEnvVars'))]", - "probes": [ - { - "type": "Liveness", - "httpGet": { - "path": "/health/check", - "port": 8080 - }, - "initialDelaySeconds": 20, - "periodSeconds": 10, - "timeoutSeconds": 5, - "failureThreshold": 6 - }, - { - "type": "Readiness", - "httpGet": { - "path": "/health/check", - "port": 8080 - }, - "initialDelaySeconds": 20, - "periodSeconds": 10, - "timeoutSeconds": 5, - "failureThreshold": 6 - } - ] - } - ], - "scale": { - "minReplicas": "[parameters('minReplicas')]", - "maxReplicas": "[parameters('maxReplicas')]", - "rules": [ - { - "name": "http-scaling", - "http": { - "metadata": { - "concurrentRequests": "50" - } - } - }, - { - "name": "cpu-scaling", - "custom": { - "type": "cpu", - "metadata": { - "type": "Utilization", - "value": "70" - } - } - } - ] - } - } - }, - "dependsOn": [] - }, - { - "type": "Microsoft.KeyVault/vaults", - "apiVersion": "2022-07-01", - "name": "[variables('keyVaultName')]", - "location": "[resourceGroup().location]", - "properties": { - "sku": { - "family": "A", - "name": "Standard" - }, - "tenantId": "[subscription().tenantId]", - "accessPolicies": [], - "enableRbacAuthorization": true, - "enabledForDeployment": false - } - }, - { - "type": "Microsoft.KeyVault/vaults/providers/roleAssignments", - "apiVersion": "2022-04-01", - "name": "[concat(variables('keyVaultName'), '/Microsoft.Authorization/', guid(uniqueString('KeyVaultSecretsOfficer', variables('containerAppName'), variables('keyVaultName'))))]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", - "[resourceId('Microsoft.App/containerApps', variables('containerAppName'))]" - ], - "properties": { - "roleDefinitionId": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/', 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7')]", - "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').identity.principalId]" - } - }, - { - "condition": "[variables('enableIntegrationStore')]", - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2019-06-01", - "name": "[variables('storageAccountName')]", - "location": "[resourceGroup().location]", - "kind": "Storage", - "sku": { - "name": "Standard_LRS" - }, - "properties": { - "supportsHttpsTrafficOnly": true, - "allowBlobPublicAccess": false, - "allowSharedKeyAccess": false - } - }, - { - "condition": "[variables('enableIntegrationStore')]", - "type": "Microsoft.Storage/storageAccounts/providers/roleAssignments", - "apiVersion": "2018-09-01-preview", - "name": "[concat(variables('storageAccountName'), '/Microsoft.Authorization/', guid(uniqueString(variables('storageAccountName'), variables('containerAppName'))))]", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]", - "[resourceId('Microsoft.App/containerApps', variables('containerAppName'))]" - ], - "properties": { - "roleDefinitionId": "[variables('storageBlobDataContributerRoleId')]", - "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').identity.principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.KeyVault/vaults/secrets", - "apiVersion": "2015-06-01", - "name": "[concat(variables('keyVaultName'), '/CosmosDb--Host')]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" - ], - "properties": { - "contentType": "text/plain", - "value": "[reference(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('cosmosDbAccountName')), '2021-06-15').documentEndpoint]" - } - } - ], - "outputs": { - "containerAppName": { - "type": "string", - "value": "[variables('containerAppName')]" - }, - "containerAppFqdn": { - "type": "string", - "value": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').properties.configuration.ingress.fqdn]" - }, - "containerAppUrl": { - "type": "string", - "value": "[concat('https://', reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').properties.configuration.ingress.fqdn)]" - }, - "exportStorageUri": { - "type": "string", - "value": "[if(parameters('enableExport'), variables('storageAccountUri'), 'null')]" - }, - "integrationStorageUri": { - "type": "string", - "value": "[if(parameters('enableImport'), variables('storageAccountUri'), 'null')]" - } - } -} diff --git a/samples/templates/aca/fhir-sql-aca.bicep b/samples/templates/aca/fhir-sql-aca.bicep new file mode 100644 index 0000000000..af0e774ab2 --- /dev/null +++ b/samples/templates/aca/fhir-sql-aca.bicep @@ -0,0 +1,154 @@ +// FHIR Server on ACA with SQL Server datastore. + +// ────────────────────────────────────────────── +// Parameters +// ────────────────────────────────────────────── + +@description('Name of the Azure Container App hosting FHIR.') +param containerAppName string + +@description('Name of the Container Apps Environment.') +param containerAppsEnvironmentName string + +@description('Name of the Key Vault used by FHIR server.') +@maxLength(24) +param keyVaultName string + +@description('FHIR version to deploy.') +@allowed(['Stu3', 'R4', 'R4B', 'R5']) +param fhirVersion string = 'R4' + +@description('Container registry server host name.') +param registryName string + +@description('Docker image tag.') +param imageTag string = 'latest' + +@description('Existing SQL server name.') +param sqlServerName string + +@description('Schema automatic updates mode.') +@allowed(['auto', 'tool']) +param sqlSchemaAutomaticUpdatesEnabled string = 'auto' + +@description('Authority URL for AAD authentication.') +param securityAuthenticationAuthority string = '' + +@description('Audience for AAD authentication.') +param securityAuthenticationAudience string = '' + +@description('Resource ID of UAMI used to pull from ACR.') +param acrPullUserAssignedManagedIdentityResourceId string + +@description('Enable export operations.') +param enableExport bool = true + +@description('Enable import operations.') +param enableImport bool = true + +@description('Enable ConvertData operations.') +param enableConvertData bool = true + +@description('Enable reindex operations.') +param enableReindex bool = true + +@description('Minimum number of replicas.') +param minReplicas int = 0 + +@description('Maximum number of replicas.') +param maxReplicas int = 8 + +@description('CPU cores per container.') +param containerCpu string = '0.5' + +@description('Memory per container.') +param containerMemory string = '1Gi' + +@description('Additional environment variables from pipeline.') +param additionalEnvVars array = [] + +// ────────────────────────────────────────────── +// Variables +// ────────────────────────────────────────────── + +var normalizedSqlServerName = toLower(sqlServerName) +var sqlManagedIdentityName = '${normalizedSqlServerName}-uami' +var sqlDatabaseName = 'FHIR${fhirVersion}' + +var sqlManagedIdentityResourceId = resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', sqlManagedIdentityName) + +var userAssignedIdentities = { + '${sqlManagedIdentityResourceId}': {} + '${acrPullUserAssignedManagedIdentityResourceId}': {} +} + +var datastoreEnvVars = [ + { name: 'DataStore', value: 'SqlServer' } + { name: 'SqlServer__Initialize', value: 'true' } + { + name: 'SqlServer__SchemaOptions__AutomaticUpdatesEnabled' + value: sqlSchemaAutomaticUpdatesEnabled == 'auto' ? 'true' : 'false' + } + { name: 'SqlServer__DeleteAllDataOnStartup', value: 'false' } + { name: 'SqlServer__AllowDatabaseCreation', value: 'true' } +] + +// ────────────────────────────────────────────── +// Modules +// ────────────────────────────────────────────── + +module common 'modules/fhir-aca-common.bicep' = { + name: '${toLower(containerAppName)}-common' + params: { + containerAppName: containerAppName + containerAppsEnvironmentName: containerAppsEnvironmentName + keyVaultName: keyVaultName + fhirVersion: fhirVersion + registryName: registryName + imageTag: imageTag + securityAuthenticationAuthority: securityAuthenticationAuthority + securityAuthenticationAudience: securityAuthenticationAudience + acrPullUserAssignedManagedIdentityResourceId: acrPullUserAssignedManagedIdentityResourceId + enableExport: enableExport + enableImport: enableImport + enableConvertData: enableConvertData + enableReindex: enableReindex + minReplicas: minReplicas + maxReplicas: maxReplicas + containerCpu: containerCpu + containerMemory: containerMemory + additionalEnvVars: additionalEnvVars + datastoreEnvVars: datastoreEnvVars + userAssignedIdentities: userAssignedIdentities + } +} + +// ────────────────────────────────────────────── +// SQL-specific resources +// ────────────────────────────────────────────── + +resource existingSqlServer 'Microsoft.Sql/servers@2021-11-01' existing = { + name: normalizedSqlServerName +} + +resource existingSqlUami 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' existing = { + name: sqlManagedIdentityName +} + +resource sqlConnectionStringSecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { + name: '${toLower(keyVaultName)}/SqlServer--ConnectionString' + properties: { + contentType: 'text/plain' + value: 'Server=tcp:${existingSqlServer.properties.fullyQualifiedDomainName},1433;Initial Catalog=${sqlDatabaseName};Persist Security Info=False;Authentication=ActiveDirectoryMSI;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=${existingSqlUami.properties.clientId};' + } +} + +// ────────────────────────────────────────────── +// Outputs +// ────────────────────────────────────────────── + +output containerAppName string = common.outputs.containerAppName +output containerAppFqdn string = common.outputs.containerAppFqdn +output containerAppUrl string = common.outputs.containerAppUrl +output exportStorageUri string = common.outputs.exportStorageUri +output integrationStorageUri string = common.outputs.integrationStorageUri diff --git a/samples/templates/aca/fhir-sql-aca.json b/samples/templates/aca/fhir-sql-aca.json deleted file mode 100644 index 4481960609..0000000000 --- a/samples/templates/aca/fhir-sql-aca.json +++ /dev/null @@ -1,402 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "containerAppName": { - "type": "string", - "metadata": { - "description": "Name of the Azure Container App hosting FHIR." - } - }, - "containerAppsEnvironmentName": { - "type": "string", - "metadata": { - "description": "Name of the Container Apps Environment." - } - }, - "keyVaultName": { - "type": "string", - "maxLength": 24, - "metadata": { - "description": "Name of the Key Vault used by FHIR server." - } - }, - "fhirVersion": { - "type": "string", - "defaultValue": "R4", - "allowedValues": [ - "Stu3", - "R4", - "R4B", - "R5" - ] - }, - "registryName": { - "type": "string", - "metadata": { - "description": "Container registry server host name." - } - }, - "imageTag": { - "type": "string", - "defaultValue": "latest" - }, - "sqlServerName": { - "type": "string", - "metadata": { - "description": "Existing SQL server name." - } - }, - "sqlSchemaAutomaticUpdatesEnabled": { - "type": "string", - "allowedValues": [ - "auto", - "tool" - ], - "defaultValue": "auto" - }, - "securityAuthenticationAuthority": { - "type": "string", - "defaultValue": "" - }, - "securityAuthenticationAudience": { - "type": "string", - "defaultValue": "" - }, - "acrPullUserAssignedManagedIdentityResourceId": { - "type": "string", - "metadata": { - "description": "Resource ID of UAMI used to pull from ACR." - } - }, - "enableExport": { - "type": "bool", - "defaultValue": true - }, - "enableImport": { - "type": "bool", - "defaultValue": true - }, - "enableConvertData": { - "type": "bool", - "defaultValue": true - }, - "enableReindex": { - "type": "bool", - "defaultValue": true - }, - "minReplicas": { - "type": "int", - "defaultValue": 0 - }, - "maxReplicas": { - "type": "int", - "defaultValue": 8 - }, - "containerCpu": { - "type": "string", - "defaultValue": "0.5" - }, - "containerMemory": { - "type": "string", - "defaultValue": "1Gi" - }, - "additionalEnvVars": { - "type": "array", - "defaultValue": [] - } - }, - "variables": { - "isMAG": "[or(contains(resourceGroup().location,'usgov'),contains(resourceGroup().location,'usdod'))]", - "containerAppName": "[toLower(parameters('containerAppName'))]", - "containerAppsEnvironmentName": "[toLower(parameters('containerAppsEnvironmentName'))]", - "keyVaultName": "[toLower(parameters('keyVaultName'))]", - "sqlManagedIdentityName": "[concat(toLower(parameters('sqlServerName')), '-uami')]", - "sqlDatabaseName": "[concat('FHIR', parameters('fhirVersion'))]", - "azureContainerRegistryUri": "[if(variables('isMAG'), '.azurecr.us', '.azurecr.io')]", - "imageRepositoryName": "[if(contains(parameters('registryName'),'mcr.'), concat(toLower(parameters('fhirVersion')), '-fhir-server'), concat(toLower(parameters('fhirVersion')), '_fhir-server'))]", - "containerImage": "[concat(parameters('registryName'), '/', variables('imageRepositoryName'), ':', parameters('imageTag'))]", - "blobStorageUri": "[if(variables('isMAG'), '.blob.core.usgovcloudapi.net', '.blob.core.windows.net')]", - "storageAccountName": "[concat(substring(replace(variables('containerAppName'), '-', ''), 0, min(11, length(replace(variables('containerAppName'), '-', '')))), uniquestring(resourceGroup().id, variables('containerAppName')))]", - "storageAccountUri": "[concat('https://', variables('storageAccountName'), variables('blobStorageUri'))]", - "keyVaultEndpoint": "[if(variables('isMAG'), concat('https://', variables('keyVaultName'), '.vault.usgovcloudapi.net/'), concat('https://', variables('keyVaultName'), '.vault.azure.net/'))]", - "enableIntegrationStore": "[or(parameters('enableExport'), parameters('enableImport'))]", - "storageBlobDataContributerRoleId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe')]", - "staticEnvVars": [ - { - "name": "ASPNETCORE_FORWARDEDHEADERS_ENABLED", - "value": "true" - }, - { - "name": "KeyVault__Endpoint", - "value": "[variables('keyVaultEndpoint')]" - }, - { - "name": "FhirServer__Security__Enabled", - "value": "true" - }, - { - "name": "FhirServer__Security__EnableAadSmartOnFhirProxy", - "value": "true" - }, - { - "name": "FhirServer__Security__Authentication__Authority", - "value": "[parameters('securityAuthenticationAuthority')]" - }, - { - "name": "FhirServer__Security__Authentication__Audience", - "value": "[parameters('securityAuthenticationAudience')]" - }, - { - "name": "DataStore", - "value": "SqlServer" - }, - { - "name": "SqlServer__Initialize", - "value": "true" - }, - { - "name": "SqlServer__SchemaOptions__AutomaticUpdatesEnabled", - "value": "[if(equals(parameters('sqlSchemaAutomaticUpdatesEnabled'),'auto'), 'true', 'false')]" - }, - { - "name": "SqlServer__DeleteAllDataOnStartup", - "value": "false" - }, - { - "name": "SqlServer__AllowDatabaseCreation", - "value": "true" - }, - { - "name": "TaskHosting__Enabled", - "value": "true" - }, - { - "name": "TaskHosting__PollingFrequencyInSeconds", - "value": "1" - }, - { - "name": "TaskHosting__MaxRunningTaskCount", - "value": "2" - }, - { - "name": "FhirServer__CoreFeatures__SearchParameterCacheRefreshIntervalSeconds", - "value": "2" - }, - { - "name": "FhirServer__Operations__Export__Enabled", - "value": "[if(parameters('enableExport'), 'true', 'false')]" - }, - { - "name": "FhirServer__Operations__Export__StorageAccountUri", - "value": "[if(parameters('enableExport'), variables('storageAccountUri'), 'null')]" - }, - { - "name": "FhirServer__Operations__Import__Enabled", - "value": "[if(parameters('enableImport'), 'true', 'false')]" - }, - { - "name": "FhirServer__Operations__IntegrationDataStore__StorageAccountUri", - "value": "[if(parameters('enableImport'), variables('storageAccountUri'), 'null')]" - }, - { - "name": "FhirServer__Operations__ConvertData__Enabled", - "value": "[if(parameters('enableConvertData'), 'true', 'false')]" - }, - { - "name": "FhirServer__Operations__ConvertData__ContainerRegistryServers__0", - "value": "[if(parameters('enableConvertData'), parameters('registryName'), 'null')]" - }, - { - "name": "FhirServer__Operations__Reindex__Enabled", - "value": "[if(parameters('enableReindex'), 'true', 'false')]" - } - ] - }, - "resources": [ - { - "type": "Microsoft.App/containerApps", - "apiVersion": "2024-03-01", - "name": "[variables('containerAppName')]", - "location": "[resourceGroup().location]", - "identity": { - "type": "SystemAssigned,UserAssigned", - "userAssignedIdentities": "[json(concat('{\"', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('sqlManagedIdentityName')), '\":{},\"', parameters('acrPullUserAssignedManagedIdentityResourceId'), '\":{}}'))]" - }, - "properties": { - "managedEnvironmentId": "[resourceId('Microsoft.App/managedEnvironments', variables('containerAppsEnvironmentName'))]", - "configuration": { - "activeRevisionsMode": "Single", - "ingress": { - "external": true, - "targetPort": 8080, - "allowInsecure": false, - "transport": "auto" - }, - "registries": [ - { - "server": "[parameters('registryName')]", - "identity": "[parameters('acrPullUserAssignedManagedIdentityResourceId')]" - } - ] - }, - "template": { - "containers": [ - { - "name": "[variables('containerAppName')]", - "image": "[variables('containerImage')]", - "resources": { - "cpu": "[json(parameters('containerCpu'))]", - "memory": "[parameters('containerMemory')]" - }, - "env": "[concat(variables('staticEnvVars'), parameters('additionalEnvVars'))]", - "probes": [ - { - "type": "Liveness", - "httpGet": { - "path": "/health/check", - "port": 8080 - }, - "initialDelaySeconds": 20, - "periodSeconds": 10, - "timeoutSeconds": 5, - "failureThreshold": 6 - }, - { - "type": "Readiness", - "httpGet": { - "path": "/health/check", - "port": 8080 - }, - "initialDelaySeconds": 20, - "periodSeconds": 10, - "timeoutSeconds": 5, - "failureThreshold": 6 - } - ] - } - ], - "scale": { - "minReplicas": "[parameters('minReplicas')]", - "maxReplicas": "[parameters('maxReplicas')]", - "rules": [ - { - "name": "http-scaling", - "http": { - "metadata": { - "concurrentRequests": "50" - } - } - }, - { - "name": "cpu-scaling", - "custom": { - "type": "cpu", - "metadata": { - "type": "Utilization", - "value": "70" - } - } - } - ] - } - } - }, - "dependsOn": [] - }, - { - "type": "Microsoft.KeyVault/vaults", - "apiVersion": "2022-07-01", - "name": "[variables('keyVaultName')]", - "location": "[resourceGroup().location]", - "properties": { - "sku": { - "family": "A", - "name": "Standard" - }, - "tenantId": "[subscription().tenantId]", - "accessPolicies": [], - "enableRbacAuthorization": true, - "enabledForDeployment": false - } - }, - { - "type": "Microsoft.KeyVault/vaults/providers/roleAssignments", - "apiVersion": "2022-04-01", - "name": "[concat(variables('keyVaultName'), '/Microsoft.Authorization/', guid(uniqueString('KeyVaultSecretsOfficer', variables('containerAppName'), variables('keyVaultName'))))]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", - "[resourceId('Microsoft.App/containerApps', variables('containerAppName'))]" - ], - "properties": { - "roleDefinitionId": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/', 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7')]", - "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').identity.principalId]" - } - }, - { - "condition": "[variables('enableIntegrationStore')]", - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2019-06-01", - "name": "[variables('storageAccountName')]", - "location": "[resourceGroup().location]", - "kind": "Storage", - "sku": { - "name": "Standard_LRS" - }, - "properties": { - "supportsHttpsTrafficOnly": true, - "allowBlobPublicAccess": false, - "allowSharedKeyAccess": false - } - }, - { - "condition": "[variables('enableIntegrationStore')]", - "type": "Microsoft.Storage/storageAccounts/providers/roleAssignments", - "apiVersion": "2018-09-01-preview", - "name": "[concat(variables('storageAccountName'), '/Microsoft.Authorization/', guid(uniqueString(variables('storageAccountName'), variables('containerAppName'))))]", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]", - "[resourceId('Microsoft.App/containerApps', variables('containerAppName'))]" - ], - "properties": { - "roleDefinitionId": "[variables('storageBlobDataContributerRoleId')]", - "principalId": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').identity.principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.KeyVault/vaults/secrets", - "apiVersion": "2015-06-01", - "name": "[concat(variables('keyVaultName'), '/SqlServer--ConnectionString')]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" - ], - "properties": { - "contentType": "text/plain", - "value": "[concat('Server=tcp:', reference(resourceId('Microsoft.Sql/servers', toLower(parameters('sqlServerName'))), '2015-05-01-preview').fullyQualifiedDomainName, ',1433;Initial Catalog=', variables('sqlDatabaseName'), ';Persist Security Info=False;Authentication=ActiveDirectoryMSI;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=', reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('sqlManagedIdentityName')), '2018-11-30').clientId, ';')]" - } - } - ], - "outputs": { - "containerAppName": { - "type": "string", - "value": "[variables('containerAppName')]" - }, - "containerAppFqdn": { - "type": "string", - "value": "[reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').properties.configuration.ingress.fqdn]" - }, - "containerAppUrl": { - "type": "string", - "value": "[concat('https://', reference(resourceId('Microsoft.App/containerApps', variables('containerAppName')), '2024-03-01', 'Full').properties.configuration.ingress.fqdn)]" - }, - "exportStorageUri": { - "type": "string", - "value": "[if(parameters('enableExport'), variables('storageAccountUri'), 'null')]" - }, - "integrationStorageUri": { - "type": "string", - "value": "[if(parameters('enableImport'), variables('storageAccountUri'), 'null')]" - } - } -} diff --git a/samples/templates/aca/modules/fhir-aca-common.bicep b/samples/templates/aca/modules/fhir-aca-common.bicep new file mode 100644 index 0000000000..9df75b0f94 --- /dev/null +++ b/samples/templates/aca/modules/fhir-aca-common.bicep @@ -0,0 +1,267 @@ +// Common FHIR ACA module — shared resources for both SQL and Cosmos deployments. + +// ────────────────────────────────────────────── +// Parameters +// ────────────────────────────────────────────── + +@description('Name of the Azure Container App hosting FHIR.') +param containerAppName string + +@description('Name of the Container Apps Environment.') +param containerAppsEnvironmentName string + +@description('Name of the Key Vault used by FHIR server.') +@maxLength(24) +param keyVaultName string + +@description('FHIR version to deploy.') +@allowed(['Stu3', 'R4', 'R4B', 'R5']) +param fhirVersion string = 'R4' + +@description('Container registry server host name.') +param registryName string + +@description('Docker image tag.') +param imageTag string = 'latest' + +@description('Authority URL for AAD authentication.') +param securityAuthenticationAuthority string = '' + +@description('Audience for AAD authentication.') +param securityAuthenticationAudience string = '' + +@description('Resource ID of UAMI used to pull from ACR.') +param acrPullUserAssignedManagedIdentityResourceId string + +@description('Enable export operations.') +param enableExport bool = true + +@description('Enable import operations.') +param enableImport bool = true + +@description('Enable ConvertData operations.') +param enableConvertData bool = true + +@description('Enable reindex operations.') +param enableReindex bool = true + +@description('Minimum number of replicas.') +param minReplicas int = 0 + +@description('Maximum number of replicas.') +param maxReplicas int = 8 + +@description('CPU cores per container.') +param containerCpu string = '0.5' + +@description('Memory per container.') +param containerMemory string = '1Gi' + +@description('Additional environment variables from pipeline.') +param additionalEnvVars array = [] + +@description('Datastore-specific environment variables (SQL or Cosmos settings).') +param datastoreEnvVars array = [] + +@description('User-assigned managed identities object to attach to the container app.') +param userAssignedIdentities object + +// ────────────────────────────────────────────── +// Variables +// ────────────────────────────────────────────── + +var normalizedAppName = toLower(containerAppName) +var normalizedEnvName = toLower(containerAppsEnvironmentName) +var normalizedKeyVaultName = toLower(keyVaultName) + +var isMAG = contains(resourceGroup().location, 'usgov') || contains(resourceGroup().location, 'usdod') + +var imageRepositoryName = contains(registryName, 'mcr.') + ? '${toLower(fhirVersion)}-fhir-server' + : '${toLower(fhirVersion)}_fhir-server' +var containerImage = '${registryName}/${imageRepositoryName}:${imageTag}' + +var blobStorageUri = isMAG ? '.blob.core.usgovcloudapi.net' : environment().suffixes.storage +var storageAccountPrefix = substring(replace(normalizedAppName, '-', ''), 0, min(11, length(replace(normalizedAppName, '-', '')))) +var storageAccountName = '${storageAccountPrefix}${uniqueString(resourceGroup().id, normalizedAppName)}' +var storageAccountUri = 'https://${storageAccountName}${blobStorageUri}' + +var keyVaultEndpoint = isMAG + ? 'https://${normalizedKeyVaultName}.vault.usgovcloudapi.net/' + : 'https://${normalizedKeyVaultName}${environment().suffixes.keyvaultDns}/' + +var enableIntegrationStore = enableExport || enableImport + +var sharedEnvVars = [ + { name: 'ASPNETCORE_FORWARDEDHEADERS_ENABLED', value: 'true' } + { name: 'KeyVault__Endpoint', value: keyVaultEndpoint } + { name: 'FhirServer__Security__Enabled', value: 'true' } + { name: 'FhirServer__Security__EnableAadSmartOnFhirProxy', value: 'true' } + { name: 'FhirServer__Security__Authentication__Authority', value: securityAuthenticationAuthority } + { name: 'FhirServer__Security__Authentication__Audience', value: securityAuthenticationAudience } + { name: 'TaskHosting__Enabled', value: 'true' } + { name: 'TaskHosting__PollingFrequencyInSeconds', value: '1' } + { name: 'TaskHosting__MaxRunningTaskCount', value: '2' } + { name: 'FhirServer__CoreFeatures__SearchParameterCacheRefreshIntervalSeconds', value: '2' } + { name: 'FhirServer__Operations__Export__Enabled', value: enableExport ? 'true' : 'false' } + { name: 'FhirServer__Operations__Export__StorageAccountUri', value: enableExport ? storageAccountUri : 'null' } + { name: 'FhirServer__Operations__Import__Enabled', value: enableImport ? 'true' : 'false' } + { name: 'FhirServer__Operations__IntegrationDataStore__StorageAccountUri', value: enableImport ? storageAccountUri : 'null' } + { name: 'FhirServer__Operations__ConvertData__Enabled', value: enableConvertData ? 'true' : 'false' } + { name: 'FhirServer__Operations__ConvertData__ContainerRegistryServers__0', value: enableConvertData ? registryName : 'null' } + { name: 'FhirServer__Operations__Reindex__Enabled', value: enableReindex ? 'true' : 'false' } +] + +var allEnvVars = concat(sharedEnvVars, datastoreEnvVars, additionalEnvVars) + +// ────────────────────────────────────────────── +// Resources +// ────────────────────────────────────────────── + +resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { + name: normalizedAppName + location: resourceGroup().location + identity: { + type: 'SystemAssigned,UserAssigned' + userAssignedIdentities: userAssignedIdentities + } + properties: { + managedEnvironmentId: resourceId('Microsoft.App/managedEnvironments', normalizedEnvName) + configuration: { + activeRevisionsMode: 'Single' + ingress: { + external: true + targetPort: 8080 + allowInsecure: false + transport: 'auto' + } + registries: [ + { + server: registryName + identity: acrPullUserAssignedManagedIdentityResourceId + } + ] + } + template: { + containers: [ + { + name: normalizedAppName + image: containerImage + resources: { + cpu: json(containerCpu) + memory: containerMemory + } + env: allEnvVars + probes: [ + { + type: 'Liveness' + httpGet: { + path: '/health/check' + port: 8080 + } + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + } + { + type: 'Readiness' + httpGet: { + path: '/health/check' + port: 8080 + } + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + } + ] + } + ] + scale: { + minReplicas: minReplicas + maxReplicas: maxReplicas + rules: [ + { + name: 'http-scaling' + http: { + metadata: { + concurrentRequests: '50' + } + } + } + { + name: 'cpu-scaling' + custom: { + type: 'cpu' + metadata: { + type: 'Utilization' + value: '70' + } + } + } + ] + } + } + } +} + +resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = { + name: normalizedKeyVaultName + location: resourceGroup().location + properties: { + sku: { + family: 'A' + name: 'standard' + } + tenantId: subscription().tenantId + accessPolicies: [] + enableRbacAuthorization: true + enabledForDeployment: false + } +} + +resource keyVaultSecretsOfficerRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(uniqueString('KeyVaultSecretsOfficer', normalizedAppName, normalizedKeyVaultName)) + scope: keyVault + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7') + principalId: containerApp.identity.principalId + } +} + +resource storageAccount 'Microsoft.Storage/storageAccounts@2019-06-01' = if (enableIntegrationStore) { + name: storageAccountName + location: resourceGroup().location + kind: 'Storage' + sku: { + name: 'Standard_LRS' + } + properties: { + supportsHttpsTrafficOnly: true + #disable-next-line BCP037 + allowBlobPublicAccess: false + allowSharedKeyAccess: false + } +} + +resource storageBlobDataContributorRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (enableIntegrationStore) { + name: guid(uniqueString(storageAccountName, normalizedAppName)) + scope: storageAccount + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') + principalId: containerApp.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// ────────────────────────────────────────────── +// Outputs +// ────────────────────────────────────────────── + +output containerAppName string = containerApp.name +output containerAppFqdn string = containerApp.properties.configuration.ingress.fqdn +output containerAppUrl string = 'https://${containerApp.properties.configuration.ingress.fqdn}' +output keyVaultName string = keyVault.name +output exportStorageUri string = enableExport ? storageAccountUri : 'null' +output integrationStorageUri string = enableImport ? storageAccountUri : 'null' From 3abe8f634b91110a3c5e99bc771aa7125e9a1ec4 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Thu, 2 Apr 2026 08:12:27 -0500 Subject: [PATCH 34/38] fixing bicep storage suffix --- samples/templates/aca/modules/fhir-aca-common.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/templates/aca/modules/fhir-aca-common.bicep b/samples/templates/aca/modules/fhir-aca-common.bicep index 9df75b0f94..c7895560e7 100644 --- a/samples/templates/aca/modules/fhir-aca-common.bicep +++ b/samples/templates/aca/modules/fhir-aca-common.bicep @@ -81,7 +81,7 @@ var imageRepositoryName = contains(registryName, 'mcr.') : '${toLower(fhirVersion)}_fhir-server' var containerImage = '${registryName}/${imageRepositoryName}:${imageTag}' -var blobStorageUri = isMAG ? '.blob.core.usgovcloudapi.net' : environment().suffixes.storage +var blobStorageUri = isMAG ? '.blob.core.usgovcloudapi.net' : '.blob.${environment().suffixes.storage}' var storageAccountPrefix = substring(replace(normalizedAppName, '-', ''), 0, min(11, length(replace(normalizedAppName, '-', '')))) var storageAccountName = '${storageAccountPrefix}${uniqueString(resourceGroup().id, normalizedAppName)}' var storageAccountUri = 'https://${storageAccountName}${blobStorageUri}' From 18bf7287bd3be874296b439e267008079b6fb2b7 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Thu, 2 Apr 2026 10:21:32 -0500 Subject: [PATCH 35/38] Setting PR min replicas to 3 to match Production scenarios --- build/jobs/provision-deploy-aca.yml | 5 ++++- build/jobs/provision-deploy-cosmos-aca.yml | 5 ++++- build/pr-pipeline.yml | 8 ++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/build/jobs/provision-deploy-aca.yml b/build/jobs/provision-deploy-aca.yml index a026734bc7..22a9c7d88f 100644 --- a/build/jobs/provision-deploy-aca.yml +++ b/build/jobs/provision-deploy-aca.yml @@ -21,6 +21,9 @@ parameters: - name: reindexEnabled type: boolean default: true +- name: minReplicas + type: number + default: 0 - name: keyVaultName type: string @@ -135,7 +138,7 @@ jobs: enableImport = $true enableConvertData = $true enableReindex = if ("${{ parameters.reindexEnabled }}" -eq "true") { $true } else { $false } - minReplicas = 0 + minReplicas = ${{ parameters.minReplicas }} maxReplicas = 8 acrPullUserAssignedManagedIdentityResourceId = '/subscriptions/a1766500-6fd5-4f5c-8515-607798271014/resourceGroups/HealthPlatformRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-uami' additionalEnvVars = $additionalEnvVars diff --git a/build/jobs/provision-deploy-cosmos-aca.yml b/build/jobs/provision-deploy-cosmos-aca.yml index f52fa92daa..c1d5aebc90 100644 --- a/build/jobs/provision-deploy-cosmos-aca.yml +++ b/build/jobs/provision-deploy-cosmos-aca.yml @@ -16,6 +16,9 @@ parameters: - name: reindexEnabled type: boolean default: true +- name: minReplicas + type: number + default: 0 - name: keyVaultName type: string @@ -184,7 +187,7 @@ jobs: enableImport = $true enableConvertData = $true enableReindex = if ("${{ parameters.reindexEnabled }}" -eq "true") { $true } else { $false } - minReplicas = 0 + minReplicas = ${{ parameters.minReplicas }} maxReplicas = 8 acrPullUserAssignedManagedIdentityResourceId = '/subscriptions/a1766500-6fd5-4f5c-8515-607798271014/resourceGroups/HealthPlatformRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-uami' additionalEnvVars = $additionalEnvVars diff --git a/build/pr-pipeline.yml b/build/pr-pipeline.yml index 631b93fbdf..194da3abe3 100644 --- a/build/pr-pipeline.yml +++ b/build/pr-pipeline.yml @@ -283,6 +283,7 @@ stages: testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true + minReplicas: 3 - stage: deployStu3Sql displayName: 'Deploy STU3 SQL Site' @@ -304,6 +305,7 @@ stages: schemaAutomaticUpdatesEnabled: 'auto' sqlServerName: $(DeploymentEnvironmentName) reindexEnabled: true + minReplicas: 3 - stage: deployR4 displayName: 'Deploy R4 CosmosDB Site' @@ -323,6 +325,7 @@ stages: testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true + minReplicas: 3 - stage: deployR4Sql displayName: 'Deploy R4 SQL Site' @@ -344,6 +347,7 @@ stages: schemaAutomaticUpdatesEnabled: 'auto' sqlServerName: $(DeploymentEnvironmentName) reindexEnabled: true + minReplicas: 3 - stage: deployR4B displayName: 'Deploy R4B CosmosDB Site' @@ -363,6 +367,7 @@ stages: testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true + minReplicas: 3 - stage: deployR4BSql displayName: 'Deploy R4B SQL Site' @@ -384,6 +389,7 @@ stages: schemaAutomaticUpdatesEnabled: 'auto' sqlServerName: $(DeploymentEnvironmentName) reindexEnabled: true + minReplicas: 3 - stage: deployR5 displayName: 'Deploy R5 CosmosDB Site' @@ -403,6 +409,7 @@ stages: testEnvironmentUrl: $(TestApplicationResource) imageTag: $(ImageTag) reindexEnabled: true + minReplicas: 3 - stage: deployR5Sql displayName: 'Deploy R5 SQL Site' @@ -424,6 +431,7 @@ stages: schemaAutomaticUpdatesEnabled: 'auto' sqlServerName: $(DeploymentEnvironmentName) reindexEnabled: true + minReplicas: 3 - stage: testStu3Cosmos displayName: 'Run Stu3 Cosmos Tests' From 41d77c18b1034bb1fb75b79f480daecef38260d6 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Fri, 3 Apr 2026 11:57:19 -0500 Subject: [PATCH 36/38] Refactor HTTP client usage and enhance retry logic for metadata fetch --- .../Rest/TestFhirServer.cs | 133 +++++++++++------- 1 file changed, 81 insertions(+), 52 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServer.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServer.cs index 8112b99059..75ded71105 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServer.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServer.cs @@ -200,98 +200,127 @@ public async Task ConfigureSecurityOptions(CancellationToken cancellationToken = { bool localSecurityEnabled = false; - var httpClient = new HttpClient(CreateMessageHandler()) + using var availabilityHttpClient = new HttpClient(CreateMessageHandler()) { BaseAddress = BaseAddress, Timeout = TimeSpan.FromSeconds(30), }; - // Retry policy for transient failures (500 errors, timeouts, etc.) during server startup - // Total timeout is 5 minutes to ensure we fail fast if the server is not coming up - var overallTimeout = TimeSpan.FromMinutes(5); - var overallStopwatch = System.Diagnostics.Stopwatch.StartNew(); + await WaitForAvailabilityAsync(availabilityHttpClient, cancellationToken); + + using var metadataHttpClient = new HttpClient(CreateMessageHandler()) + { + BaseAddress = BaseAddress, + Timeout = TimeSpan.FromSeconds(120), + }; + + string content = await GetMetadataAsync(metadataHttpClient, cancellationToken); + + CapabilityStatement metadata = new FhirJsonParser().Parse(content); + Metadata = metadata.ToResourceElement(); + +#if Stu3 || R4 || R4B + foreach (var rest in metadata.Rest.Where(r => r.Mode == RestfulCapabilityMode.Server)) +#else + foreach (var rest in metadata.Rest.Where(r => r.Mode == CapabilityStatement.RestfulCapabilityMode.Server)) +#endif + { + var oauth = rest.Security?.GetExtension(Core.Features.Security.Constants.SmartOAuthUriExtension); + if (oauth != null) + { + var tokenUrl = oauth.GetExtensionValue(Core.Features.Security.Constants.SmartOAuthUriExtensionToken).Value; + var authorizeUrl = oauth.GetExtensionValue(Core.Features.Security.Constants.SmartOAuthUriExtensionAuthorize).Value; + + localSecurityEnabled = true; + TokenUri = new Uri(tokenUrl); + AuthorizeUri = new Uri(authorizeUrl); + + break; + } + } + + SecurityEnabled = localSecurityEnabled; + } + + private async Task WaitForAvailabilityAsync(HttpClient httpClient, CancellationToken cancellationToken) + { + await SendGetWithRetryAsync( + httpClient, + new Uri(BaseAddress, "$versions"), + "Availability check", + TimeSpan.FromMinutes(5), + cancellationToken); + } + + private async Task GetMetadataAsync(HttpClient httpClient, CancellationToken cancellationToken) + { + return await SendGetWithRetryAsync( + httpClient, + new Uri(BaseAddress, "metadata"), + "Metadata fetch", + TimeSpan.FromMinutes(10), + cancellationToken); + } + + private static async Task SendGetWithRetryAsync(HttpClient httpClient, Uri requestUri, string operationName, TimeSpan overallTimeout, CancellationToken cancellationToken) + { + // Retry policy for transient failures during server startup with exponential backoff. + var overallStopwatch = Stopwatch.StartNew(); const int baseDelaySeconds = 5; const int maxDelaySeconds = 30; - HttpResponseMessage response = null; - string content = null; int attempt = 0; + HttpStatusCode? lastStatusCode = null; + string lastErrorMessage = null; while (overallStopwatch.Elapsed < overallTimeout) { attempt++; try { - using HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Get, new Uri(BaseAddress, "metadata")); - response = await httpClient.SendAsync(requestMessage, cancellationToken); - content = await response.Content.ReadAsStringAsync(); + using HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Get, requestUri); + using HttpResponseMessage response = await httpClient.SendAsync(requestMessage, cancellationToken); + string content = await response.Content.ReadAsStringAsync(); if (response.IsSuccessStatusCode) { - Console.WriteLine($"[ConfigureSecurityOptions] Metadata fetch successful on attempt {attempt} after {overallStopwatch.Elapsed.TotalSeconds:F1}s."); - break; + Console.WriteLine($"[ConfigureSecurityOptions] {operationName} successful on attempt {attempt} after {overallStopwatch.Elapsed.TotalSeconds:F1}s."); + return content; } - // Retry on 5xx errors (server not ready) or 401/503 (transient auth/availability issues) if within timeout + lastStatusCode = response.StatusCode; + lastErrorMessage = $"Last status: {response.StatusCode}"; + if (((int)response.StatusCode >= 500 || response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.ServiceUnavailable) && overallStopwatch.Elapsed < overallTimeout) { - int delaySeconds = Math.Min(baseDelaySeconds * (int)Math.Pow(2, Math.Min(attempt - 1, 3)), maxDelaySeconds); // Cap growth at attempt 4 - Console.WriteLine($"[ConfigureSecurityOptions] Metadata fetch returned {response.StatusCode} on attempt {attempt}. Elapsed: {overallStopwatch.Elapsed.TotalSeconds:F1}s. Retrying in {delaySeconds}s..."); + int delaySeconds = Math.Min(baseDelaySeconds * (int)Math.Pow(2, Math.Min(attempt - 1, 3)), maxDelaySeconds); + Console.WriteLine($"[ConfigureSecurityOptions] {operationName} returned {response.StatusCode} on attempt {attempt}. Elapsed: {overallStopwatch.Elapsed.TotalSeconds:F1}s. Retrying in {delaySeconds}s..."); await Task.Delay(TimeSpan.FromSeconds(delaySeconds), cancellationToken); - response.Dispose(); continue; } - // Non-retryable error or timeout exhausted response.EnsureSuccessStatusCode(); } catch (Exception ex) when (ex is TaskCanceledException || ex is HttpRequestException || ex is IOException) { + lastErrorMessage = $"Last error: {ex.Message}"; + if (overallStopwatch.Elapsed < overallTimeout) { int delaySeconds = Math.Min(baseDelaySeconds * (int)Math.Pow(2, Math.Min(attempt - 1, 3)), maxDelaySeconds); - Console.WriteLine($"[ConfigureSecurityOptions] Metadata fetch failed with {ex.GetType().Name} on attempt {attempt}. Elapsed: {overallStopwatch.Elapsed.TotalSeconds:F1}s. Retrying in {delaySeconds}s..."); + Console.WriteLine($"[ConfigureSecurityOptions] {operationName} failed with {ex.GetType().Name} on attempt {attempt}. Elapsed: {overallStopwatch.Elapsed.TotalSeconds:F1}s. Retrying in {delaySeconds}s..."); await Task.Delay(TimeSpan.FromSeconds(delaySeconds), cancellationToken); continue; } - throw new HttpRequestException($"ConfigureSecurityOptions failed after {attempt} attempts over {overallStopwatch.Elapsed.TotalSeconds:F1}s. Last error: {ex.Message}", ex); - } - } - - // If we exited the loop due to timeout without success - if (response == null || !response.IsSuccessStatusCode) - { - string errorMessage = response != null - ? $"ConfigureSecurityOptions failed after {attempt} attempts over {overallStopwatch.Elapsed.TotalSeconds:F1}s. Last status: {response.StatusCode}" - : $"ConfigureSecurityOptions failed after {attempt} attempts over {overallStopwatch.Elapsed.TotalSeconds:F1}s. No response received."; - throw new HttpRequestException(errorMessage); - } - - CapabilityStatement metadata = new FhirJsonParser().Parse(content); - Metadata = metadata.ToResourceElement(); - -#if Stu3 || R4 || R4B - foreach (var rest in metadata.Rest.Where(r => r.Mode == RestfulCapabilityMode.Server)) -#else - foreach (var rest in metadata.Rest.Where(r => r.Mode == CapabilityStatement.RestfulCapabilityMode.Server)) -#endif - { - var oauth = rest.Security?.GetExtension(Core.Features.Security.Constants.SmartOAuthUriExtension); - if (oauth != null) - { - var tokenUrl = oauth.GetExtensionValue(Core.Features.Security.Constants.SmartOAuthUriExtensionToken).Value; - var authorizeUrl = oauth.GetExtensionValue(Core.Features.Security.Constants.SmartOAuthUriExtensionAuthorize).Value; - - localSecurityEnabled = true; - TokenUri = new Uri(tokenUrl); - AuthorizeUri = new Uri(authorizeUrl); - - break; + throw new HttpRequestException($"{operationName} failed after {attempt} attempts over {overallStopwatch.Elapsed.TotalSeconds:F1}s. {lastErrorMessage}", ex); } } - SecurityEnabled = localSecurityEnabled; + string errorMessage = lastStatusCode.HasValue + ? $"{operationName} failed after {attempt} attempts over {overallStopwatch.Elapsed.TotalSeconds:F1}s. Last status: {lastStatusCode.Value}" + : $"{operationName} failed after {attempt} attempts over {overallStopwatch.Elapsed.TotalSeconds:F1}s. {lastErrorMessage ?? "No response received."}"; + throw new HttpRequestException(errorMessage); } public virtual ValueTask DisposeAsync() From a2d7323b9bc02410f50acd3e051e923eb13b3b98 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Fri, 3 Apr 2026 12:18:53 -0500 Subject: [PATCH 37/38] Add warmup path and status properties to FHIR server configuration --- samples/templates/default-azuredeploy-docker.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/templates/default-azuredeploy-docker.json b/samples/templates/default-azuredeploy-docker.json index 4df2c03b7e..7f26df1b1a 100644 --- a/samples/templates/default-azuredeploy-docker.json +++ b/samples/templates/default-azuredeploy-docker.json @@ -422,7 +422,7 @@ "[if(variables('deployAppInsights'),concat('Microsoft.Insights/components/', variables('appInsightsName')),resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName')))]", "[if(equals(parameters('solutionType'),'FhirServerCosmosDB'), resourceId('Microsoft.KeyVault/vaults/secrets', variables('keyVaultName'), 'CosmosDb--Host'), resourceId('Microsoft.KeyVault/vaults/secrets', variables('keyVaultName'), 'SqlServer--ConnectionString'))]" ], - "properties": "[union(variables('combinedFhirServerConfigProperties'), json(concat('{ \"FhirServer__ResourceManager__DataStoreResourceId\": \"', if(equals(parameters('solutionType'),'FhirServerCosmosDB'), resourceId('Microsoft.DocumentDb/databaseAccounts', variables('serviceName')), resourceId('Microsoft.Sql/servers/', variables('sqlServerDerivedName'))), '\", ', if(variables('deployAppInsights'), concat('\"Telemetry__Provider\": \"', parameters('telemetryProviderType'), '\",', '\"Telemetry__InstrumentationKey\": \"', reference(resourceId('Microsoft.Insights/components', variables('appInsightsName'))).InstrumentationKey, '\",', '\"Telemetry__ConnectionString\": \"', reference(resourceId('Microsoft.Insights/components', variables('appInsightsName'))).ConnectionString, '\"'), ''), '}')))]" + "properties": "[union(variables('combinedFhirServerConfigProperties'), json(concat('{ \"FhirServer__ResourceManager__DataStoreResourceId\": \"', if(equals(parameters('solutionType'),'FhirServerCosmosDB'), resourceId('Microsoft.DocumentDb/databaseAccounts', variables('serviceName')), resourceId('Microsoft.Sql/servers/', variables('sqlServerDerivedName'))), '\", ', if(variables('deployAppInsights'), concat('\"Telemetry__Provider\": \"', parameters('telemetryProviderType'), '\",', '\"Telemetry__InstrumentationKey\": \"', reference(resourceId('Microsoft.Insights/components', variables('appInsightsName'))).InstrumentationKey, '\",', '\"Telemetry__ConnectionString\": \"', reference(resourceId('Microsoft.Insights/components', variables('appInsightsName'))).ConnectionString, '\",'), ''), '\"WEBSITE_WARMUP_PATH\": \"/health/check\",', '\"WEBSITE_WARMUP_STATUSES\": \"200\",', '\"WEBSITES_CONTAINER_START_TIME_LIMIT\": \"600\"', '}')))]" }, { "apiVersion": "2018-11-01", From 100b30cd4c481d96be8e4d10d893f1d8d9216e45 Mon Sep 17 00:00:00 2001 From: John Estrada Date: Fri, 3 Apr 2026 13:57:30 -0500 Subject: [PATCH 38/38] Update to 180 seconds for metadata --- .../Rest/TestFhirServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServer.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServer.cs index 75ded71105..56cc1db464 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServer.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/TestFhirServer.cs @@ -211,7 +211,7 @@ public async Task ConfigureSecurityOptions(CancellationToken cancellationToken = using var metadataHttpClient = new HttpClient(CreateMessageHandler()) { BaseAddress = BaseAddress, - Timeout = TimeSpan.FromSeconds(120), + Timeout = TimeSpan.FromSeconds(180), }; string content = await GetMetadataAsync(metadataHttpClient, cancellationToken);