diff --git a/.github/workflows/develop-ci.yml b/.github/workflows/develop-ci.yml index be52900..942fb9a 100644 --- a/.github/workflows/develop-ci.yml +++ b/.github/workflows/develop-ci.yml @@ -1,38 +1,45 @@ -# This is a basic workflow to help you get started with Actions - name: CI -# Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the master branch push: branches: [ develop ] pull_request: branches: [ develop ] - - # Allows you to run this workflow manually from the Actions tab workflow_dispatch: -# A workflow run is made up of one or more jobs that can run sequentially or in parallel +permissions: + contents: read + checks: write # required by test-reporter + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: - runs-on: windows-2019 + runs-on: windows-latest - # Steps represent a sequence of tasks that will be executed as part of the job steps: - - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Login via Az module - uses: azure/login@v1 + uses: azure/login@v3 with: - creds: ${{secrets.AZURE_CREDENTIALS}} - enable-AzPSSession: true + creds: ${{ secrets.AZURE_CREDENTIALS }} + enable-AzPSSession: true - - name: Run Azure PowerShell script - uses: azure/powershell@v1 + - name: Run All Unit and Integration Tests + uses: azure/powershell@v3 with: inlineScript: | - test/!RunAllTests.ps1 -folder 'D:/a/azure.datafactory.tools/azure.datafactory.tools/' -TestFilenameFilter "*" + test/!RunAllTests.ps1 -folder '${{ github.workspace }}/' -TestFilenameFilter "*" -InstallModules:$false -ResultOutputFormat "JUnitXml" azPSVersion: "latest" + + - name: Publish Pester Test Results + uses: dorny/test-reporter@v3 + if: ${{ !cancelled() }} # run this step even if previous step failed + with: + name: Pester Tests + path: '**/TEST-*.xml' + reporter: java-junit + collapsed: 'auto' diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index ee88b00..8bccb32 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -12,7 +12,7 @@ RootModule = 'azure.datafactory.tools.psm1' # Version number of this module. - ModuleVersion = '1.14.0' + ModuleVersion = '1.15.0' # Supported PSEditions # CompatiblePSEditions = @() @@ -27,7 +27,7 @@ CompanyName = 'SQLPlayer' # Copyright statement for this module - Copyright = '(c) 2020-2025 Kamil Nowinski. All rights reserved.' + Copyright = '(c) 2020-2026 Kamil Nowinski. All rights reserved.' # Description of the functionality provided by this module Description = 'PowerShell module to help with CI&CD for Azure Data Factory, mainly to publish to ADF service in multiple environments. Check https://github.com/Azure-Player/azure.datafactory.tools/ & https://azureplayer.net/adf/' diff --git a/changelog.md b/changelog.md index 03aab5e..91a3b61 100644 --- a/changelog.md +++ b/changelog.md @@ -4,8 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [Unreleased] +## [1.15.0] - 2026-05-26 +### Added +* Credential type of ADF objects is now supported for deployment via REST API ### Fixed +* Starting blob event trigger fails with 'Resource cannot be updated during provisioning' - now waits for provisioning to complete before retrying #484, #474, #463 +* Fixed REST API calls failing with 401 Unauthorized due to Az.Accounts 5.x returning SecureString from Get-AzAccessToken * Fixed DryRun not loading deployment state from storage for hash comparison #476 * README.md updated and new Structured Documentation created diff --git a/private/Deploy-AdfObjectOnly.ps1 b/private/Deploy-AdfObjectOnly.ps1 index d583d25..1398035 100644 --- a/private/Deploy-AdfObjectOnly.ps1 +++ b/private/Deploy-AdfObjectOnly.ps1 @@ -123,8 +123,16 @@ function Deploy-AdfObjectOnly { } 'credential' { - Write-Warning "Credentials are not yet supported. The deployment for the object is skipped." - Write-Warning "Any reference to the object causes error, unless to deploy it before." + $resType = Get-AzureResourceType $obj.Type + $resName = $obj.AzureResourceName() + + New-AzResource ` + -ResourceType $resType ` + -ResourceGroupName $ResourceGroupName ` + -ResourceName "$resName" ` + -ApiVersion "2018-06-01" ` + -Properties $json ` + -IsFullObject -Force | Out-Null } 'AzResource' { diff --git a/private/Get-AzDFV2Credential.ps1 b/private/Get-AzDFV2Credential.ps1 index 07a2c09..c0cffad 100644 --- a/private/Get-AzDFV2Credential.ps1 +++ b/private/Get-AzDFV2Credential.ps1 @@ -5,6 +5,13 @@ function Get-AzDFV2Credential { ) Write-Debug "BEGIN: Get-AzDFV2Credential" + # Retrieve all credentials via API without parsing + $token = Get-AzAccessToken -ResourceUrl 'https://management.azure.com' + $tokenStr = [System.Net.NetworkCredential]::new('', $token.Token).Password + $authHeader = @{ + 'Content-Type' = 'application/json' + 'Authorization' = 'Bearer ' + $tokenStr + } $url = "$($script:BaseApiUrl)$($adfi.DataFactoryId)/credentials?api-version=2018-06-01" # Retrieve all credentials via Rest API @@ -14,7 +21,7 @@ function Get-AzDFV2Credential { return $null } - [System.Collections.ArrayList] $items = @{} + [System.Collections.ArrayList] $items = @() ($r.Content | ConvertFrom-Json).value | ForEach-Object { $i = [AdfPSCredential]::New($_); $items.Add($i) | Out-Null; } Write-Debug "END: Get-AzDFV2Credential()" diff --git a/private/Remove-AdfObjectRestAPI.ps1 b/private/Remove-AdfObjectRestAPI.ps1 index 683da8c..8aba3d6 100644 --- a/private/Remove-AdfObjectRestAPI.ps1 +++ b/private/Remove-AdfObjectRestAPI.ps1 @@ -8,6 +8,12 @@ function Remove-AdfObjectRestAPI { Write-Debug "BEGIN: Remove-AdfObjectRestAPI()" + $token = Get-AzAccessToken -ResourceUrl 'https://management.azure.com' + $tokenStr = [System.Net.NetworkCredential]::new('', $token.Token).Password + $authHeader = @{ + 'Content-Type' = 'application/json' + 'Authorization' = 'Bearer ' + $tokenStr + } $url = "$($script:BaseApiUrl)$($adfInstance.Id)/$type_plural/$($name)?api-version=2018-06-01" # Delete given object via Rest API diff --git a/private/Start-Trigger.ps1 b/private/Start-Trigger.ps1 index e291689..8cd37ee 100644 --- a/private/Start-Trigger.ps1 +++ b/private/Start-Trigger.ps1 @@ -23,15 +23,34 @@ function Start-Trigger { break } catch { + $errMsg = $_.Exception.Message if ($i -lt $attempts) { - Write-Verbose "Attempt #$i of starting trigger failed. Retry in 2 seconds." - Start-Sleep -Seconds 2 - } - else + if ($errMsg -like "*cannot be updated during provisioning*") { + Write-Verbose "Attempt #${i}: Trigger '$Name' is still provisioning. Waiting for provisioning to complete..." + $provisioningTimeout = 300 + $pollInterval = 10 + $elapsed = 0 + while ($elapsed -lt $provisioningTimeout) { + Start-Sleep -Seconds $pollInterval + $elapsed += $pollInterval + $trigger = Get-AzDataFactoryV2Trigger ` + -ResourceGroupName $ResourceGroupName ` + -DataFactoryName $DataFactoryName ` + -Name $Name + $provState = $trigger.Properties.ProvisioningState + Write-Verbose "Trigger provisioning state: $provState ($elapsed sec elapsed)" + if ($provState -ne 'Provisioning') { break } + } + } else { + Write-Verbose "Attempt #$i of starting trigger failed. Retry in 2 seconds." + Start-Sleep -Seconds 2 + } + } + else { Write-Host "Failed starting trigger after $attempts attempts." - Write-Warning -Message $_.Exception.Message + Write-Warning -Message $errMsg } } } diff --git a/test/!RunAllTests.ps1 b/test/!RunAllTests.ps1 index 4ffe0ef..ba01f7c 100644 --- a/test/!RunAllTests.ps1 +++ b/test/!RunAllTests.ps1 @@ -8,6 +8,9 @@ Param( [Parameter(Mandatory=$false)] [Switch]$MajorRelease, + [Parameter(Mandatory=$false)] + [string]$ResultOutputFormat = "NUnitXml", + [Parameter(Mandatory=$true)] [Switch]$InstallModules ) @@ -70,7 +73,7 @@ try { $configuration.Run.Exit = $true $configuration.Should.ErrorAction = 'Continue' $configuration.CodeCoverage.Enabled = $false - $configuration.TestResult.OutputFormat = "NUnitXml" + $configuration.TestResult.OutputFormat = "JUnitXml" $configuration.TestResult.OutputPath = "$folder\TEST-Results.xml" $configuration.TestResult.Enabled = $true $configuration.Output.Verbosity = 'Detailed' diff --git a/test/Get-AzDFV2Credential.Tests.ps1 b/test/Get-AzDFV2Credential.Tests.ps1 new file mode 100644 index 0000000..2b80afd --- /dev/null +++ b/test/Get-AzDFV2Credential.Tests.ps1 @@ -0,0 +1,111 @@ +BeforeDiscovery { + $ModuleRootPath = $PSScriptRoot | Split-Path -Parent + $moduleManifestName = 'azure.datafactory.tools.psd1' + $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName + + Import-Module -Name $moduleManifestPath -Force -Verbose:$false +} + +InModuleScope azure.datafactory.tools { + $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' + Import-Module -Name $testHelperPath -Force + + Describe 'Get-AzDFV2Credential' -Tag 'Unit' { + + It 'Should exist' { + { Get-Command -Name Get-AzDFV2Credential -ErrorAction Stop } | Should -Not -Throw + } + + Context 'When the API returns no credentials' { + BeforeEach { + $script:adfi = [PSCustomObject]@{ DataFactoryId = '/subscriptions/sub-123/resourceGroups/rg/providers/Microsoft.DataFactory/factories/adf1' } + + Mock Get-AzAccessToken { + return [PSCustomObject]@{ Token = 'fake-token-abc' } + } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ StatusCode = 200; Content = '{"value":[]}' } + } + } + + It 'Should return an empty list' { + $result = Get-AzDFV2Credential -adfi $script:adfi + @($result).Count | Should -Be 0 + } + + It 'Should call Get-AzAccessToken once' { + Get-AzDFV2Credential -adfi $script:adfi + Assert-MockCalled Get-AzAccessToken -Times 1 -Exactly + } + + It 'Should call Invoke-AzRestMethod once' { + Get-AzDFV2Credential -adfi $script:adfi + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly + } + + It 'Should call Invoke-AzRestMethod with correct credentials URL' { + Get-AzDFV2Credential -adfi $script:adfi + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly -ParameterFilter { + $Uri -eq "https://management.azure.com$($script:adfi.DataFactoryId)/credentials?api-version=2018-06-01" + } + } + + It 'Should call Invoke-AzRestMethod using GET method' { + Get-AzDFV2Credential -adfi $script:adfi + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly -ParameterFilter { + $Method -eq 'GET' + } + } + } + + Context 'When the API returns multiple credentials' { + BeforeEach { + $script:adfi = [PSCustomObject]@{ DataFactoryId = '/subscriptions/sub-123/resourceGroups/rg/providers/Microsoft.DataFactory/factories/adf1' } + + $cred1 = [PSCustomObject]@{ name = 'cred1'; type = 'Microsoft.DataFactory/factories/credentials'; properties = @{} } + $cred2 = [PSCustomObject]@{ name = 'cred2'; type = 'Microsoft.DataFactory/factories/credentials'; properties = @{} } + $script:credJson = @{ value = @($cred1, $cred2) } | ConvertTo-Json -Depth 5 + + Mock Get-AzAccessToken { + return [PSCustomObject]@{ Token = 'fake-token-abc' } + } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ StatusCode = 200; Content = $script:credJson } + } + } + + It 'Should return all credentials as AdfPSCredential objects' { + $result = Get-AzDFV2Credential -adfi $script:adfi + $result.Count | Should -Be 2 + $result | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSCredential' } + } + + It 'Should populate the Name property from the API response' { + $result = Get-AzDFV2Credential -adfi $script:adfi + $result[0].Name | Should -Be 'cred1' + $result[1].Name | Should -Be 'cred2' + } + + It 'Should populate the Child property with the raw API object' { + $result = Get-AzDFV2Credential -adfi $script:adfi + $result[0].Child.name | Should -Be 'cred1' + $result[1].Child.name | Should -Be 'cred2' + } + } + + Context 'When Invoke-AzRestMethod throws' { + BeforeEach { + $script:adfi = [PSCustomObject]@{ DataFactoryId = '/subscriptions/sub-123/resourceGroups/rg/providers/Microsoft.DataFactory/factories/adf1' } + + Mock Get-AzAccessToken { + return [PSCustomObject]@{ Token = 'fake-token-abc' } + } + Mock Invoke-AzRestMethod { throw 'Unauthorized' } + } + + It 'Should propagate the error' { + { Get-AzDFV2Credential -adfi $script:adfi } | Should -Throw + } + } + } +} diff --git a/test/Publish-AdfV2FromJson-1.Tests.ps1 b/test/Publish-AdfV2FromJson-1.Tests.ps1 index 8bd6aec..d9df0d9 100644 --- a/test/Publish-AdfV2FromJson-1.Tests.ps1 +++ b/test/Publish-AdfV2FromJson-1.Tests.ps1 @@ -34,6 +34,17 @@ InModuleScope azure.datafactory.tools { -Location $script:Location -DataFactoryName $script:DataFactoryName -Option $o } + It 'adf2 Should deploy credential1 to ADF service' { + $script:RootFolder = Join-Path $PSScriptRoot "adf2" + $o = New-AdfPublishOption + $o.StopStartTriggers = $false + $o.Includes.Add("credential.credential1", "") + Publish-AdfV2FromJson -RootFolder $script:RootFolder -ResourceGroupName $script:ResourceGroupName ` + -Location $script:Location -DataFactoryName $script:DataFactoryName -Option $o + $adfIns = Get-AdfFromService -FactoryName $script:DataFactoryName -ResourceGroupName $script:ResourceGroupName + $adfIns.Credentials | Where-Object { $_.Name -eq 'credential1' } | Should -Not -BeNullOrEmpty + } + } } diff --git a/test/Start-Trigger.Tests.ps1 b/test/Start-Trigger.Tests.ps1 index 6e203ee..b3d8186 100644 --- a/test/Start-Trigger.Tests.ps1 +++ b/test/Start-Trigger.Tests.ps1 @@ -25,18 +25,38 @@ InModuleScope azure.datafactory.tools { It 'Should retry after the first failure' { $script:attempts = 2 - Mock Start-AzDataFactoryV2Trigger { $attempts--; if ($attempts -eq 0) { Write-Host 'OK' } else { Write-Error 'BadRequest' } } - Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' + Mock Start-AzDataFactoryV2Trigger { $attempts--; if ($attempts -eq 0) { Write-Host 'OK' } else { throw 'BadRequest' } } + Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' Assert-MockCalled Start-AzDataFactoryV2Trigger -Times 2 } It 'Should retry max 5 times when failure' { - Mock Start-AzDataFactoryV2Trigger { Write-Error 'BadRequest' } - Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' + Mock Start-AzDataFactoryV2Trigger { throw 'BadRequest' } + Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' Assert-MockCalled Start-AzDataFactoryV2Trigger -Times 5 } - } + It 'Should wait for provisioning and retry when trigger cannot be updated during provisioning' { + $script:startAttempt = 0 + Mock Start-AzDataFactoryV2Trigger { + $script:startAttempt++ + if ($script:startAttempt -eq 1) { throw 'BadRequest: Resource cannot be updated during provisioning' } + } + $script:getAttempt = 0 + Mock Get-AzDataFactoryV2Trigger { + $script:getAttempt++ + $state = if ($script:getAttempt -eq 1) { 'Provisioning' } else { 'Succeeded' } + return [PSCustomObject]@{ Properties = [PSCustomObject]@{ ProvisioningState = $state } } + } + Mock Start-Sleep {} + + Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' + + Assert-MockCalled Start-AzDataFactoryV2Trigger -Times 2 + Assert-MockCalled Get-AzDataFactoryV2Trigger -Times 2 + } + + } } diff --git a/test/TestHelper/TestHelper.psm1 b/test/TestHelper/TestHelper.psm1 index e92bc68..b116f50 100644 --- a/test/TestHelper/TestHelper.psm1 +++ b/test/TestHelper/TestHelper.psm1 @@ -226,6 +226,9 @@ function Get-TargetEnv { SrcFolder = "$rootPath\$AdfOrigName" } $c = Get-AzContext + if ($null -eq $c) { + throw "No active Azure context found. Run Connect-AzAccount before executing integration tests." + } $guid = $c.Subscription.Id.Substring(0,8) $target.DataFactoryName = $AdfOrigName + "-$guid" return $target