From 58e68d92120063c8df0c31aec49f6c03cdb65433 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 27 May 2026 01:57:25 +0100 Subject: [PATCH 01/20] This yml should only starts when file changes on `private` or `public` or `test` folder. --- .github/workflows/develop-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/develop-ci.yml b/.github/workflows/develop-ci.yml index 942fb9a..9ada017 100644 --- a/.github/workflows/develop-ci.yml +++ b/.github/workflows/develop-ci.yml @@ -3,8 +3,16 @@ name: CI on: push: branches: [ develop ] + paths: + - 'private/**' + - 'public/**' + - 'test/**' pull_request: branches: [ develop ] + paths: + - 'private/**' + - 'public/**' + - 'test/**' workflow_dispatch: permissions: From 5f1f07f3eb6cbcd2a1bba1887edc633d2cf65586 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 27 May 2026 02:16:52 +0100 Subject: [PATCH 02/20] Fixed #480 (preview) --- azure.datafactory.tools.psd1 | 2 +- changelog.md | 5 +++ private/AdfPSObjects.class.ps1 | 40 +++++++++++++++++++ private/Get-AdfObjectsFromServiceRestAPI.ps1 | 36 +++++++++++++++++ public/Get-AdfFromService.ps1 | 42 +++++++++++++++++--- 5 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 private/AdfPSObjects.class.ps1 create mode 100644 private/Get-AdfObjectsFromServiceRestAPI.ps1 diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index 8bccb32..f99e0db 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -12,7 +12,7 @@ RootModule = 'azure.datafactory.tools.psm1' # Version number of this module. - ModuleVersion = '1.15.0' + ModuleVersion = '1.16.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/changelog.md b/changelog.md index 91a3b61..6b73285 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.16.0] - 2026-05-27 +### Fixed +* `DeleteNotInSource` no longer fails with "Unable to deserialize the response" when ADF contains object types unsupported by the installed Az.DataFactory module version - REST API fallback added to `Get-AdfFromService` #480 #473 + + ## [1.15.0] - 2026-05-26 ### Added * Credential type of ADF objects is now supported for deployment via REST API diff --git a/private/AdfPSObjects.class.ps1 b/private/AdfPSObjects.class.ps1 new file mode 100644 index 0000000..f2ca37c --- /dev/null +++ b/private/AdfPSObjects.class.ps1 @@ -0,0 +1,40 @@ + +# Lightweight wrapper classes for ADF objects retrieved via REST API. +# Class names follow the AdfPS pattern so that Get-SimplifiedType +# strips the "AdfPS" prefix and returns the correct simplified type (e.g. "Dataset"). +# These are used as a fallback when Get-AzDataFactoryV2* cmdlets fail to deserialize +# objects whose types are not yet known to the installed Az.DataFactory module version. + +class AdfPSDataset { + [String] $Name + AdfPSDataset([String] $name) { $this.Name = $name } +} + +class AdfPSDataFlow { + [String] $Name + AdfPSDataFlow([String] $name) { $this.Name = $name } +} + +class AdfPSPipeline { + [String] $Name + AdfPSPipeline([String] $name) { $this.Name = $name } +} + +class AdfPSLinkedService { + [String] $Name + AdfPSLinkedService([String] $name) { $this.Name = $name } +} + +class AdfPSIntegrationRuntime { + [String] $Name + AdfPSIntegrationRuntime([String] $name) { $this.Name = $name } +} + +class AdfPSTrigger { + [String] $Name + [String] $RuntimeState + AdfPSTrigger([String] $name, [String] $runtimeState) { + $this.Name = $name + $this.RuntimeState = $runtimeState + } +} diff --git a/private/Get-AdfObjectsFromServiceRestAPI.ps1 b/private/Get-AdfObjectsFromServiceRestAPI.ps1 new file mode 100644 index 0000000..a9ad167 --- /dev/null +++ b/private/Get-AdfObjectsFromServiceRestAPI.ps1 @@ -0,0 +1,36 @@ +function Get-AdfObjectsFromServiceRestAPI { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] $adfi, + [parameter(Mandatory = $true)] [String] $typePlural, + [parameter(Mandatory = $true)] [String] $simpleType + ) + + Write-Debug "BEGIN: Get-AdfObjectsFromServiceRestAPI(typePlural=$typePlural, simpleType=$simpleType)" + + $url = "$($script:BaseApiUrl)$($adfi.DataFactoryId)/$typePlural`?api-version=2018-06-01" + $r = Invoke-AzRestMethod -Method 'GET' -Uri $url + if ($r.StatusCode -ne 200) { + Write-Error -Message "Unexpected response code: $($r.StatusCode) from REST API when listing $typePlural." + return $null + } + + [System.Collections.ArrayList] $items = @() + ($r.Content | ConvertFrom-Json).value | ForEach-Object { + $name = $_.name + $obj = $null + switch ($simpleType) { + 'Dataset' { $obj = [AdfPSDataset]::New($name) } + 'DataFlow' { $obj = [AdfPSDataFlow]::New($name) } + 'Pipeline' { $obj = [AdfPSPipeline]::New($name) } + 'LinkedService' { $obj = [AdfPSLinkedService]::New($name) } + 'IntegrationRuntime' { $obj = [AdfPSIntegrationRuntime]::New($name) } + 'Trigger' { $obj = [AdfPSTrigger]::New($name, [string]$_.properties.runtimeState) } + default { Write-Warning "Get-AdfObjectsFromServiceRestAPI: unsupported type '$simpleType'" } + } + if ($null -ne $obj) { $items.Add($obj) | Out-Null } + } + + Write-Debug "END: Get-AdfObjectsFromServiceRestAPI()" + return $items +} diff --git a/public/Get-AdfFromService.ps1 b/public/Get-AdfFromService.ps1 index 40476c9..37cd98f 100644 --- a/public/Get-AdfFromService.ps1 +++ b/public/Get-AdfFromService.ps1 @@ -37,17 +37,47 @@ function Get-AdfFromService { $adf.Id = $adfi.DataFactoryId $adf.Location = $adfi.Location - $adf.DataSets = Get-AzDataFactoryV2Dataset -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.DataSets = Get-AzDataFactoryV2Dataset -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + } catch { + Write-Warning "Failed to list Datasets via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.DataSets = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'datasets' -simpleType 'Dataset' | ToArray + } Write-Host ("DataSets: {0} object(s) loaded." -f $adf.DataSets.Count) - $adf.IntegrationRuntimes = Get-AzDataFactoryV2IntegrationRuntime -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.IntegrationRuntimes = Get-AzDataFactoryV2IntegrationRuntime -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + } catch { + Write-Warning "Failed to list IntegrationRuntimes via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.IntegrationRuntimes = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'integrationruntimes' -simpleType 'IntegrationRuntime' | ToArray + } Write-Host ("IntegrationRuntimes: {0} object(s) loaded." -f $adf.IntegrationRuntimes.Count) - $adf.LinkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.LinkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + } catch { + Write-Warning "Failed to list LinkedServices via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.LinkedServices = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'linkedservices' -simpleType 'LinkedService' | ToArray + } Write-Host ("LinkedServices: {0} object(s) loaded." -f $adf.LinkedServices.Count) - $adf.Pipelines = Get-AzDataFactoryV2Pipeline -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.Pipelines = Get-AzDataFactoryV2Pipeline -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + } catch { + Write-Warning "Failed to list Pipelines via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.Pipelines = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'pipelines' -simpleType 'Pipeline' | ToArray + } Write-Host ("Pipelines: {0} object(s) loaded." -f $adf.Pipelines.Count) - $adf.DataFlows = Get-AzDataFactoryV2DataFlow -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.DataFlows = Get-AzDataFactoryV2DataFlow -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + } catch { + Write-Warning "Failed to list DataFlows via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.DataFlows = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'dataflows' -simpleType 'DataFlow' | ToArray + } Write-Host ("DataFlows: {0} object(s) loaded." -f $adf.DataFlows.Count) - $adf.Triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.Triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + } catch { + Write-Warning "Failed to list Triggers via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.Triggers = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'triggers' -simpleType 'Trigger' | ToArray + } Write-Host ("Triggers: {0} object(s) loaded." -f $adf.Triggers.Count) $adf.Credentials = Get-AzDFV2Credential -adfi $adfi | ToArray Write-Host ("Credentials: {0} object(s) loaded." -f $adf.Credentials.Count) From 4f6d8a736fdbc8e508608154452e0d86f6849134 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 27 May 2026 03:05:21 +0100 Subject: [PATCH 03/20] Initial fix #472 --- changelog.md | 4 + private/Deploy-AdfObject.ps1 | 2 +- private/Deploy-AdfObjectOnly.ps1 | 4 +- private/Update-PropertiesFromFile.ps1 | 4 +- public/Publish-AdfV2FromJson.ps1 | 8 +- ...lish-AdfV2FromJson-ErrorHandling.Tests.ps1 | 141 ++++++++++++++++++ 6 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 test/Publish-AdfV2FromJson-ErrorHandling.Tests.ps1 diff --git a/changelog.md b/changelog.md index 91a3b61..13cbd5b 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## Unreleased +### Fixed +* `Publish-AdfV2FromJson` no longer silently continues after errors: config path errors (ADFT0010) and deployment failures (e.g. Azure Policy blocks) now propagate as terminating errors to the caller #472 + ## [1.15.0] - 2026-05-26 ### Added * Credential type of ADF objects is now supported for deployment via REST API diff --git a/private/Deploy-AdfObject.ps1 b/private/Deploy-AdfObject.ps1 index 9b67c54..d6b0ce4 100644 --- a/private/Deploy-AdfObject.ps1 +++ b/private/Deploy-AdfObject.ps1 @@ -31,7 +31,7 @@ function Deploy-AdfObject { if ($adf.PublishOptions.IgnoreLackOfReferencedObject -eq $true) { Write-Warning "ADFT0006: Referenced object [$type].[$name] was not found. No error raised as user wanted to carry on." } else { - Write-Error "ADFT0005: Referenced object [$type].[$name] was not found." + throw "ADFT0005: Referenced object [$type].[$name] was not found." } } elseif ($type -notin [AdfObject]::IgnoreTypes) { Deploy-AdfObject -obj $depobj diff --git a/private/Deploy-AdfObjectOnly.ps1 b/private/Deploy-AdfObjectOnly.ps1 index 1398035..0d9af2e 100644 --- a/private/Deploy-AdfObjectOnly.ps1 +++ b/private/Deploy-AdfObjectOnly.ps1 @@ -73,7 +73,7 @@ function Deploy-AdfObjectOnly { -Force | Out-Null } else { - Write-Error "ADFT0012: Deployment for this kind of Integration Runtime is not supported yet." + throw "ADFT0012: Deployment for this kind of Integration Runtime is not supported yet." } } 'linkedService' @@ -155,7 +155,7 @@ function Deploy-AdfObjectOnly { } default { - Write-Error "ADFT0013: Type $($obj.Type) is not supported." + throw "ADFT0013: Type $($obj.Type) is not supported." } } diff --git a/private/Update-PropertiesFromFile.ps1 b/private/Update-PropertiesFromFile.ps1 index 542675a..f62f900 100644 --- a/private/Update-PropertiesFromFile.ps1 +++ b/private/Update-PropertiesFromFile.ps1 @@ -72,7 +72,7 @@ function Update-PropertiesFromFile { if ($option.FailsWhenConfigItemNotFound -eq $false) { Write-Warning "Could not find object: $type.$name, skipping..." } else { - Write-Error -Message "ADFT0007: Could not find object: $type.$name" + throw "ADFT0007: Could not find object: $type.$name" } } else { Write-Verbose "- Performing: $action for object(path): $type.$name(properties.$path)" @@ -131,7 +131,7 @@ function Update-PropertiesForObject { Write-Warning "Wrong path defined in config for object(path): $type.$name(properties.$path), skipping..." } else { $exc = ([System.Data.DataException]::new("ADFT0010: Wrong path defined in config for object(path): $type.$name(properties.$path)")) - Write-Error -Exception $exc + throw $exc } } diff --git a/public/Publish-AdfV2FromJson.ps1 b/public/Publish-AdfV2FromJson.ps1 index 0f079e0..3d687e7 100644 --- a/public/Publish-AdfV2FromJson.ps1 +++ b/public/Publish-AdfV2FromJson.ps1 @@ -272,16 +272,16 @@ function Publish-AdfV2FromJson { $adf.Factories[0].ToBeDeployed = $false } } - $adf.AllObjects() | ForEach-Object { - Deploy-AdfObject -obj $_ + foreach ($obj in $adf.AllObjects()) { + Deploy-AdfObject -obj $obj } Write-Host "==================================================================================="; Write-Host "STEP: Deleting objects not in source ..." if ($opt.DeleteNotInSource -eq $true) { $adfIns = Get-AdfFromService -FactoryName "$DataFactoryName" -ResourceGroupName "$ResourceGroupName" - $adfIns.AllObjects() | ForEach-Object { - Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $_ -adfInstance $adfIns + foreach ($targetObj in $adfIns.AllObjects()) { + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $targetObj -adfInstance $adfIns } Write-Host "Deleted $($adf.DeletedObjectNames.Count) objects from ADF service." } else { diff --git a/test/Publish-AdfV2FromJson-ErrorHandling.Tests.ps1 b/test/Publish-AdfV2FromJson-ErrorHandling.Tests.ps1 new file mode 100644 index 0000000..9a4a3a2 --- /dev/null +++ b/test/Publish-AdfV2FromJson-ErrorHandling.Tests.ps1 @@ -0,0 +1,141 @@ +BeforeDiscovery { + $ModuleRootPath = $PSScriptRoot | Split-Path -Parent + $moduleManifestName = 'azure.datafactory.tools.psd1' + $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName + Import-Module -Name $moduleManifestPath -Force -Verbose:$false +} + +InModuleScope azure.datafactory.tools { + $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' + Import-Module -Name $testHelperPath -Force + + $script:SrcFolder = "$PSScriptRoot\BigFactorySample2" + $script:TmpFolder = (New-TemporaryDirectory).FullName + $script:RootFolder = Join-Path -Path $script:TmpFolder -ChildPath (Split-Path -Path $script:SrcFolder -Leaf) + Copy-Item -Path "$SrcFolder" -Destination "$TmpFolder" -Filter "*.*" -Recurse:$true -Force + + AfterAll { + Remove-Item -Path $script:TmpFolder -Recurse -Force -ErrorAction SilentlyContinue + } + + # ------------------------------------------------------------------------- + # Issue #472 – Scenario 1: wrong property path in config (ADFT0010) + # + # config-c004-wrongpath.csv contains "typeProperties:baseUrl" (colon + # separator instead of dot), which causes Write-Error "ADFT0010" inside + # Update-PropertiesForObject. The error is non-terminating, so the + # ForEach-Object loop in Update-PropertiesFromFile swallows it and + # Publish-AdfV2FromJson continues to print "deployed successfully". + # + # Expected fix: the error must propagate as a terminating error so the + # caller observes a failure regardless of $ErrorActionPreference scope. + # ------------------------------------------------------------------------- + Describe 'Publish-AdfV2FromJson - Error propagation for wrong config path (Issue #472)' -Tag 'Unit' { + + Context 'When stage config has a wrong property path and FailsWhenPathNotFound is true (default)' { + + It 'Should throw when Publish-AdfV2FromJson is called with -DryRun' { + # BUG REPRODUCTION: currently does NOT throw (Write-Error is swallowed). + # After fix this test must pass. + $o = New-AdfPublishOption + # FailsWhenPathNotFound defaults to $true + { + Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName 'rg-test' ` + -DataFactoryName 'adf-test' ` + -Stage 'c004-wrongpath' ` + -Option $o ` + -DryRun + } | Should -Throw + } + } + + Context 'When stage config has a wrong property path and FailsWhenPathNotFound is false' { + + It 'Should not throw (wrong path is only a warning)' { + $o = New-AdfPublishOption + $o.FailsWhenPathNotFound = $false + { + Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName 'rg-test' ` + -DataFactoryName 'adf-test' ` + -Stage 'c004-wrongpath' ` + -Option $o ` + -DryRun + } | Should -Not -Throw + } + } + } + + # ------------------------------------------------------------------------- + # Issue #472 – Scenario 2: ADF object deployment fails at runtime + # + # When Deploy-AdfObjectOnly emits a non-terminating Write-Error (e.g. an + # Azure Policy block: "RequestDisallowedByPolicy"), the ForEach-Object + # pipeline in Publish-AdfV2FromJson absorbs the error and the function + # continues to report "deployed successfully". + # + # Expected fix: the deployment loop must propagate errors so the function + # terminates and the caller sees a failure. + # ------------------------------------------------------------------------- + Describe 'Publish-AdfV2FromJson - Error propagation for deployment failure (Issue #472)' -Tag 'Unit' { + + Context 'When an ADF object deployment produces a terminating error' { + + BeforeAll { + Mock Get-AzDataFactoryV2 { + [PSCustomObject]@{ Name = 'adf-test'; ResourceGroupName = 'rg-test' } + } + # Azure cmdlets (e.g. New-AzResource) throw terminating errors for + # serious failures like Azure Policy blocks. In PowerShell 5.1 the + # ForEach-Object pipeline was absorbing these terminating errors and + # allowing the loop to continue, so the function reported success. + # The fix replaces ForEach-Object with a foreach statement, which + # propagates terminating errors correctly in all PS versions. + Mock Deploy-AdfObjectOnly { + throw 'Simulated: RequestDisallowedByPolicy - resource was disallowed by policy' + } + } + + It 'Should throw when any object deployment fails' { + # BUG REPRODUCTION: previously did NOT throw because ForEach-Object + # absorbed the terminating error in PS 5.1. + # After fix (foreach loop) this test must pass. + $o = New-AdfPublishOption + $o.StopStartTriggers = $false + $o.DeleteNotInSource = $false + { + Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName 'rg-test' ` + -DataFactoryName 'adf-test' ` + -Option $o + } | Should -Throw + } + + It 'Should throw even when the caller has not set $ErrorActionPreference to Stop' { + # BUG REPRODUCTION: the module scope isolates $ErrorActionPreference + # from the caller's script scope, so setting it externally has no effect. + # The fix must make the function fail independently of the caller's EAP. + $o = New-AdfPublishOption + $o.StopStartTriggers = $false + $o.DeleteNotInSource = $false + $threw = $false + # Deliberately NOT using Should -Throw so Pester does not alter EAP + try { + Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName 'rg-test' ` + -DataFactoryName 'adf-test' ` + -Option $o + } + catch { + $threw = $true + } + $threw | Should -BeTrue -Because 'deployment failures must propagate to the caller' + } + } + } +} From 73939583cd4739e05115f7bfc50bb9f25f83aae7 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Tue, 9 Jun 2026 22:44:13 +0100 Subject: [PATCH 04/20] All in one CI/CD --- .github/workflows/ci-cd.yml | 55 +++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/ci-cd.yml diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml new file mode 100644 index 0000000..3ddcf5f --- /dev/null +++ b/.github/workflows/ci-cd.yml @@ -0,0 +1,55 @@ +name: CI/CD + +on: + push: + branches: ['**'] + pull_request: + workflow_dispatch: + +permissions: + contents: read + checks: write # required by dorny/test-reporter + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Run Unit Tests + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + + - name: Run Unit Tests + shell: pwsh + run: | + test/!RunAllTests.ps1 -folder '${{ github.workspace }}/' -TestFilenameFilter "*" -InstallModules:$false -ResultOutputFormat "JUnitXml" + + - name: Publish Test Results + uses: dorny/test-reporter@v3 + if: ${{ !cancelled() }} + with: + name: Pester Tests + path: '**/TEST-*.xml' + reporter: java-junit + collapsed: 'auto' + + publish: + name: Publish to PS Gallery + runs-on: windows-latest + needs: test + if: ${{ github.event_name != 'pull_request' }} + + steps: + - uses: actions/checkout@v4 + + - name: Publish module to PowerShell Gallery + shell: pwsh + env: + PSGALLERY_API_KEY: ${{ secrets.PSGALLERY_API_KEY }} + run: | + $isPreview = '${{ github.ref_name }}' -ne 'master' + Write-Host "Branch: ${{ github.ref_name }} | Preview: $isPreview" + ./tools/Publish-ToGallery.ps1 -IsPreview:$isPreview -NuGetApiKey $env:PSGALLERY_API_KEY From 16f2fd740d988a274fc0882fb5a48fa604bd1b15 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 10 Jun 2026 03:51:30 +0100 Subject: [PATCH 05/20] Login to run correct tests --- .github/workflows/ci-cd.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 3ddcf5f..0c93938 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -22,10 +22,18 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Login via Az module + uses: azure/login@v3 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + enable-AzPSSession: true + - name: Run Unit Tests - shell: pwsh - run: | - test/!RunAllTests.ps1 -folder '${{ github.workspace }}/' -TestFilenameFilter "*" -InstallModules:$false -ResultOutputFormat "JUnitXml" + uses: azure/powershell@v3 + with: + inlineScript: | + test/!RunAllTests.ps1 -folder '${{ github.workspace }}/' -TestFilenameFilter "*" -InstallModules:$false -ResultOutputFormat "JUnitXml" + azPSVersion: "latest" - name: Publish Test Results uses: dorny/test-reporter@v3 From bcbdc3473f59f687dedcc558dcc7a60c74eb0176 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 10 Jun 2026 03:57:46 +0100 Subject: [PATCH 06/20] deploy preview module version only from `develop` branch --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 0c93938..bdcabaf 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -48,7 +48,7 @@ jobs: name: Publish to PS Gallery runs-on: windows-latest needs: test - if: ${{ github.event_name != 'pull_request' }} + if: ${{ github.event_name != 'pull_request' && (github.ref_name == 'master' || github.ref_name == 'develop') }} steps: - uses: actions/checkout@v4 From c4bc7b70908536170a8d26aed33f240f12f543d5 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Tue, 30 Jun 2026 22:58:03 +0100 Subject: [PATCH 07/20] Refactor Get-AdfObjectsFromServiceRestAPI to handle pagination and improve error handling; update Get-AdfFromService to use -ErrorAction Stop for Az cmdlets --- private/Get-AdfObjectsFromServiceRestAPI.ps1 | 41 +-- public/Get-AdfFromService.ps1 | 12 +- test/zGet-AdfFromService.Tests.ps1 | 303 ++++++++++++++++++- 3 files changed, 325 insertions(+), 31 deletions(-) diff --git a/private/Get-AdfObjectsFromServiceRestAPI.ps1 b/private/Get-AdfObjectsFromServiceRestAPI.ps1 index a9ad167..e27f563 100644 --- a/private/Get-AdfObjectsFromServiceRestAPI.ps1 +++ b/private/Get-AdfObjectsFromServiceRestAPI.ps1 @@ -9,27 +9,32 @@ function Get-AdfObjectsFromServiceRestAPI { Write-Debug "BEGIN: Get-AdfObjectsFromServiceRestAPI(typePlural=$typePlural, simpleType=$simpleType)" $url = "$($script:BaseApiUrl)$($adfi.DataFactoryId)/$typePlural`?api-version=2018-06-01" - $r = Invoke-AzRestMethod -Method 'GET' -Uri $url - if ($r.StatusCode -ne 200) { - Write-Error -Message "Unexpected response code: $($r.StatusCode) from REST API when listing $typePlural." - return $null - } [System.Collections.ArrayList] $items = @() - ($r.Content | ConvertFrom-Json).value | ForEach-Object { - $name = $_.name - $obj = $null - switch ($simpleType) { - 'Dataset' { $obj = [AdfPSDataset]::New($name) } - 'DataFlow' { $obj = [AdfPSDataFlow]::New($name) } - 'Pipeline' { $obj = [AdfPSPipeline]::New($name) } - 'LinkedService' { $obj = [AdfPSLinkedService]::New($name) } - 'IntegrationRuntime' { $obj = [AdfPSIntegrationRuntime]::New($name) } - 'Trigger' { $obj = [AdfPSTrigger]::New($name, [string]$_.properties.runtimeState) } - default { Write-Warning "Get-AdfObjectsFromServiceRestAPI: unsupported type '$simpleType'" } + do { + $r = Invoke-AzRestMethod -Method 'GET' -Uri $url + if ($r.StatusCode -ne 200) { + Write-Error -Message "Unexpected response code: $($r.StatusCode) from REST API when listing $typePlural." + return $null } - if ($null -ne $obj) { $items.Add($obj) | Out-Null } - } + $content = $r.Content | ConvertFrom-Json + $url = if ($content.PSObject.Properties['nextLink']) { $content.nextLink } else { $null } + $content.value | ForEach-Object { + $item = $_ + $name = $item.name + $obj = $null + switch ($simpleType) { + 'Dataset' { $obj = [AdfPSDataset]::New($name) } + 'DataFlow' { $obj = [AdfPSDataFlow]::New($name) } + 'Pipeline' { $obj = [AdfPSPipeline]::New($name) } + 'LinkedService' { $obj = [AdfPSLinkedService]::New($name) } + 'IntegrationRuntime' { $obj = [AdfPSIntegrationRuntime]::New($name) } + 'Trigger' { $obj = [AdfPSTrigger]::New($name, [string]$item.properties.runtimeState) } + default { Write-Warning "Get-AdfObjectsFromServiceRestAPI: unsupported type '$simpleType'" } + } + if ($null -ne $obj) { $items.Add($obj) | Out-Null } + } + } while ($url) Write-Debug "END: Get-AdfObjectsFromServiceRestAPI()" return $items diff --git a/public/Get-AdfFromService.ps1 b/public/Get-AdfFromService.ps1 index 37cd98f..0e577a9 100644 --- a/public/Get-AdfFromService.ps1 +++ b/public/Get-AdfFromService.ps1 @@ -38,42 +38,42 @@ function Get-AdfFromService { $adf.Location = $adfi.Location try { - $adf.DataSets = Get-AzDataFactoryV2Dataset -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + $adf.DataSets = Get-AzDataFactoryV2Dataset -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray } catch { Write-Warning "Failed to list Datasets via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" $adf.DataSets = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'datasets' -simpleType 'Dataset' | ToArray } Write-Host ("DataSets: {0} object(s) loaded." -f $adf.DataSets.Count) try { - $adf.IntegrationRuntimes = Get-AzDataFactoryV2IntegrationRuntime -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + $adf.IntegrationRuntimes = Get-AzDataFactoryV2IntegrationRuntime -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray } catch { Write-Warning "Failed to list IntegrationRuntimes via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" $adf.IntegrationRuntimes = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'integrationruntimes' -simpleType 'IntegrationRuntime' | ToArray } Write-Host ("IntegrationRuntimes: {0} object(s) loaded." -f $adf.IntegrationRuntimes.Count) try { - $adf.LinkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + $adf.LinkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray } catch { Write-Warning "Failed to list LinkedServices via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" $adf.LinkedServices = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'linkedservices' -simpleType 'LinkedService' | ToArray } Write-Host ("LinkedServices: {0} object(s) loaded." -f $adf.LinkedServices.Count) try { - $adf.Pipelines = Get-AzDataFactoryV2Pipeline -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + $adf.Pipelines = Get-AzDataFactoryV2Pipeline -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray } catch { Write-Warning "Failed to list Pipelines via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" $adf.Pipelines = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'pipelines' -simpleType 'Pipeline' | ToArray } Write-Host ("Pipelines: {0} object(s) loaded." -f $adf.Pipelines.Count) try { - $adf.DataFlows = Get-AzDataFactoryV2DataFlow -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + $adf.DataFlows = Get-AzDataFactoryV2DataFlow -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray } catch { Write-Warning "Failed to list DataFlows via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" $adf.DataFlows = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'dataflows' -simpleType 'DataFlow' | ToArray } Write-Host ("DataFlows: {0} object(s) loaded." -f $adf.DataFlows.Count) try { - $adf.Triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + $adf.Triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray } catch { Write-Warning "Failed to list Triggers via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" $adf.Triggers = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'triggers' -simpleType 'Trigger' | ToArray diff --git a/test/zGet-AdfFromService.Tests.ps1 b/test/zGet-AdfFromService.Tests.ps1 index 82f7b8a..1b0ce07 100644 --- a/test/zGet-AdfFromService.Tests.ps1 +++ b/test/zGet-AdfFromService.Tests.ps1 @@ -10,21 +10,310 @@ InModuleScope azure.datafactory.tools { $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' Import-Module -Name $testHelperPath -Force - # Variables for use in tests - $t = Get-TargetEnv 'adf2' - $script:adfName = $t.DataFactoryName - $script:rg = $t.ResourceGroupName - + # Shared factory stub + $script:fakeAdfi = [PSCustomObject]@{ + DataFactoryId = '/subscriptions/sub-123/resourceGroups/rg-test/providers/Microsoft.DataFactory/factories/adf-test' + Location = 'northeurope' + } + + # --------------------------------------------------------------------------- Describe 'Get-AdfFromService' -Tag 'Unit' { + It 'Should exist' { { Get-Command -Name Get-AdfFromService -ErrorAction Stop } | Should -Not -Throw } - } + Context 'When all Az cmdlets succeed (happy path)' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + Mock Get-AzDataFactoryV2Dataset { @() } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { @() } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { @() } + Mock Get-AzDFV2Credential { @() } + } + + It 'Should return an AdfInstance with the correct factory name' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.Name | Should -Be 'adf-test' + } + + It 'Should not call Invoke-AzRestMethod when Az cmdlets succeed' { + Mock Invoke-AzRestMethod { throw 'Should not be called' } + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + } + + Context 'When Get-AzDataFactoryV2Dataset throws a deserialization error (issue #480)' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + Mock Get-AzDataFactoryV2Dataset { throw 'Unable to deserialize the response.' } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { @() } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { @() } + Mock Get-AzDFV2Credential { @() } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ds_adls_csv","properties":{}},{"name":"ds_servicenow_v2","properties":{}}]}' + } + } + } + + It 'Should not throw' { + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + + It 'Should fall back to REST API and return all datasets by name' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.DataSets.Count | Should -Be 2 + $result.DataSets.Name | Should -Contain 'ds_adls_csv' + $result.DataSets.Name | Should -Contain 'ds_servicenow_v2' + } + + It 'Should return datasets as AdfPSDataset wrapper objects' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.DataSets | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSDataset' } + } + + It 'Should call Invoke-AzRestMethod with the correct datasets URL' { + Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly -ParameterFilter { + $Uri -like '*/datasets?api-version=2018-06-01' + } + } + } + + Context 'When Get-AzDataFactoryV2LinkedService throws a deserialization error' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + Mock Get-AzDataFactoryV2Dataset { @() } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { throw 'Unable to deserialize the response.' } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { @() } + Mock Get-AzDFV2Credential { @() } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ls_adls","properties":{}},{"name":"ls_servicenow","properties":{}}]}' + } + } + } + + It 'Should not throw' { + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + + It 'Should return linked services via REST fallback as AdfPSLinkedService objects' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.LinkedServices.Count | Should -Be 2 + $result.LinkedServices | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSLinkedService' } + } + } + + Context 'When Get-AzDataFactoryV2Trigger throws a deserialization error' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + Mock Get-AzDataFactoryV2Dataset { @() } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { @() } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { throw 'Unable to deserialize the response.' } + Mock Get-AzDFV2Credential { @() } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"tr_daily","properties":{"runtimeState":"Started"}},{"name":"tr_hourly","properties":{"runtimeState":"Stopped"}}]}' + } + } + } + + It 'Should not throw' { + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + + It 'Should return triggers as AdfPSTrigger objects with correct RuntimeState' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.Triggers.Count | Should -Be 2 + $result.Triggers | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSTrigger' } + ($result.Triggers | Where-Object Name -eq 'tr_daily').RuntimeState | Should -Be 'Started' + ($result.Triggers | Where-Object Name -eq 'tr_hourly').RuntimeState | Should -Be 'Stopped' + } + } + } + + # --------------------------------------------------------------------------- + Describe 'Get-AdfObjectsFromServiceRestAPI' -Tag 'Unit' { + + It 'Should exist' { + { Get-Command -Name Get-AdfObjectsFromServiceRestAPI -ErrorAction Stop } | Should -Not -Throw + } + + Context 'When the API returns an empty list' { + BeforeEach { + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ StatusCode = 200; Content = '{"value":[]}' } + } + } + + It 'Should return an empty collection' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + @($result).Count | Should -Be 0 + } + + It 'Should call Invoke-AzRestMethod exactly once' { + Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly + } + } + + Context 'When the API returns a single page of datasets' { + BeforeEach { + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ds_one","properties":{}},{"name":"ds_two","properties":{}},{"name":"ds_three","properties":{}}]}' + } + } + } + + It 'Should return all items from the page' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $result.Count | Should -Be 3 + } + + It 'Should return AdfPSDataset wrapper objects' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $result | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSDataset' } + } + + It 'Should preserve the dataset names' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $result.Name | Should -Contain 'ds_one' + $result.Name | Should -Contain 'ds_two' + $result.Name | Should -Contain 'ds_three' + } + + It 'Should call Invoke-AzRestMethod exactly once (no extra pages)' { + Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly + } + } + + Context 'When the API returns two pages (pagination)' { + BeforeEach { + $script:page2Url = 'https://management.azure.com/subscriptions/sub-123/.../datasets?api-version=2018-06-01&skipToken=page2' + $script:callCount = 0 + Mock Invoke-AzRestMethod { + $script:callCount++ + if ($script:callCount -eq 1) { + return [PSCustomObject]@{ + StatusCode = 200 + Content = "{`"value`":[{`"name`":`"ds_p1_a`",`"properties`":{}},{`"name`":`"ds_p1_b`",`"properties`":{}}],`"nextLink`":`"$($script:page2Url)`"}" + } + } else { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ds_p2_a","properties":{}}]}' + } + } + } + } + + It 'Should return items from all pages combined' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $result.Count | Should -Be 3 + $result.Name | Should -Contain 'ds_p1_a' + $result.Name | Should -Contain 'ds_p1_b' + $result.Name | Should -Contain 'ds_p2_a' + } + + It 'Should call Invoke-AzRestMethod twice (once per page)' { + Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + Assert-MockCalled Invoke-AzRestMethod -Times 2 -Exactly + } + + It 'Should use the nextLink URL for the second request' { + Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly -ParameterFilter { + $Uri -eq $script:page2Url + } + } + } + + Context 'When called for each supported object type' { + BeforeEach { + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"obj1","properties":{"runtimeState":"Stopped"}}]}' + } + } + } + + It 'Should return AdfPSDataset for simpleType Dataset' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $r[0].GetType().Name | Should -Be 'AdfPSDataset' + } + + It 'Should return AdfPSPipeline for simpleType Pipeline' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'pipelines' -simpleType 'Pipeline' + $r[0].GetType().Name | Should -Be 'AdfPSPipeline' + } + + It 'Should return AdfPSLinkedService for simpleType LinkedService' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'linkedservices' -simpleType 'LinkedService' + $r[0].GetType().Name | Should -Be 'AdfPSLinkedService' + } + + It 'Should return AdfPSIntegrationRuntime for simpleType IntegrationRuntime' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'integrationruntimes' -simpleType 'IntegrationRuntime' + $r[0].GetType().Name | Should -Be 'AdfPSIntegrationRuntime' + } + + It 'Should return AdfPSDataFlow for simpleType DataFlow' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'dataflows' -simpleType 'DataFlow' + $r[0].GetType().Name | Should -Be 'AdfPSDataFlow' + } + + It 'Should return AdfPSTrigger for simpleType Trigger with RuntimeState' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'triggers' -simpleType 'Trigger' + $r[0].GetType().Name | Should -Be 'AdfPSTrigger' + $r[0].RuntimeState | Should -Be 'Stopped' + } + } + + Context 'When the API returns a non-200 status code' { + BeforeEach { + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ StatusCode = 403; Content = '{"error":{"code":"AuthorizationFailed"}}' } + } + } + + It 'Should throw when ErrorAction Stop is used' { + { Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' -ErrorAction Stop } | Should -Throw + } + } + } + + # --------------------------------------------------------------------------- Describe 'Get-AdfFromService' -Tag 'Integration' { + # Variables for use in integration tests + $t = Get-TargetEnv 'adf2' + $script:adfName = $t.DataFactoryName + $script:rg = $t.ResourceGroupName + It 'Should execute' { Get-AdfFromService -FactoryName $adfName -ResourceGroupName $rg } - } + } } + From 3e6cf731852b3406e51ebd95ee17260e1567332b Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Tue, 30 Jun 2026 23:47:40 +0100 Subject: [PATCH 08/20] Add tests for handling non-terminating errors in Get-AdfFromService --- test/zGet-AdfFromService.Tests.ps1 | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/zGet-AdfFromService.Tests.ps1 b/test/zGet-AdfFromService.Tests.ps1 index 1b0ce07..a919338 100644 --- a/test/zGet-AdfFromService.Tests.ps1 +++ b/test/zGet-AdfFromService.Tests.ps1 @@ -88,6 +88,49 @@ InModuleScope azure.datafactory.tools { } } + # This context specifically tests the real-world failure mode from issue #480: + # The Az.DataFactory SDK writes a *non-terminating* Write-Error (not a throw) when + # it cannot deserialize an unsupported object type such as ServiceNow V2. + # Without -ErrorAction Stop on the cmdlet call the try-catch never fires; with it, + # the non-terminating error is promoted to terminating and the catch fires correctly. + Context 'When Get-AzDataFactoryV2Dataset emits a non-terminating deserialization error (real Az.DataFactory behaviour)' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + # Simulate the actual Az.DataFactory SDK behaviour: Write-Error (non-terminating), + # NOT throw. The cmdlet call in Get-AdfFromService uses -ErrorAction Stop which + # must promote this to a terminating error so the catch block fires. + Mock Get-AzDataFactoryV2Dataset { Write-Error 'Unable to deserialize the response.' } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { @() } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { @() } + Mock Get-AzDFV2Credential { @() } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ds_adls_csv","properties":{}},{"name":"ds_servicenow_v2","properties":{}}]}' + } + } + } + + It 'Should not throw even when the cmdlet only writes a non-terminating error' { + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + + It 'Should activate the REST API fallback and return all datasets' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.DataSets.Count | Should -Be 2 + $result.DataSets.Name | Should -Contain 'ds_adls_csv' + $result.DataSets.Name | Should -Contain 'ds_servicenow_v2' + } + + It 'Should return AdfPSDataset wrapper objects from the REST fallback' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.DataSets | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSDataset' } + } + } + Context 'When Get-AzDataFactoryV2LinkedService throws a deserialization error' { BeforeEach { Mock Get-AzDataFactoryV2 { $script:fakeAdfi } From ba7dd153b6a3a4c3e8f93e98c6e9ea5310869aee Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Tue, 30 Jun 2026 23:56:17 +0100 Subject: [PATCH 09/20] changelog update - 1.17.0 --- changelog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changelog.md b/changelog.md index 6b73285..db539bc 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.17.0] - 2026-07-01 +### Fixed +* Further fixes of `DeleteNotInSource` with deserialization issue #480 + ## [1.16.0] - 2026-05-27 ### Fixed * `DeleteNotInSource` no longer fails with "Unable to deserialize the response" when ADF contains object types unsupported by the installed Az.DataFactory module version - REST API fallback added to `Get-AdfFromService` #480 #473 From 516ae683faa4e5d866d832e137164c3356b3524f Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 1 Jul 2026 00:03:08 +0100 Subject: [PATCH 10/20] Fixed simple error in unit test --- test/Incremental-Deployment.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Incremental-Deployment.Tests.ps1 b/test/Incremental-Deployment.Tests.ps1 index 5382161..bb07c0a 100644 --- a/test/Incremental-Deployment.Tests.ps1 +++ b/test/Incremental-Deployment.Tests.ps1 @@ -29,7 +29,7 @@ InModuleScope azure.datafactory.tools { $opt.IncrementalDeployment = $true $opt.StopStartTriggers = $false $script:gp = "" - $script:dstate = [AdfDeploymentState]::new($verStr) + $script:dstate = [AdfDeploymentState]::new($script:verStr) $script:dstate.LastUpdate = [System.DateTime]::UtcNow $script:dstateJson = $script:dstate | ConvertTo-Json $script:StorageUri= "https://sqlplayer2020.blob.core.windows.net" From cd2bd4e64587382c3e33a8654f78178146bd84be Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 1 Jul 2026 00:20:12 +0100 Subject: [PATCH 11/20] Fix unit test issue c.d. --- test/!RunAllTests.ps1 | 5 ++++- test/Incremental-Deployment.Tests.ps1 | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/!RunAllTests.ps1 b/test/!RunAllTests.ps1 index ba01f7c..1c4922d 100644 --- a/test/!RunAllTests.ps1 +++ b/test/!RunAllTests.ps1 @@ -51,6 +51,7 @@ if ($InstallModules) { #Install-Module 'Az.DataFactory' -Force -MinimumVersion 1.16.0 -Repository 'PSGallery' #Install-Module 'PSScriptAnalyzer' -Force #Install-Module 'Pester' -Force -MinimumVersion 5.1.1 + Install-Module 'Az.Storage' -Force -AllowClobber -Repository 'PSGallery' [Environment]::SetEnvironmentVariable("azure.datafactory.tools.unitTestInstalledModules", $true, 'Process'); } } else { @@ -58,6 +59,7 @@ if ($InstallModules) { } Import-Module 'Pester' -MinimumVersion 5.3.3 Import-Module 'PSScriptAnalyzer' +Import-Module 'Az.Storage' Import-Module "$folder\azure.datafactory.tools.psd1" Write-Host "=============== Modules ================" @@ -96,4 +98,5 @@ try { Pop-Location } -# . ".\test\!RunAllTests.ps1" -folder '.\test' -TestFilenameFilter '*' -MajorRelease:$true -InstallModules:$false +# Connect-AzAccount +# . ".\test\!RunAllTests.ps1" -folder '.\test' -TestFilenameFilter 'Inc*' -MajorRelease:$true -InstallModules:$false diff --git a/test/Incremental-Deployment.Tests.ps1 b/test/Incremental-Deployment.Tests.ps1 index bb07c0a..d5824c2 100644 --- a/test/Incremental-Deployment.Tests.ps1 +++ b/test/Incremental-Deployment.Tests.ps1 @@ -12,6 +12,9 @@ InModuleScope azure.datafactory.tools { $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' Import-Module -Name $testHelperPath -Force + $m = Get-Module -Name 'azure.datafactory.tools' + $script:verStr = $m.Version.ToString(2) + "." + $m.Version.Build.ToString("000") + # $VerbosePreference = 'Continue' # $DebugPreference = 'Continue' From 26f26591437e328a3c00601d170dd8029ec09fb6 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 1 Jul 2026 00:52:31 +0100 Subject: [PATCH 12/20] ver.1.17.0 --- azure.datafactory.tools.psd1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index f99e0db..4fd4a53 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -12,7 +12,7 @@ RootModule = 'azure.datafactory.tools.psm1' # Version number of this module. - ModuleVersion = '1.16.0' + ModuleVersion = '1.17.0' # Supported PSEditions # CompatiblePSEditions = @() From bca0ad1cd2302535e9504a2397ad4e9fc96afe69 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 1 Jul 2026 01:15:57 +0100 Subject: [PATCH 13/20] Added unit test for #475 --- changelog.md | 2 +- test/Publish-AdfV2FromJson-DryRun.Tests.ps1 | 54 +++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 test/Publish-AdfV2FromJson-DryRun.Tests.ps1 diff --git a/changelog.md b/changelog.md index be8ebf9..a4cb5b7 100644 --- a/changelog.md +++ b/changelog.md @@ -20,7 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed * Starting blob event trigger fails with 'Resource cannot be updated during provisioning' - now waits for provisioning to complete before retrying #484, #474, #463 * Fixed REST API calls failing with 401 Unauthorized due to Az.Accounts 5.x returning SecureString from Get-AzAccessToken -* Fixed DryRun not loading deployment state from storage for hash comparison #476 +* Fixed DryRun not loading deployment state from storage for hash comparison #475 * README.md updated and new Structured Documentation created ## [1.14.0] - 2025-10-24 diff --git a/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 b/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 new file mode 100644 index 0000000..1da24b1 --- /dev/null +++ b/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 @@ -0,0 +1,54 @@ +BeforeDiscovery { + $ModuleRootPath = $PSScriptRoot | Split-Path -Parent + $moduleManifestName = 'azure.datafactory.tools.psd1' + $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName + + Import-Module -Name $moduleManifestPath -Force -Verbose:$false + $m = Get-Module -Name 'azure.datafactory.tools' + $script:verStr = $m.Version.ToString(2) + "." + $m.Version.Build.ToString("000") +} + +InModuleScope azure.datafactory.tools { + $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' + Import-Module -Name $testHelperPath -Force + + $m = Get-Module -Name 'azure.datafactory.tools' + $script:verStr = $m.Version.ToString(2) + "." + $m.Version.Build.ToString("000") + + $script:RootFolder = Join-Path $PSScriptRoot 'adf2' + $script:ResourceGroupName = 'rg-test' + $script:DataFactoryName = 'adf-test' + $script:Location = 'West Europe' + + Describe 'Publish-AdfV2FromJson DryRun incremental deployment' -Tag 'Unit' { + BeforeEach { + Mock Get-StateFromStorage { + [AdfDeploymentState]::new($script:verStr) + } + + Mock Set-StateToStorage { + } + } + + It 'should still load deployment state from storage when DryRun is enabled' { + $opt = New-AdfPublishOption + $opt.IncrementalDeployment = $true + $opt.IncrementalDeploymentStorageUri = 'https://example.blob.core.windows.net/adftools/folder' + $opt.StopStartTriggers = $false + $opt.DeleteNotInSource = $false + + { + Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName $script:ResourceGroupName ` + -DataFactoryName $script:DataFactoryName ` + -Location $script:Location ` + -Option $opt ` + -DryRun + } | Should -Not -Throw + + Should -Invoke -CommandName Get-StateFromStorage -Times 1 -Exactly + Should -Invoke -CommandName Set-StateToStorage -Times 0 -Exactly + } + } +} \ No newline at end of file From 4a43a49ab210aa39592946e7121d1a2c5b833c43 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 1 Jul 2026 01:23:40 +0100 Subject: [PATCH 14/20] ci-cd is active --- .github/workflows/ci-cd.yml | 11 ++++++++++- .github/workflows/develop-ci.yml | 24 ++++++++++++------------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index bdcabaf..7167fd2 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -2,8 +2,17 @@ name: CI/CD on: push: - branches: ['**'] + branches: [ develop ] + paths: + - 'private/**' + - 'public/**' + - 'test/**' pull_request: + branches: [ develop ] + paths: + - 'private/**' + - 'public/**' + - 'test/**' workflow_dispatch: permissions: diff --git a/.github/workflows/develop-ci.yml b/.github/workflows/develop-ci.yml index 9ada017..2274f18 100644 --- a/.github/workflows/develop-ci.yml +++ b/.github/workflows/develop-ci.yml @@ -1,18 +1,18 @@ name: CI on: - push: - branches: [ develop ] - paths: - - 'private/**' - - 'public/**' - - 'test/**' - pull_request: - branches: [ develop ] - paths: - - 'private/**' - - 'public/**' - - 'test/**' + # push: + # branches: [ develop ] + # paths: + # - 'private/**' + # - 'public/**' + # - 'test/**' + # pull_request: + # branches: [ develop ] + # paths: + # - 'private/**' + # - 'public/**' + # - 'test/**' workflow_dispatch: permissions: From 66182d34dec543890fef508318534db5275f1c69 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 1 Jul 2026 02:23:38 +0100 Subject: [PATCH 15/20] Bump module version to 1.17.1 and update changelog for timezone handling improvements #402 --- azure.datafactory.tools.psd1 | 2 +- changelog.md | 3 ++- private/ObjectProperty.ps1 | 11 +++++++--- test/Update-PropertiesFromFile.Tests.ps1 | 27 ++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index 4fd4a53..af6b098 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -12,7 +12,7 @@ RootModule = 'azure.datafactory.tools.psm1' # Version number of this module. - ModuleVersion = '1.17.0' + ModuleVersion = '1.17.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/changelog.md b/changelog.md index a4cb5b7..14dce27 100644 --- a/changelog.md +++ b/changelog.md @@ -4,10 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [1.17.0] - 2026-07-01 +## [1.17.1] - 2026-07-01 ### Fixed * Further fixes of `DeleteNotInSource` with deserialization issue #480 * `Publish-AdfV2FromJson` no longer silently continues after errors: config path errors (ADFT0010) and deployment failures (e.g. Azure Policy blocks) now propagate as terminating errors to the caller #472 +* Scheduled trigger recurrence times now preserve explicit timezone offsets during config-driven updates, preventing shifted deployment times #402 ## [1.16.0] - 2026-06-06 ### Fixed diff --git a/private/ObjectProperty.ps1 b/private/ObjectProperty.ps1 index 2c7996f..010b95d 100644 --- a/private/ObjectProperty.ps1 +++ b/private/ObjectProperty.ps1 @@ -25,9 +25,14 @@ function Update-ObjectProperty { $exp = "`$obj.$path = `$$value" } elseif ($fieldType -eq [DateTime]) { Write-Debug "Setting as DateTime value" - $datevalue = [DateTime]$value - $utcvalue = Get-Date $datevalue -Format "yyyy-MM-ddTHH:mm:ss.fffZ" - $exp = "`$obj.$path = `"$utcvalue`"" + # Keep explicit timezone suffixes unchanged to avoid timezone shifts (Issue #402). + if ($value -match '(?i)(Z|[+-][0-9]{2}:[0-9]{2})$') { + $exp = "`$obj.$path = `"$value`"" + } else { + $date_value = [DateTime]$value + $utc_value = Get-Date $date_value -Format "yyyy-MM-ddTHH:mm:ss.fffZ" + $exp = "`$obj.$path = `"$utc_value`"" + } } elseif ($fieldType -eq [System.Management.Automation.PSCustomObject]) { Write-Debug "Setting as json value" $jvalue = ConvertFrom-Json $value diff --git a/test/Update-PropertiesFromFile.Tests.ps1 b/test/Update-PropertiesFromFile.Tests.ps1 index 7701999..8ebea0c 100644 --- a/test/Update-PropertiesFromFile.Tests.ps1 +++ b/test/Update-PropertiesFromFile.Tests.ps1 @@ -375,6 +375,33 @@ InModuleScope azure.datafactory.tools { } + Describe 'Update-PropertiesFromFile - Issue #402' -Tag 'Unit','private','issue402' { + Context 'When trigger recurrence timestamps contain explicit timezone offsets' { + BeforeAll { + $script:adf = Import-AdfFromFolder -FactoryName "xyz" -RootFolder "$RootFolder" + $script:adf.PublishOptions = New-AdfPublishOption + + $script:configFilePath = Join-Path -Path $script:DeploymentFolder -ChildPath "config-issue-402-timezone.csv" + @( + 'type,name,path,value' + 'trigger,TR_RunEveryDay,"$.properties.typeProperties.recurrence.startTime","2020-06-01T23:22:11.000+10:00"' + 'trigger,TR_RunEveryDay,"$.properties.typeProperties.recurrence.endTime","2020-06-05T23:22:11.000+10:00"' + ) | Set-Content -Path $script:configFilePath -Encoding UTF8 + + Update-PropertiesFromFile -adf $script:adf -stage $script:configFilePath + $script:trigger = Get-AdfObjectByName -adf $script:adf -name "TR_RunEveryDay" -type "trigger" + } + + It 'Should keep recurrence.startTime offset unchanged' { + $script:trigger.Body.properties.typeProperties.recurrence.startTime | Should -Be "2020-06-01T23:22:11.000+10:00" + } + + It 'Should keep recurrence.endTime offset unchanged' { + $script:trigger.Body.properties.typeProperties.recurrence.endTime | Should -Be "2020-06-05T23:22:11.000+10:00" + } + } + } + Describe 'Publish-AdfV2FromJson with DryRun' -Tag 'Unit','private' { $cases = @( @{ configFile = 'config-endpoint.csv' }, @{ configFile = 'config-endpoint2.json' } ) From f5477a0170d6ee718c0abbd7c60ee2f2acb3e6da Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 1 Jul 2026 02:29:19 +0100 Subject: [PATCH 16/20] Update documentation: IgnoreLackOfReferencedObject option #198 --- README.md | 7 ++++++- docs/GUIDE/PUBLISH_OPTIONS.md | 17 +++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1f34204..1d9610e 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,7 @@ $opt = New-AdfPublishOption * [Boolean] **DeployGlobalParams** - indicates whether deploy Global Parameters of ADF. Nothing happens when parameters are not defined. (default: *true*) * [Boolean] **FailsWhenConfigItemNotFound** - indicates whether configuration items not found fails the script. (default: *true*) * [Boolean] **FailsWhenPathNotFound** - indicates whether missing paths fails the script. (default: *true*) +* [Boolean] **IgnoreLackOfReferencedObject** - indicates whether missing referenced objects should only raise warnings. (default: *false*) * [Boolean] **DoNotStopStartExcludedTriggers** - specifies whether excluded triggers will be stopped before deployment (default: *false*) * [Boolean] **DoNotDeleteExcludedObjects** - specifies whether excluded objects can be removed. Applies when `DeleteNotInSource` is set to *True* only. (default: *true*) * [Boolean] **IncrementalDeployment** - specifies whether Incremental Deployment mode is enabled (default: *false*) @@ -265,7 +266,11 @@ $opt.FailsWhenConfigItemNotFound = $false $opt = New-AdfPublishOption $opt.FailsWhenPathNotFound = $false -# Example 7: Exclude Infrastructure-type of objects from deployment +# Example 7: Ignore missing referenced objects (will just write warning to standard output instead) +$opt = New-AdfPublishOption +$opt.IgnoreLackOfReferencedObject = $true + +# Example 8: Exclude Infrastructure-type of objects from deployment $opt = New-AdfPublishOption $opt.CreateNewInstance = $false $opt.Excludes.Add("integrationruntime.*", "") diff --git a/docs/GUIDE/PUBLISH_OPTIONS.md b/docs/GUIDE/PUBLISH_OPTIONS.md index c9b474b..412b14b 100644 --- a/docs/GUIDE/PUBLISH_OPTIONS.md +++ b/docs/GUIDE/PUBLISH_OPTIONS.md @@ -67,6 +67,11 @@ $opt = New-AdfPublishOption - **Default:** true - **Purpose:** Fail deployment if config path doesn't exist (vs. warning only) +#### IgnoreLackOfReferencedObject +- **Type:** Boolean +- **Default:** false +- **Purpose:** Continue deployment when a referenced object is missing and emit a warning instead of failing + ### Trigger Control Options #### StopStartTriggers (see also: TriggerStopMethod, TriggerStartMethod) @@ -155,7 +160,15 @@ $opt.FailsWhenConfigItemNotFound = $false # Warnings will be printed instead of failing ``` -### Example 6: Incremental Deployment +### Example 6: Ignore Missing Referenced Objects + +```powershell +$opt = New-AdfPublishOption +$opt.IgnoreLackOfReferencedObject = $true +# Referenced objects that are not present will warn instead of failing +``` + +### Example 7: Incremental Deployment ```powershell $opt = New-AdfPublishOption @@ -163,7 +176,7 @@ $opt.IncrementalDeployment = $true $opt.IncrementalDeploymentStorageUri = 'https://storageaccount.file.core.windows.net/adftools' ``` -### Example 7: Selective Deployment with Smart Triggers +### Example 8: Selective Deployment with Smart Triggers ```powershell $opt = New-AdfPublishOption From fd7870028763880a59f06ac428581b3037d3927c Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 1 Jul 2026 02:56:03 +0100 Subject: [PATCH 17/20] DryRun shows deployment plan in terraform-like format --- README.md | 1 + docs/ADVANCED/CMDLET_REFERENCE.md | 4 +- docs/GUIDE/PUBLISHING.md | 5 +- private/ApplyExclusionOptions.ps1 | 166 +++++++++++++++++++- public/Publish-AdfV2FromJson.ps1 | 38 ++++- test/Publish-AdfV2FromJson-DryRun.Tests.ps1 | 63 ++++++++ 6 files changed, 272 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1d9610e..e601f2b 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,7 @@ Publish-AdfV2FromJson [-Option] [-Method] [-DryRun] + [-Plan] ``` Assuming your ADF is named ```SQLPlayerDemo``` and the code is located in ```c:\GitHub\AdfName\```, replace the values for *SubscriptionName*, *ResourceGroupName*, *DataFactoryName* and run the following command using PowerShell CLI: diff --git a/docs/ADVANCED/CMDLET_REFERENCE.md b/docs/ADVANCED/CMDLET_REFERENCE.md index 431ec2a..e334681 100644 --- a/docs/ADVANCED/CMDLET_REFERENCE.md +++ b/docs/ADVANCED/CMDLET_REFERENCE.md @@ -18,7 +18,8 @@ Publish-AdfV2FromJson ` [-Stage ] ` [-Option ] ` [-Method ] ` - [-DryRun] + [-DryRun] ` + [-Plan] ``` **Parameters:** @@ -33,6 +34,7 @@ Publish-AdfV2FromJson ` | Option | AdfPublishOption | No | Publish options object (filtering, triggers, etc.) | | Method | String | No | Publishing method: 'AzDataFactory' or 'AzResource' (default) | | DryRun | Switch | No | Validate without deploying | +| Plan | Switch | No | Alias behavior of DryRun with terraform-like naming | **Returns:** AdfPublishOption (deployment result) diff --git a/docs/GUIDE/PUBLISHING.md b/docs/GUIDE/PUBLISHING.md index d3ef583..68a4670 100644 --- a/docs/GUIDE/PUBLISHING.md +++ b/docs/GUIDE/PUBLISHING.md @@ -33,9 +33,12 @@ Publish-AdfV2FromJson ` [-Stage] ` [-Option] ` [-Method] ` - [-DryRun] + [-DryRun] ` + [-Plan] ``` +`-Plan` is an alias behavior of `-DryRun` and prints a terraform-like plan without deploying. + ### Complete Example ```powershell diff --git a/private/ApplyExclusionOptions.ps1 b/private/ApplyExclusionOptions.ps1 index 4042a3c..eda5618 100644 --- a/private/ApplyExclusionOptions.ps1 +++ b/private/ApplyExclusionOptions.ps1 @@ -49,7 +49,9 @@ function ApplyExclusionOptions { function ToBeDeployedStat { param( - [Parameter(Mandatory=$True)] [Adf] $adf + [Parameter(Mandatory=$True)] [Adf] $adf, + [Parameter(Mandatory=$False)] $targetAdfInstance, + [switch] $TerraformStyle ) $ToBeDeployedList = ($adf.AllObjects() | Where-Object { $_.ToBeDeployed -eq $true } | ToArray) @@ -59,4 +61,166 @@ function ToBeDeployedStat { Write-Host "- $($_.FullName($true))" } + if (!$TerraformStyle) { + return $null + } + + $plan = Get-DryRunPlan -adf $adf -targetAdfInstance $targetAdfInstance + Write-DryRunPlan -plan $plan + return $plan + +} + +function ConvertTo-CanonicalAdfType { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] + [String] $Type + ) + + $t = $Type.ToLowerInvariant() + switch ($t) + { + 'pipeline' { return 'pipeline' } + 'dataset' { return 'dataset' } + 'dataflow' { return 'dataflow' } + 'linkedservice' { return 'linkedService' } + 'integrationruntime' { return 'integrationRuntime' } + 'trigger' { return 'trigger' } + 'factory' { return 'factory' } + 'managedvirtualnetwork' { return 'managedVirtualNetwork' } + 'managedprivateendpoint' { return 'managedPrivateEndpoint' } + 'credential' { return 'credential' } + default { return $Type } + } +} + +function Get-AdfObjectFullName { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] + $Object + ) + + $name = $Object.Name + $simtype = '' + + if ($Object -is [AdfObject]) { + $simtype = Get-SimplifiedType -Type $Object.Type + if ($Object.Type -like 'Microsoft.DataFactory/factories*') { + $simtype = ConvertTo-AdfType -AzType $Object.Type + $simtype = Get-SimplifiedType -Type $simtype + } + } + else { + $simtype = Get-SimplifiedType -Type $Object.GetType().Name + } + + $simtype = ConvertTo-CanonicalAdfType -Type $simtype + return "$simtype.$name" +} + +function Get-DryRunPlan { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] [Adf] $adf, + [parameter(Mandatory = $false)] $targetAdfInstance + ) + + $create = @() + $update = @() + $delete = @() + $unchanged = @() + + $sourceByName = @{} + $adf.AllObjects() | ForEach-Object { + $fullName = Get-AdfObjectFullName -Object $_ + $sourceByName[$fullName] = $_ + } + + $targetByName = @{} + if ($null -ne $targetAdfInstance) { + $targetAdfInstance.AllObjects() | ForEach-Object { + $fullName = Get-AdfObjectFullName -Object $_ + $targetByName[$fullName] = $_ + } + } + + $adf.AllObjects() | ForEach-Object { + $fullName = Get-AdfObjectFullName -Object $_ + if ($_.ToBeDeployed -eq $false) { + $unchanged += $fullName + } + elseif ($targetByName.ContainsKey($fullName)) { + $update += $fullName + } + else { + $create += $fullName + } + } + + if ($null -ne $targetAdfInstance -and $adf.PublishOptions.DeleteNotInSource) { + $targetByName.Keys | ForEach-Object { + $fullName = $_ + if (!$sourceByName.ContainsKey($fullName)) { + $oname = [AdfObjectName]::new($fullName) + $canDelete = $true + if ($adf.PublishOptions.DoNotDeleteExcludedObjects) { + $canDelete = !($oname.IsNameExcluded($adf.PublishOptions)) + } + if ($canDelete) { + $delete += $fullName + } + } + } + } + + $create = @($create | Sort-Object) + $update = @($update | Sort-Object) + $delete = @($delete | Sort-Object) + $unchanged = @($unchanged | Sort-Object) + + return [PSCustomObject]@{ + Create = @($create) + Update = @($update) + Delete = @($delete) + Unchanged = @($unchanged) + } +} + +function Write-DryRunPlan { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] + $plan + ) + + Write-Host "===================================================================================" + Write-Host "Terraform-like plan (DryRun):" + Write-Host "Plan: $($plan.Create.Count) to add, $($plan.Update.Count) to change, $($plan.Delete.Count) to destroy." + + if ($plan.Create.Count -gt 0) { + Write-Host "" + Write-Host " + Create" + $plan.Create | ForEach-Object { Write-Host " + $_" } + } + + if ($plan.Update.Count -gt 0) { + Write-Host "" + Write-Host " ~ Update" + $plan.Update | ForEach-Object { Write-Host " ~ $_" } + } + + if ($plan.Delete.Count -gt 0) { + Write-Host "" + Write-Host " - Delete" + $plan.Delete | ForEach-Object { Write-Host " - $_" } + } + + if ($plan.Unchanged.Count -gt 0) { + Write-Host "" + Write-Host " = Unchanged / skipped" + $plan.Unchanged | ForEach-Object { Write-Host " = $_" } + } + } \ No newline at end of file diff --git a/public/Publish-AdfV2FromJson.ps1 b/public/Publish-AdfV2FromJson.ps1 index 3d687e7..410ac85 100644 --- a/public/Publish-AdfV2FromJson.ps1 +++ b/public/Publish-AdfV2FromJson.ps1 @@ -35,6 +35,9 @@ AzResource method has been introduced due to bugs in Az.DataFactory PS module. Optional switch parameter. When provided, process will not make any changes to target data factory but instead return the ADF object that would be used in deployment. +.PARAMETER Plan +Optional switch parameter. Alias for DryRun with terraform-style intent. Behaves the same as DryRun. + .EXAMPLE # Publish entire ADF $ResourceGroupName = 'rg-devops-factory' @@ -74,6 +77,10 @@ Publish-AdfV2FromJson -RootFolder "$RootFolder" -ResourceGroupName "$ResourceGro # Execute dry run of intended publishing changes Publish-AdfV2FromJson -RootFolder "$RootFolder" -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$DataFactoryName" -Location "$Location" -Stage "UAT" -DryRun +.EXAMPLE +# Execute plan (alias of DryRun) to preview intended changes +Publish-AdfV2FromJson -RootFolder "$RootFolder" -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$DataFactoryName" -Location "$Location" -Stage "UAT" -Plan + .LINK Online version: https://github.com/SQLPlayer/azure.datafactory.tools/ #> @@ -104,9 +111,16 @@ function Publish-AdfV2FromJson { [String]$Method = 'AzResource', [parameter(Mandatory = $false)] - [switch]$DryRun + [switch]$DryRun, + + [parameter(Mandatory = $false)] + [switch]$Plan ) + if ($Plan.IsPresent) { + $DryRun = $true + } + $m = Get-Module -Name "azure.datafactory.tools" $verStr = $m.Version.ToString(2) + "." + $m.Version.Build.ToString("000"); Write-Host "======================================================================================"; @@ -123,6 +137,7 @@ function Publish-AdfV2FromJson { Write-Host "Options provided: $($null -ne $Option)"; Write-Host "Publishing method: $Method"; Write-Host "Is Dry Run?: $($DryRun.IsPresent)"; + Write-Host "Is Plan Mode?: $($Plan.IsPresent)"; Write-Host "======================================================================================"; if ($null -ne $Option) { Write-Host "Options:" @@ -248,10 +263,29 @@ function Publish-AdfV2FromJson { } Write-Host "Found $unchanged_count unchanged object(s)." } - ToBeDeployedStat -adf $adf + + $dryRunPlan = $null + if ($DryRun.IsPresent) { + $targetAdfInstance = $null + try { + Write-Host "DRY RUN: Loading target ADF objects to classify plan changes..." + $targetAdfInstance = Get-AdfFromService -FactoryName "$DataFactoryName" -ResourceGroupName "$ResourceGroupName" + } + catch { + Write-Warning "DRY RUN: Unable to load target ADF state for full plan classification. Falling back to local-only plan. Details: $_" + } + + $dryRunPlan = ToBeDeployedStat -adf $adf -targetAdfInstance $targetAdfInstance -TerraformStyle + } + else { + ToBeDeployedStat -adf $adf + } if ($DryRun.IsPresent) { Write-Host "DRY RUN: Terminating script pre-deployment - inspect returned object to review planned changes." + if ($null -ne $dryRunPlan) { + $adf | Add-Member -MemberType NoteProperty -Name DryRunPlan -Value $dryRunPlan -Force + } Write-Host "==================================================================================="; return $adf } diff --git a/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 b/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 index 1da24b1..70514ae 100644 --- a/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 +++ b/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 @@ -28,6 +28,10 @@ InModuleScope azure.datafactory.tools { Mock Set-StateToStorage { } + + Mock Get-AdfFromService { + $null + } } It 'should still load deployment state from storage when DryRun is enabled' { @@ -51,4 +55,63 @@ InModuleScope azure.datafactory.tools { Should -Invoke -CommandName Set-StateToStorage -Times 0 -Exactly } } + + Describe 'DryRun terraform-like plan output' -Tag 'Unit' { + It 'should classify add, change, destroy and expose DryRunPlan on returned object' { + $opt = New-AdfPublishOption + $opt.StopStartTriggers = $false + $opt.DeleteNotInSource = $true + $opt.DoNotDeleteExcludedObjects = $false + $opt.Includes.Add('pipeline.PL_ExecSparkJob', '') + + Mock Get-AdfFromService { + $target = New-Object -TypeName AdfInstance + $target.Pipelines = @( + [AdfPSPipeline]::new('PL_ExecSparkJob'), + [AdfPSPipeline]::new('PL_Obsolete') + ) + $target + } + + $result = Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName $script:ResourceGroupName ` + -DataFactoryName $script:DataFactoryName ` + -Location $script:Location ` + -Option $opt ` + -DryRun + + $result | Should -Not -BeNullOrEmpty + $result.PSObject.Properties.Name | Should -Contain 'DryRunPlan' + + $plan = $result.DryRunPlan + $plan | Should -Not -BeNullOrEmpty + $plan.Update | Should -Contain 'pipeline.PL_ExecSparkJob' + $plan.Delete | Should -Contain 'pipeline.PL_Obsolete' + $plan.Unchanged | Should -Contain 'dataset.DS_Json' + $plan.Create.Count | Should -Be 0 + } + + It 'should support -Plan as an alias behavior of -DryRun' { + $opt = New-AdfPublishOption + $opt.StopStartTriggers = $false + + Mock Get-AdfFromService { + $target = New-Object -TypeName AdfInstance + $target + } + + $result = Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName $script:ResourceGroupName ` + -DataFactoryName $script:DataFactoryName ` + -Location $script:Location ` + -Option $opt ` + -Plan + + $result | Should -Not -BeNullOrEmpty + $result.PSObject.Properties.Name | Should -Contain 'DryRunPlan' + $result.DryRunPlan | Should -Not -BeNullOrEmpty + } + } } \ No newline at end of file From 7906ec4935c2deb0316a0102676d441a39cc88cd Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 1 Jul 2026 02:58:43 +0100 Subject: [PATCH 18/20] Documentation about DryRun/Plan feature to preview deployment changes with terraform-like output #360 --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e601f2b..7688171 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,8 @@ The main advantage of the module is the ability to publish all the Azure Data Fa - [Stage value as full path to CSV config file](#stage-value-as-full-path-to-csv-config-file) - [JSON format of Config file](#json-format-of-config-file) - [Step: Deployment Plan](#step-deployment-plan) - - [Step: Stoping triggers](#step-stoping-triggers) + - [DryRun / Plan output and CI usage](#dryrun--plan-output-and-ci-usage) + - [Step: Stopping triggers](#step-stopping-triggers) - [Step: Deployment of ADF objects](#step-deployment-of-adf-objects) - [Step: Save deployment state](#step-save-deployment-state) - [Step: Deleting objects not in source](#step-deleting-objects-not-in-source) @@ -623,8 +624,45 @@ If you prefer using JSON rather than CSV for setting up configuration - JSON fil This step identifies objects to be deployed using `Includes` and `Excludes` list provided in *Publish Options*. Afterwards, if `IncrementalDeployment = true`, it excludes objects by comparing hashes from **Deployment State** to hashed of awaiting objects. +### DryRun / Plan output and CI usage -## Step: Stoping triggers +Use `-DryRun` or `-Plan` to preview intended changes without deploying. The process prints a terraform-like summary: + +```text +Terraform-like plan (DryRun): +Plan: 0 to add, 1 to change, 1 to destroy. + + ~ Update + ~ pipeline.PL_ExecSparkJob + + - Delete + - pipeline.PL_Obsolete + + = Unchanged / skipped + = dataset.DS_Json +``` + +The returned object contains a `DryRunPlan` property, so CI can enforce policy checks before deployment: + +```powershell +$result = Publish-AdfV2FromJson ` + -RootFolder $RootFolder ` + -ResourceGroupName $ResourceGroupName ` + -DataFactoryName $DataFactoryName ` + -Location $Location ` + -Option $opt ` + -Plan + +if ($result.DryRunPlan.Delete.Count -gt 0) { + Write-Error "Plan contains destroy actions. Review required before deployment." + exit 1 +} + +Write-Host "Plan accepted. Add=$($result.DryRunPlan.Create.Count); Change=$($result.DryRunPlan.Update.Count); Destroy=$($result.DryRunPlan.Delete.Count)" +``` + + +## Step: Stopping triggers 💬 In log you'll see line: `STEP: Stopping triggers...` From 26edcf1f49aa54da3a88694b1c7b93f30c2baff3 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 1 Jul 2026 03:04:52 +0100 Subject: [PATCH 19/20] Bump module version to 1.18.0 and update changelog for new DryRun/Plan feature #360 --- azure.datafactory.tools.psd1 | 2 +- changelog.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index af6b098..287169e 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -12,7 +12,7 @@ RootModule = 'azure.datafactory.tools.psm1' # Version number of this module. - ModuleVersion = '1.17.1' + ModuleVersion = '1.18.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/changelog.md b/changelog.md index 14dce27..1855d2d 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.18.0] - 2026-07-01 +### Added +* `Publish-AdfV2FromJson` now supports terraform-like planning via `-DryRun`/`-Plan`, including structured `DryRunPlan` output (Create/Update/Delete/Unchanged) #360 + ## [1.17.1] - 2026-07-01 ### Fixed * Further fixes of `DeleteNotInSource` with deserialization issue #480 From 7b5ed81406ac5622f7ae3e25d96cbad1368f067c Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Mon, 6 Jul 2026 23:38:08 +0100 Subject: [PATCH 20/20] Refactor Get-DryRunPlan to use List for collection management --- private/ApplyExclusionOptions.ps1 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/private/ApplyExclusionOptions.ps1 b/private/ApplyExclusionOptions.ps1 index eda5618..be91f8b 100644 --- a/private/ApplyExclusionOptions.ps1 +++ b/private/ApplyExclusionOptions.ps1 @@ -127,10 +127,10 @@ function Get-DryRunPlan { [parameter(Mandatory = $false)] $targetAdfInstance ) - $create = @() - $update = @() - $delete = @() - $unchanged = @() + $create = [System.Collections.Generic.List[string]]::new() + $update = [System.Collections.Generic.List[string]]::new() + $delete = [System.Collections.Generic.List[string]]::new() + $unchanged = [System.Collections.Generic.List[string]]::new() $sourceByName = @{} $adf.AllObjects() | ForEach-Object { @@ -149,13 +149,13 @@ function Get-DryRunPlan { $adf.AllObjects() | ForEach-Object { $fullName = Get-AdfObjectFullName -Object $_ if ($_.ToBeDeployed -eq $false) { - $unchanged += $fullName + $unchanged.Add($fullName) | Out-Null } elseif ($targetByName.ContainsKey($fullName)) { - $update += $fullName + $update.Add($fullName) | Out-Null } else { - $create += $fullName + $create.Add($fullName) | Out-Null } } @@ -169,7 +169,7 @@ function Get-DryRunPlan { $canDelete = !($oname.IsNameExcluded($adf.PublishOptions)) } if ($canDelete) { - $delete += $fullName + $delete.Add($fullName) | Out-Null } } }