From 5f1f07f3eb6cbcd2a1bba1887edc633d2cf65586 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 27 May 2026 02:16:52 +0100 Subject: [PATCH 1/6] Fixed #480 (preview) --- azure.datafactory.tools.psd1 | 2 +- changelog.md | 5 +++ private/AdfPSObjects.class.ps1 | 40 +++++++++++++++++++ private/Get-AdfObjectsFromServiceRestAPI.ps1 | 36 +++++++++++++++++ public/Get-AdfFromService.ps1 | 42 +++++++++++++++++--- 5 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 private/AdfPSObjects.class.ps1 create mode 100644 private/Get-AdfObjectsFromServiceRestAPI.ps1 diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index 8bccb32..f99e0db 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -12,7 +12,7 @@ RootModule = 'azure.datafactory.tools.psm1' # Version number of this module. - ModuleVersion = '1.15.0' + ModuleVersion = '1.16.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/changelog.md b/changelog.md index 91a3b61..6b73285 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.16.0] - 2026-05-27 +### Fixed +* `DeleteNotInSource` no longer fails with "Unable to deserialize the response" when ADF contains object types unsupported by the installed Az.DataFactory module version - REST API fallback added to `Get-AdfFromService` #480 #473 + + ## [1.15.0] - 2026-05-26 ### Added * Credential type of ADF objects is now supported for deployment via REST API diff --git a/private/AdfPSObjects.class.ps1 b/private/AdfPSObjects.class.ps1 new file mode 100644 index 0000000..f2ca37c --- /dev/null +++ b/private/AdfPSObjects.class.ps1 @@ -0,0 +1,40 @@ + +# Lightweight wrapper classes for ADF objects retrieved via REST API. +# Class names follow the AdfPS pattern so that Get-SimplifiedType +# strips the "AdfPS" prefix and returns the correct simplified type (e.g. "Dataset"). +# These are used as a fallback when Get-AzDataFactoryV2* cmdlets fail to deserialize +# objects whose types are not yet known to the installed Az.DataFactory module version. + +class AdfPSDataset { + [String] $Name + AdfPSDataset([String] $name) { $this.Name = $name } +} + +class AdfPSDataFlow { + [String] $Name + AdfPSDataFlow([String] $name) { $this.Name = $name } +} + +class AdfPSPipeline { + [String] $Name + AdfPSPipeline([String] $name) { $this.Name = $name } +} + +class AdfPSLinkedService { + [String] $Name + AdfPSLinkedService([String] $name) { $this.Name = $name } +} + +class AdfPSIntegrationRuntime { + [String] $Name + AdfPSIntegrationRuntime([String] $name) { $this.Name = $name } +} + +class AdfPSTrigger { + [String] $Name + [String] $RuntimeState + AdfPSTrigger([String] $name, [String] $runtimeState) { + $this.Name = $name + $this.RuntimeState = $runtimeState + } +} diff --git a/private/Get-AdfObjectsFromServiceRestAPI.ps1 b/private/Get-AdfObjectsFromServiceRestAPI.ps1 new file mode 100644 index 0000000..a9ad167 --- /dev/null +++ b/private/Get-AdfObjectsFromServiceRestAPI.ps1 @@ -0,0 +1,36 @@ +function Get-AdfObjectsFromServiceRestAPI { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] $adfi, + [parameter(Mandatory = $true)] [String] $typePlural, + [parameter(Mandatory = $true)] [String] $simpleType + ) + + Write-Debug "BEGIN: Get-AdfObjectsFromServiceRestAPI(typePlural=$typePlural, simpleType=$simpleType)" + + $url = "$($script:BaseApiUrl)$($adfi.DataFactoryId)/$typePlural`?api-version=2018-06-01" + $r = Invoke-AzRestMethod -Method 'GET' -Uri $url + if ($r.StatusCode -ne 200) { + Write-Error -Message "Unexpected response code: $($r.StatusCode) from REST API when listing $typePlural." + return $null + } + + [System.Collections.ArrayList] $items = @() + ($r.Content | ConvertFrom-Json).value | ForEach-Object { + $name = $_.name + $obj = $null + switch ($simpleType) { + 'Dataset' { $obj = [AdfPSDataset]::New($name) } + 'DataFlow' { $obj = [AdfPSDataFlow]::New($name) } + 'Pipeline' { $obj = [AdfPSPipeline]::New($name) } + 'LinkedService' { $obj = [AdfPSLinkedService]::New($name) } + 'IntegrationRuntime' { $obj = [AdfPSIntegrationRuntime]::New($name) } + 'Trigger' { $obj = [AdfPSTrigger]::New($name, [string]$_.properties.runtimeState) } + default { Write-Warning "Get-AdfObjectsFromServiceRestAPI: unsupported type '$simpleType'" } + } + if ($null -ne $obj) { $items.Add($obj) | Out-Null } + } + + Write-Debug "END: Get-AdfObjectsFromServiceRestAPI()" + return $items +} diff --git a/public/Get-AdfFromService.ps1 b/public/Get-AdfFromService.ps1 index 40476c9..37cd98f 100644 --- a/public/Get-AdfFromService.ps1 +++ b/public/Get-AdfFromService.ps1 @@ -37,17 +37,47 @@ function Get-AdfFromService { $adf.Id = $adfi.DataFactoryId $adf.Location = $adfi.Location - $adf.DataSets = Get-AzDataFactoryV2Dataset -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.DataSets = Get-AzDataFactoryV2Dataset -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + } catch { + Write-Warning "Failed to list Datasets via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.DataSets = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'datasets' -simpleType 'Dataset' | ToArray + } Write-Host ("DataSets: {0} object(s) loaded." -f $adf.DataSets.Count) - $adf.IntegrationRuntimes = Get-AzDataFactoryV2IntegrationRuntime -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.IntegrationRuntimes = Get-AzDataFactoryV2IntegrationRuntime -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + } catch { + Write-Warning "Failed to list IntegrationRuntimes via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.IntegrationRuntimes = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'integrationruntimes' -simpleType 'IntegrationRuntime' | ToArray + } Write-Host ("IntegrationRuntimes: {0} object(s) loaded." -f $adf.IntegrationRuntimes.Count) - $adf.LinkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.LinkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + } catch { + Write-Warning "Failed to list LinkedServices via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.LinkedServices = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'linkedservices' -simpleType 'LinkedService' | ToArray + } Write-Host ("LinkedServices: {0} object(s) loaded." -f $adf.LinkedServices.Count) - $adf.Pipelines = Get-AzDataFactoryV2Pipeline -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.Pipelines = Get-AzDataFactoryV2Pipeline -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + } catch { + Write-Warning "Failed to list Pipelines via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.Pipelines = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'pipelines' -simpleType 'Pipeline' | ToArray + } Write-Host ("Pipelines: {0} object(s) loaded." -f $adf.Pipelines.Count) - $adf.DataFlows = Get-AzDataFactoryV2DataFlow -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.DataFlows = Get-AzDataFactoryV2DataFlow -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + } catch { + Write-Warning "Failed to list DataFlows via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.DataFlows = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'dataflows' -simpleType 'DataFlow' | ToArray + } Write-Host ("DataFlows: {0} object(s) loaded." -f $adf.DataFlows.Count) - $adf.Triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.Triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + } catch { + Write-Warning "Failed to list Triggers via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.Triggers = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'triggers' -simpleType 'Trigger' | ToArray + } Write-Host ("Triggers: {0} object(s) loaded." -f $adf.Triggers.Count) $adf.Credentials = Get-AzDFV2Credential -adfi $adfi | ToArray Write-Host ("Credentials: {0} object(s) loaded." -f $adf.Credentials.Count) From c4bc7b70908536170a8d26aed33f240f12f543d5 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Tue, 30 Jun 2026 22:58:03 +0100 Subject: [PATCH 2/6] Refactor Get-AdfObjectsFromServiceRestAPI to handle pagination and improve error handling; update Get-AdfFromService to use -ErrorAction Stop for Az cmdlets --- private/Get-AdfObjectsFromServiceRestAPI.ps1 | 41 +-- public/Get-AdfFromService.ps1 | 12 +- test/zGet-AdfFromService.Tests.ps1 | 303 ++++++++++++++++++- 3 files changed, 325 insertions(+), 31 deletions(-) diff --git a/private/Get-AdfObjectsFromServiceRestAPI.ps1 b/private/Get-AdfObjectsFromServiceRestAPI.ps1 index a9ad167..e27f563 100644 --- a/private/Get-AdfObjectsFromServiceRestAPI.ps1 +++ b/private/Get-AdfObjectsFromServiceRestAPI.ps1 @@ -9,27 +9,32 @@ function Get-AdfObjectsFromServiceRestAPI { Write-Debug "BEGIN: Get-AdfObjectsFromServiceRestAPI(typePlural=$typePlural, simpleType=$simpleType)" $url = "$($script:BaseApiUrl)$($adfi.DataFactoryId)/$typePlural`?api-version=2018-06-01" - $r = Invoke-AzRestMethod -Method 'GET' -Uri $url - if ($r.StatusCode -ne 200) { - Write-Error -Message "Unexpected response code: $($r.StatusCode) from REST API when listing $typePlural." - return $null - } [System.Collections.ArrayList] $items = @() - ($r.Content | ConvertFrom-Json).value | ForEach-Object { - $name = $_.name - $obj = $null - switch ($simpleType) { - 'Dataset' { $obj = [AdfPSDataset]::New($name) } - 'DataFlow' { $obj = [AdfPSDataFlow]::New($name) } - 'Pipeline' { $obj = [AdfPSPipeline]::New($name) } - 'LinkedService' { $obj = [AdfPSLinkedService]::New($name) } - 'IntegrationRuntime' { $obj = [AdfPSIntegrationRuntime]::New($name) } - 'Trigger' { $obj = [AdfPSTrigger]::New($name, [string]$_.properties.runtimeState) } - default { Write-Warning "Get-AdfObjectsFromServiceRestAPI: unsupported type '$simpleType'" } + do { + $r = Invoke-AzRestMethod -Method 'GET' -Uri $url + if ($r.StatusCode -ne 200) { + Write-Error -Message "Unexpected response code: $($r.StatusCode) from REST API when listing $typePlural." + return $null } - if ($null -ne $obj) { $items.Add($obj) | Out-Null } - } + $content = $r.Content | ConvertFrom-Json + $url = if ($content.PSObject.Properties['nextLink']) { $content.nextLink } else { $null } + $content.value | ForEach-Object { + $item = $_ + $name = $item.name + $obj = $null + switch ($simpleType) { + 'Dataset' { $obj = [AdfPSDataset]::New($name) } + 'DataFlow' { $obj = [AdfPSDataFlow]::New($name) } + 'Pipeline' { $obj = [AdfPSPipeline]::New($name) } + 'LinkedService' { $obj = [AdfPSLinkedService]::New($name) } + 'IntegrationRuntime' { $obj = [AdfPSIntegrationRuntime]::New($name) } + 'Trigger' { $obj = [AdfPSTrigger]::New($name, [string]$item.properties.runtimeState) } + default { Write-Warning "Get-AdfObjectsFromServiceRestAPI: unsupported type '$simpleType'" } + } + if ($null -ne $obj) { $items.Add($obj) | Out-Null } + } + } while ($url) Write-Debug "END: Get-AdfObjectsFromServiceRestAPI()" return $items diff --git a/public/Get-AdfFromService.ps1 b/public/Get-AdfFromService.ps1 index 37cd98f..0e577a9 100644 --- a/public/Get-AdfFromService.ps1 +++ b/public/Get-AdfFromService.ps1 @@ -38,42 +38,42 @@ function Get-AdfFromService { $adf.Location = $adfi.Location try { - $adf.DataSets = Get-AzDataFactoryV2Dataset -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + $adf.DataSets = Get-AzDataFactoryV2Dataset -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray } catch { Write-Warning "Failed to list Datasets via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" $adf.DataSets = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'datasets' -simpleType 'Dataset' | ToArray } Write-Host ("DataSets: {0} object(s) loaded." -f $adf.DataSets.Count) try { - $adf.IntegrationRuntimes = Get-AzDataFactoryV2IntegrationRuntime -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + $adf.IntegrationRuntimes = Get-AzDataFactoryV2IntegrationRuntime -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray } catch { Write-Warning "Failed to list IntegrationRuntimes via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" $adf.IntegrationRuntimes = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'integrationruntimes' -simpleType 'IntegrationRuntime' | ToArray } Write-Host ("IntegrationRuntimes: {0} object(s) loaded." -f $adf.IntegrationRuntimes.Count) try { - $adf.LinkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + $adf.LinkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray } catch { Write-Warning "Failed to list LinkedServices via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" $adf.LinkedServices = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'linkedservices' -simpleType 'LinkedService' | ToArray } Write-Host ("LinkedServices: {0} object(s) loaded." -f $adf.LinkedServices.Count) try { - $adf.Pipelines = Get-AzDataFactoryV2Pipeline -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + $adf.Pipelines = Get-AzDataFactoryV2Pipeline -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray } catch { Write-Warning "Failed to list Pipelines via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" $adf.Pipelines = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'pipelines' -simpleType 'Pipeline' | ToArray } Write-Host ("Pipelines: {0} object(s) loaded." -f $adf.Pipelines.Count) try { - $adf.DataFlows = Get-AzDataFactoryV2DataFlow -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + $adf.DataFlows = Get-AzDataFactoryV2DataFlow -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray } catch { Write-Warning "Failed to list DataFlows via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" $adf.DataFlows = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'dataflows' -simpleType 'DataFlow' | ToArray } Write-Host ("DataFlows: {0} object(s) loaded." -f $adf.DataFlows.Count) try { - $adf.Triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + $adf.Triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray } catch { Write-Warning "Failed to list Triggers via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" $adf.Triggers = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'triggers' -simpleType 'Trigger' | ToArray diff --git a/test/zGet-AdfFromService.Tests.ps1 b/test/zGet-AdfFromService.Tests.ps1 index 82f7b8a..1b0ce07 100644 --- a/test/zGet-AdfFromService.Tests.ps1 +++ b/test/zGet-AdfFromService.Tests.ps1 @@ -10,21 +10,310 @@ InModuleScope azure.datafactory.tools { $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' Import-Module -Name $testHelperPath -Force - # Variables for use in tests - $t = Get-TargetEnv 'adf2' - $script:adfName = $t.DataFactoryName - $script:rg = $t.ResourceGroupName - + # Shared factory stub + $script:fakeAdfi = [PSCustomObject]@{ + DataFactoryId = '/subscriptions/sub-123/resourceGroups/rg-test/providers/Microsoft.DataFactory/factories/adf-test' + Location = 'northeurope' + } + + # --------------------------------------------------------------------------- Describe 'Get-AdfFromService' -Tag 'Unit' { + It 'Should exist' { { Get-Command -Name Get-AdfFromService -ErrorAction Stop } | Should -Not -Throw } - } + Context 'When all Az cmdlets succeed (happy path)' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + Mock Get-AzDataFactoryV2Dataset { @() } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { @() } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { @() } + Mock Get-AzDFV2Credential { @() } + } + + It 'Should return an AdfInstance with the correct factory name' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.Name | Should -Be 'adf-test' + } + + It 'Should not call Invoke-AzRestMethod when Az cmdlets succeed' { + Mock Invoke-AzRestMethod { throw 'Should not be called' } + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + } + + Context 'When Get-AzDataFactoryV2Dataset throws a deserialization error (issue #480)' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + Mock Get-AzDataFactoryV2Dataset { throw 'Unable to deserialize the response.' } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { @() } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { @() } + Mock Get-AzDFV2Credential { @() } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ds_adls_csv","properties":{}},{"name":"ds_servicenow_v2","properties":{}}]}' + } + } + } + + It 'Should not throw' { + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + + It 'Should fall back to REST API and return all datasets by name' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.DataSets.Count | Should -Be 2 + $result.DataSets.Name | Should -Contain 'ds_adls_csv' + $result.DataSets.Name | Should -Contain 'ds_servicenow_v2' + } + + It 'Should return datasets as AdfPSDataset wrapper objects' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.DataSets | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSDataset' } + } + + It 'Should call Invoke-AzRestMethod with the correct datasets URL' { + Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly -ParameterFilter { + $Uri -like '*/datasets?api-version=2018-06-01' + } + } + } + + Context 'When Get-AzDataFactoryV2LinkedService throws a deserialization error' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + Mock Get-AzDataFactoryV2Dataset { @() } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { throw 'Unable to deserialize the response.' } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { @() } + Mock Get-AzDFV2Credential { @() } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ls_adls","properties":{}},{"name":"ls_servicenow","properties":{}}]}' + } + } + } + + It 'Should not throw' { + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + + It 'Should return linked services via REST fallback as AdfPSLinkedService objects' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.LinkedServices.Count | Should -Be 2 + $result.LinkedServices | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSLinkedService' } + } + } + + Context 'When Get-AzDataFactoryV2Trigger throws a deserialization error' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + Mock Get-AzDataFactoryV2Dataset { @() } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { @() } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { throw 'Unable to deserialize the response.' } + Mock Get-AzDFV2Credential { @() } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"tr_daily","properties":{"runtimeState":"Started"}},{"name":"tr_hourly","properties":{"runtimeState":"Stopped"}}]}' + } + } + } + + It 'Should not throw' { + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + + It 'Should return triggers as AdfPSTrigger objects with correct RuntimeState' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.Triggers.Count | Should -Be 2 + $result.Triggers | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSTrigger' } + ($result.Triggers | Where-Object Name -eq 'tr_daily').RuntimeState | Should -Be 'Started' + ($result.Triggers | Where-Object Name -eq 'tr_hourly').RuntimeState | Should -Be 'Stopped' + } + } + } + + # --------------------------------------------------------------------------- + Describe 'Get-AdfObjectsFromServiceRestAPI' -Tag 'Unit' { + + It 'Should exist' { + { Get-Command -Name Get-AdfObjectsFromServiceRestAPI -ErrorAction Stop } | Should -Not -Throw + } + + Context 'When the API returns an empty list' { + BeforeEach { + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ StatusCode = 200; Content = '{"value":[]}' } + } + } + + It 'Should return an empty collection' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + @($result).Count | Should -Be 0 + } + + It 'Should call Invoke-AzRestMethod exactly once' { + Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly + } + } + + Context 'When the API returns a single page of datasets' { + BeforeEach { + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ds_one","properties":{}},{"name":"ds_two","properties":{}},{"name":"ds_three","properties":{}}]}' + } + } + } + + It 'Should return all items from the page' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $result.Count | Should -Be 3 + } + + It 'Should return AdfPSDataset wrapper objects' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $result | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSDataset' } + } + + It 'Should preserve the dataset names' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $result.Name | Should -Contain 'ds_one' + $result.Name | Should -Contain 'ds_two' + $result.Name | Should -Contain 'ds_three' + } + + It 'Should call Invoke-AzRestMethod exactly once (no extra pages)' { + Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly + } + } + + Context 'When the API returns two pages (pagination)' { + BeforeEach { + $script:page2Url = 'https://management.azure.com/subscriptions/sub-123/.../datasets?api-version=2018-06-01&skipToken=page2' + $script:callCount = 0 + Mock Invoke-AzRestMethod { + $script:callCount++ + if ($script:callCount -eq 1) { + return [PSCustomObject]@{ + StatusCode = 200 + Content = "{`"value`":[{`"name`":`"ds_p1_a`",`"properties`":{}},{`"name`":`"ds_p1_b`",`"properties`":{}}],`"nextLink`":`"$($script:page2Url)`"}" + } + } else { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ds_p2_a","properties":{}}]}' + } + } + } + } + + It 'Should return items from all pages combined' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $result.Count | Should -Be 3 + $result.Name | Should -Contain 'ds_p1_a' + $result.Name | Should -Contain 'ds_p1_b' + $result.Name | Should -Contain 'ds_p2_a' + } + + It 'Should call Invoke-AzRestMethod twice (once per page)' { + Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + Assert-MockCalled Invoke-AzRestMethod -Times 2 -Exactly + } + + It 'Should use the nextLink URL for the second request' { + Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly -ParameterFilter { + $Uri -eq $script:page2Url + } + } + } + + Context 'When called for each supported object type' { + BeforeEach { + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"obj1","properties":{"runtimeState":"Stopped"}}]}' + } + } + } + + It 'Should return AdfPSDataset for simpleType Dataset' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $r[0].GetType().Name | Should -Be 'AdfPSDataset' + } + + It 'Should return AdfPSPipeline for simpleType Pipeline' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'pipelines' -simpleType 'Pipeline' + $r[0].GetType().Name | Should -Be 'AdfPSPipeline' + } + + It 'Should return AdfPSLinkedService for simpleType LinkedService' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'linkedservices' -simpleType 'LinkedService' + $r[0].GetType().Name | Should -Be 'AdfPSLinkedService' + } + + It 'Should return AdfPSIntegrationRuntime for simpleType IntegrationRuntime' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'integrationruntimes' -simpleType 'IntegrationRuntime' + $r[0].GetType().Name | Should -Be 'AdfPSIntegrationRuntime' + } + + It 'Should return AdfPSDataFlow for simpleType DataFlow' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'dataflows' -simpleType 'DataFlow' + $r[0].GetType().Name | Should -Be 'AdfPSDataFlow' + } + + It 'Should return AdfPSTrigger for simpleType Trigger with RuntimeState' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'triggers' -simpleType 'Trigger' + $r[0].GetType().Name | Should -Be 'AdfPSTrigger' + $r[0].RuntimeState | Should -Be 'Stopped' + } + } + + Context 'When the API returns a non-200 status code' { + BeforeEach { + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ StatusCode = 403; Content = '{"error":{"code":"AuthorizationFailed"}}' } + } + } + + It 'Should throw when ErrorAction Stop is used' { + { Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' -ErrorAction Stop } | Should -Throw + } + } + } + + # --------------------------------------------------------------------------- Describe 'Get-AdfFromService' -Tag 'Integration' { + # Variables for use in integration tests + $t = Get-TargetEnv 'adf2' + $script:adfName = $t.DataFactoryName + $script:rg = $t.ResourceGroupName + It 'Should execute' { Get-AdfFromService -FactoryName $adfName -ResourceGroupName $rg } - } + } } + From 3e6cf731852b3406e51ebd95ee17260e1567332b Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Tue, 30 Jun 2026 23:47:40 +0100 Subject: [PATCH 3/6] Add tests for handling non-terminating errors in Get-AdfFromService --- test/zGet-AdfFromService.Tests.ps1 | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/zGet-AdfFromService.Tests.ps1 b/test/zGet-AdfFromService.Tests.ps1 index 1b0ce07..a919338 100644 --- a/test/zGet-AdfFromService.Tests.ps1 +++ b/test/zGet-AdfFromService.Tests.ps1 @@ -88,6 +88,49 @@ InModuleScope azure.datafactory.tools { } } + # This context specifically tests the real-world failure mode from issue #480: + # The Az.DataFactory SDK writes a *non-terminating* Write-Error (not a throw) when + # it cannot deserialize an unsupported object type such as ServiceNow V2. + # Without -ErrorAction Stop on the cmdlet call the try-catch never fires; with it, + # the non-terminating error is promoted to terminating and the catch fires correctly. + Context 'When Get-AzDataFactoryV2Dataset emits a non-terminating deserialization error (real Az.DataFactory behaviour)' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + # Simulate the actual Az.DataFactory SDK behaviour: Write-Error (non-terminating), + # NOT throw. The cmdlet call in Get-AdfFromService uses -ErrorAction Stop which + # must promote this to a terminating error so the catch block fires. + Mock Get-AzDataFactoryV2Dataset { Write-Error 'Unable to deserialize the response.' } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { @() } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { @() } + Mock Get-AzDFV2Credential { @() } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ds_adls_csv","properties":{}},{"name":"ds_servicenow_v2","properties":{}}]}' + } + } + } + + It 'Should not throw even when the cmdlet only writes a non-terminating error' { + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + + It 'Should activate the REST API fallback and return all datasets' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.DataSets.Count | Should -Be 2 + $result.DataSets.Name | Should -Contain 'ds_adls_csv' + $result.DataSets.Name | Should -Contain 'ds_servicenow_v2' + } + + It 'Should return AdfPSDataset wrapper objects from the REST fallback' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.DataSets | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSDataset' } + } + } + Context 'When Get-AzDataFactoryV2LinkedService throws a deserialization error' { BeforeEach { Mock Get-AzDataFactoryV2 { $script:fakeAdfi } From ba7dd153b6a3a4c3e8f93e98c6e9ea5310869aee Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Tue, 30 Jun 2026 23:56:17 +0100 Subject: [PATCH 4/6] changelog update - 1.17.0 --- changelog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changelog.md b/changelog.md index 6b73285..db539bc 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.17.0] - 2026-07-01 +### Fixed +* Further fixes of `DeleteNotInSource` with deserialization issue #480 + ## [1.16.0] - 2026-05-27 ### Fixed * `DeleteNotInSource` no longer fails with "Unable to deserialize the response" when ADF contains object types unsupported by the installed Az.DataFactory module version - REST API fallback added to `Get-AdfFromService` #480 #473 From 516ae683faa4e5d866d832e137164c3356b3524f Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 1 Jul 2026 00:03:08 +0100 Subject: [PATCH 5/6] Fixed simple error in unit test --- test/Incremental-Deployment.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Incremental-Deployment.Tests.ps1 b/test/Incremental-Deployment.Tests.ps1 index 5382161..bb07c0a 100644 --- a/test/Incremental-Deployment.Tests.ps1 +++ b/test/Incremental-Deployment.Tests.ps1 @@ -29,7 +29,7 @@ InModuleScope azure.datafactory.tools { $opt.IncrementalDeployment = $true $opt.StopStartTriggers = $false $script:gp = "" - $script:dstate = [AdfDeploymentState]::new($verStr) + $script:dstate = [AdfDeploymentState]::new($script:verStr) $script:dstate.LastUpdate = [System.DateTime]::UtcNow $script:dstateJson = $script:dstate | ConvertTo-Json $script:StorageUri= "https://sqlplayer2020.blob.core.windows.net" From cd2bd4e64587382c3e33a8654f78178146bd84be Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 1 Jul 2026 00:20:12 +0100 Subject: [PATCH 6/6] Fix unit test issue c.d. --- test/!RunAllTests.ps1 | 5 ++++- test/Incremental-Deployment.Tests.ps1 | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/!RunAllTests.ps1 b/test/!RunAllTests.ps1 index ba01f7c..1c4922d 100644 --- a/test/!RunAllTests.ps1 +++ b/test/!RunAllTests.ps1 @@ -51,6 +51,7 @@ if ($InstallModules) { #Install-Module 'Az.DataFactory' -Force -MinimumVersion 1.16.0 -Repository 'PSGallery' #Install-Module 'PSScriptAnalyzer' -Force #Install-Module 'Pester' -Force -MinimumVersion 5.1.1 + Install-Module 'Az.Storage' -Force -AllowClobber -Repository 'PSGallery' [Environment]::SetEnvironmentVariable("azure.datafactory.tools.unitTestInstalledModules", $true, 'Process'); } } else { @@ -58,6 +59,7 @@ if ($InstallModules) { } Import-Module 'Pester' -MinimumVersion 5.3.3 Import-Module 'PSScriptAnalyzer' +Import-Module 'Az.Storage' Import-Module "$folder\azure.datafactory.tools.psd1" Write-Host "=============== Modules ================" @@ -96,4 +98,5 @@ try { Pop-Location } -# . ".\test\!RunAllTests.ps1" -folder '.\test' -TestFilenameFilter '*' -MajorRelease:$true -InstallModules:$false +# Connect-AzAccount +# . ".\test\!RunAllTests.ps1" -folder '.\test' -TestFilenameFilter 'Inc*' -MajorRelease:$true -InstallModules:$false diff --git a/test/Incremental-Deployment.Tests.ps1 b/test/Incremental-Deployment.Tests.ps1 index bb07c0a..d5824c2 100644 --- a/test/Incremental-Deployment.Tests.ps1 +++ b/test/Incremental-Deployment.Tests.ps1 @@ -12,6 +12,9 @@ InModuleScope azure.datafactory.tools { $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' Import-Module -Name $testHelperPath -Force + $m = Get-Module -Name 'azure.datafactory.tools' + $script:verStr = $m.Version.ToString(2) + "." + $m.Version.Build.ToString("000") + # $VerbosePreference = 'Continue' # $DebugPreference = 'Continue'