Fixed #480 (preview)

azure.datafactory.tools.psd1

@@ -12,7 +12,7 @@
     RootModule = 'azure.datafactory.tools.psm1'
 
     # Version number of this module.
-    ModuleVersion = '1.15.0'
+    ModuleVersion = '1.16.0'
 
     # Supported PSEditions
     # CompatiblePSEditions = @()

changelog.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [1.17.0] - 2026-07-01
+### Fixed
+* Further fixes of `DeleteNotInSource` with deserialization issue #480 
+
+## [1.16.0] - 2026-05-27
+### Fixed
+* `DeleteNotInSource` no longer fails with "Unable to deserialize the response" when ADF contains object types unsupported by the installed Az.DataFactory module version - REST API fallback added to `Get-AdfFromService` #480 #473
+
+
 ## [1.15.0] - 2026-05-26
 ### Added
 * Credential type of ADF objects is now supported for deployment via REST API

private/AdfPSObjects.class.ps1

@@ -0,0 +1,40 @@
+
+# Lightweight wrapper classes for ADF objects retrieved via REST API.
+# Class names follow the AdfPS<SimplifiedType> pattern so that Get-SimplifiedType
+# strips the "AdfPS" prefix and returns the correct simplified type (e.g. "Dataset").
+# These are used as a fallback when Get-AzDataFactoryV2* cmdlets fail to deserialize
+# objects whose types are not yet known to the installed Az.DataFactory module version.
+
+class AdfPSDataset {
+    [String] $Name
+    AdfPSDataset([String] $name) { $this.Name = $name }
+}
+
+class AdfPSDataFlow {
+    [String] $Name
+    AdfPSDataFlow([String] $name) { $this.Name = $name }
+}
+
+class AdfPSPipeline {
+    [String] $Name
+    AdfPSPipeline([String] $name) { $this.Name = $name }
+}
+
+class AdfPSLinkedService {
+    [String] $Name
+    AdfPSLinkedService([String] $name) { $this.Name = $name }
+}
+
+class AdfPSIntegrationRuntime {
+    [String] $Name
+    AdfPSIntegrationRuntime([String] $name) { $this.Name = $name }
+}
+
+class AdfPSTrigger {
+    [String] $Name
+    [String] $RuntimeState
+    AdfPSTrigger([String] $name, [String] $runtimeState) {
+        $this.Name = $name
+        $this.RuntimeState = $runtimeState
+    }
+}

private/Get-AdfObjectsFromServiceRestAPI.ps1

@@ -0,0 +1,41 @@
+function Get-AdfObjectsFromServiceRestAPI {
+    [CmdletBinding()]
+    param (
+        [parameter(Mandatory = $true)] $adfi,
+        [parameter(Mandatory = $true)] [String] $typePlural,
+        [parameter(Mandatory = $true)] [String] $simpleType
+    )
+
+    Write-Debug "BEGIN: Get-AdfObjectsFromServiceRestAPI(typePlural=$typePlural, simpleType=$simpleType)"
+
+    $url = "$($script:BaseApiUrl)$($adfi.DataFactoryId)/$typePlural`?api-version=2018-06-01"
+
+    [System.Collections.ArrayList] $items = @()
+    do {
+        $r = Invoke-AzRestMethod -Method 'GET' -Uri $url
+        if ($r.StatusCode -ne 200) {
+            Write-Error -Message "Unexpected response code: $($r.StatusCode) from REST API when listing $typePlural."
+            return $null
+        }
+        $content = $r.Content | ConvertFrom-Json
+        $url = if ($content.PSObject.Properties['nextLink']) { $content.nextLink } else { $null }
+        $content.value | ForEach-Object {
+            $item = $_
+            $name = $item.name
+            $obj = $null
+            switch ($simpleType) {
+                'Dataset'            { $obj = [AdfPSDataset]::New($name) }
+                'DataFlow'           { $obj = [AdfPSDataFlow]::New($name) }
+                'Pipeline'           { $obj = [AdfPSPipeline]::New($name) }
+                'LinkedService'      { $obj = [AdfPSLinkedService]::New($name) }
+                'IntegrationRuntime' { $obj = [AdfPSIntegrationRuntime]::New($name) }
+                'Trigger'            { $obj = [AdfPSTrigger]::New($name, [string]$item.properties.runtimeState) }
+                default              { Write-Warning "Get-AdfObjectsFromServiceRestAPI: unsupported type '$simpleType'" }
+            }
+            if ($null -ne $obj) { $items.Add($obj) | Out-Null }
+        }
+    } while ($url)
+
+    Write-Debug "END: Get-AdfObjectsFromServiceRestAPI()"
+    return $items
+}

public/Get-AdfFromService.ps1

@@ -37,17 +37,47 @@ function Get-AdfFromService {
     $adf.Id = $adfi.DataFactoryId
     $adf.Location = $adfi.Location
 
-    $adf.DataSets = Get-AzDataFactoryV2Dataset -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray
+    try {
+        $adf.DataSets = Get-AzDataFactoryV2Dataset -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray
+    } catch {
+        Write-Warning "Failed to list Datasets via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_"
+        $adf.DataSets = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'datasets' -simpleType 'Dataset' | ToArray
+    }
     Write-Host ("DataSets: {0} object(s) loaded." -f $adf.DataSets.Count)
-    $adf.IntegrationRuntimes = Get-AzDataFactoryV2IntegrationRuntime -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray
+    try {
+        $adf.IntegrationRuntimes = Get-AzDataFactoryV2IntegrationRuntime -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray
+    } catch {
+        Write-Warning "Failed to list IntegrationRuntimes via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_"
+        $adf.IntegrationRuntimes = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'integrationruntimes' -simpleType 'IntegrationRuntime' | ToArray
+    }
     Write-Host ("IntegrationRuntimes: {0} object(s) loaded." -f $adf.IntegrationRuntimes.Count)
-    $adf.LinkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray
+    try {
+        $adf.LinkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray
+    } catch {
+        Write-Warning "Failed to list LinkedServices via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_"
+        $adf.LinkedServices = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'linkedservices' -simpleType 'LinkedService' | ToArray
+    }
     Write-Host ("LinkedServices: {0} object(s) loaded." -f $adf.LinkedServices.Count)
-    $adf.Pipelines = Get-AzDataFactoryV2Pipeline -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray
+    try {
+        $adf.Pipelines = Get-AzDataFactoryV2Pipeline -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray
+    } catch {
+        Write-Warning "Failed to list Pipelines via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_"
+        $adf.Pipelines = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'pipelines' -simpleType 'Pipeline' | ToArray
+    }
     Write-Host ("Pipelines: {0} object(s) loaded." -f $adf.Pipelines.Count)
-    $adf.DataFlows = Get-AzDataFactoryV2DataFlow -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray
+    try {
+        $adf.DataFlows = Get-AzDataFactoryV2DataFlow -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray
+    } catch {
+        Write-Warning "Failed to list DataFlows via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_"
+        $adf.DataFlows = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'dataflows' -simpleType 'DataFlow' | ToArray
+    }
     Write-Host ("DataFlows: {0} object(s) loaded." -f $adf.DataFlows.Count)
-    $adf.Triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray
+    try {
+        $adf.Triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray
+    } catch {
+        Write-Warning "Failed to list Triggers via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_"
+        $adf.Triggers = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'triggers' -simpleType 'Trigger' | ToArray
+    }
     Write-Host ("Triggers: {0} object(s) loaded." -f $adf.Triggers.Count)
     $adf.Credentials = Get-AzDFV2Credential -adfi $adfi | ToArray
     Write-Host ("Credentials: {0} object(s) loaded." -f $adf.Credentials.Count)

test/!RunAllTests.ps1

@@ -51,13 +51,15 @@ if ($InstallModules) {
         #Install-Module 'Az.DataFactory' -Force -MinimumVersion 1.16.0 -Repository 'PSGallery'
         #Install-Module 'PSScriptAnalyzer' -Force
         #Install-Module 'Pester' -Force -MinimumVersion 5.1.1
+        Install-Module 'Az.Storage' -Force -AllowClobber -Repository 'PSGallery'
         [Environment]::SetEnvironmentVariable("azure.datafactory.tools.unitTestInstalledModules", $true, 'Process');
     }
 } else {
     Write-Host "Installation skipped."
 }
 Import-Module 'Pester' -MinimumVersion 5.3.3
 Import-Module 'PSScriptAnalyzer'
+Import-Module 'Az.Storage'
 Import-Module "$folder\azure.datafactory.tools.psd1"
 
 Write-Host "=============== Modules ================"
@@ -96,4 +98,5 @@ try {
     Pop-Location
 }
 
-# . ".\test\!RunAllTests.ps1" -folder '.\test' -TestFilenameFilter '*' -MajorRelease:$true -InstallModules:$false 
+# Connect-AzAccount
+# . ".\test\!RunAllTests.ps1" -folder '.\test' -TestFilenameFilter 'Inc*' -MajorRelease:$true -InstallModules:$false 

test/Incremental-Deployment.Tests.ps1

@@ -12,6 +12,9 @@ InModuleScope azure.datafactory.tools {
     $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper'
     Import-Module -Name $testHelperPath -Force
 
+    $m = Get-Module -Name 'azure.datafactory.tools'
+    $script:verStr = $m.Version.ToString(2) + "." + $m.Version.Build.ToString("000")
+
     # $VerbosePreference = 'Continue'
     # $DebugPreference = 'Continue'
 
@@ -29,7 +32,7 @@ InModuleScope azure.datafactory.tools {
     $opt.IncrementalDeployment = $true
     $opt.StopStartTriggers = $false
     $script:gp = "" 
-    $script:dstate = [AdfDeploymentState]::new($verStr)
+    $script:dstate = [AdfDeploymentState]::new($script:verStr)
     $script:dstate.LastUpdate = [System.DateTime]::UtcNow
     $script:dstateJson = $script:dstate | ConvertTo-Json
     $script:StorageUri= "https://sqlplayer2020.blob.core.windows.net"