diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml new file mode 100644 index 0000000..7167fd2 --- /dev/null +++ b/.github/workflows/ci-cd.yml @@ -0,0 +1,72 @@ +name: CI/CD + +on: + push: + branches: [ develop ] + paths: + - 'private/**' + - 'public/**' + - 'test/**' + pull_request: + branches: [ develop ] + paths: + - 'private/**' + - 'public/**' + - 'test/**' + workflow_dispatch: + +permissions: + contents: read + checks: write # required by dorny/test-reporter + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Run Unit Tests + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + + - name: Login via Az module + uses: azure/login@v3 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + enable-AzPSSession: true + + - name: Run Unit Tests + uses: azure/powershell@v3 + with: + inlineScript: | + test/!RunAllTests.ps1 -folder '${{ github.workspace }}/' -TestFilenameFilter "*" -InstallModules:$false -ResultOutputFormat "JUnitXml" + azPSVersion: "latest" + + - name: Publish Test Results + uses: dorny/test-reporter@v3 + if: ${{ !cancelled() }} + with: + name: Pester Tests + path: '**/TEST-*.xml' + reporter: java-junit + collapsed: 'auto' + + publish: + name: Publish to PS Gallery + runs-on: windows-latest + needs: test + if: ${{ github.event_name != 'pull_request' && (github.ref_name == 'master' || github.ref_name == 'develop') }} + + steps: + - uses: actions/checkout@v4 + + - name: Publish module to PowerShell Gallery + shell: pwsh + env: + PSGALLERY_API_KEY: ${{ secrets.PSGALLERY_API_KEY }} + run: | + $isPreview = '${{ github.ref_name }}' -ne 'master' + Write-Host "Branch: ${{ github.ref_name }} | Preview: $isPreview" + ./tools/Publish-ToGallery.ps1 -IsPreview:$isPreview -NuGetApiKey $env:PSGALLERY_API_KEY diff --git a/.github/workflows/develop-ci.yml b/.github/workflows/develop-ci.yml index 942fb9a..2274f18 100644 --- a/.github/workflows/develop-ci.yml +++ b/.github/workflows/develop-ci.yml @@ -1,10 +1,18 @@ name: CI on: - push: - branches: [ develop ] - pull_request: - branches: [ develop ] + # push: + # branches: [ develop ] + # paths: + # - 'private/**' + # - 'public/**' + # - 'test/**' + # pull_request: + # branches: [ develop ] + # paths: + # - 'private/**' + # - 'public/**' + # - 'test/**' workflow_dispatch: permissions: diff --git a/README.md b/README.md index 1f34204..7688171 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,8 @@ The main advantage of the module is the ability to publish all the Azure Data Fa - [Stage value as full path to CSV config file](#stage-value-as-full-path-to-csv-config-file) - [JSON format of Config file](#json-format-of-config-file) - [Step: Deployment Plan](#step-deployment-plan) - - [Step: Stoping triggers](#step-stoping-triggers) + - [DryRun / Plan output and CI usage](#dryrun--plan-output-and-ci-usage) + - [Step: Stopping triggers](#step-stopping-triggers) - [Step: Deployment of ADF objects](#step-deployment-of-adf-objects) - [Step: Save deployment state](#step-save-deployment-state) - [Step: Deleting objects not in source](#step-deleting-objects-not-in-source) @@ -185,6 +186,7 @@ Publish-AdfV2FromJson [-Option] [-Method] [-DryRun] + [-Plan] ``` Assuming your ADF is named ```SQLPlayerDemo``` and the code is located in ```c:\GitHub\AdfName\```, replace the values for *SubscriptionName*, *ResourceGroupName*, *DataFactoryName* and run the following command using PowerShell CLI: @@ -221,6 +223,7 @@ $opt = New-AdfPublishOption * [Boolean] **DeployGlobalParams** - indicates whether deploy Global Parameters of ADF. Nothing happens when parameters are not defined. (default: *true*) * [Boolean] **FailsWhenConfigItemNotFound** - indicates whether configuration items not found fails the script. (default: *true*) * [Boolean] **FailsWhenPathNotFound** - indicates whether missing paths fails the script. (default: *true*) +* [Boolean] **IgnoreLackOfReferencedObject** - indicates whether missing referenced objects should only raise warnings. (default: *false*) * [Boolean] **DoNotStopStartExcludedTriggers** - specifies whether excluded triggers will be stopped before deployment (default: *false*) * [Boolean] **DoNotDeleteExcludedObjects** - specifies whether excluded objects can be removed. Applies when `DeleteNotInSource` is set to *True* only. (default: *true*) * [Boolean] **IncrementalDeployment** - specifies whether Incremental Deployment mode is enabled (default: *false*) @@ -265,7 +268,11 @@ $opt.FailsWhenConfigItemNotFound = $false $opt = New-AdfPublishOption $opt.FailsWhenPathNotFound = $false -# Example 7: Exclude Infrastructure-type of objects from deployment +# Example 7: Ignore missing referenced objects (will just write warning to standard output instead) +$opt = New-AdfPublishOption +$opt.IgnoreLackOfReferencedObject = $true + +# Example 8: Exclude Infrastructure-type of objects from deployment $opt = New-AdfPublishOption $opt.CreateNewInstance = $false $opt.Excludes.Add("integrationruntime.*", "") @@ -617,8 +624,45 @@ If you prefer using JSON rather than CSV for setting up configuration - JSON fil This step identifies objects to be deployed using `Includes` and `Excludes` list provided in *Publish Options*. Afterwards, if `IncrementalDeployment = true`, it excludes objects by comparing hashes from **Deployment State** to hashed of awaiting objects. +### DryRun / Plan output and CI usage + +Use `-DryRun` or `-Plan` to preview intended changes without deploying. The process prints a terraform-like summary: + +```text +Terraform-like plan (DryRun): +Plan: 0 to add, 1 to change, 1 to destroy. + + ~ Update + ~ pipeline.PL_ExecSparkJob + + - Delete + - pipeline.PL_Obsolete + + = Unchanged / skipped + = dataset.DS_Json +``` + +The returned object contains a `DryRunPlan` property, so CI can enforce policy checks before deployment: + +```powershell +$result = Publish-AdfV2FromJson ` + -RootFolder $RootFolder ` + -ResourceGroupName $ResourceGroupName ` + -DataFactoryName $DataFactoryName ` + -Location $Location ` + -Option $opt ` + -Plan + +if ($result.DryRunPlan.Delete.Count -gt 0) { + Write-Error "Plan contains destroy actions. Review required before deployment." + exit 1 +} + +Write-Host "Plan accepted. Add=$($result.DryRunPlan.Create.Count); Change=$($result.DryRunPlan.Update.Count); Destroy=$($result.DryRunPlan.Delete.Count)" +``` + -## Step: Stoping triggers +## Step: Stopping triggers 💬 In log you'll see line: `STEP: Stopping triggers...` diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index f99e0db..287169e 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -12,7 +12,7 @@ RootModule = 'azure.datafactory.tools.psm1' # Version number of this module. - ModuleVersion = '1.16.0' + ModuleVersion = '1.18.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/changelog.md b/changelog.md index 78210eb..1855d2d 100644 --- a/changelog.md +++ b/changelog.md @@ -4,10 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [1.16.0] - 2026-06-06 +## [1.18.0] - 2026-07-01 +### Added +* `Publish-AdfV2FromJson` now supports terraform-like planning via `-DryRun`/`-Plan`, including structured `DryRunPlan` output (Create/Update/Delete/Unchanged) #360 +## [1.17.1] - 2026-07-01 ### Fixed +* Further fixes of `DeleteNotInSource` with deserialization issue #480 +* `Publish-AdfV2FromJson` no longer silently continues after errors: config path errors (ADFT0010) and deployment failures (e.g. Azure Policy blocks) now propagate as terminating errors to the caller #472 +* Scheduled trigger recurrence times now preserve explicit timezone offsets during config-driven updates, preventing shifted deployment times #402 +## [1.16.0] - 2026-06-06 +### Fixed +* `DeleteNotInSource` no longer fails with "Unable to deserialize the response" when ADF contains object types unsupported by the installed Az.DataFactory module version - REST API fallback added to `Get-AdfFromService` #480 #473 * Fixed ClientAssertionCredential authentication failure for federated identity (OIDC) credentials by removing redundant `Get-AzAccessToken` calls in `Get-AzDFV2Credential` and `Remove-AdfObjectRestAPI` #492 ## [1.15.0] - 2026-05-26 @@ -16,7 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed * Starting blob event trigger fails with 'Resource cannot be updated during provisioning' - now waits for provisioning to complete before retrying #484, #474, #463 * Fixed REST API calls failing with 401 Unauthorized due to Az.Accounts 5.x returning SecureString from Get-AzAccessToken -* Fixed DryRun not loading deployment state from storage for hash comparison #476 +* Fixed DryRun not loading deployment state from storage for hash comparison #475 * README.md updated and new Structured Documentation created ## [1.14.0] - 2025-10-24 diff --git a/docs/ADVANCED/CMDLET_REFERENCE.md b/docs/ADVANCED/CMDLET_REFERENCE.md index 431ec2a..e334681 100644 --- a/docs/ADVANCED/CMDLET_REFERENCE.md +++ b/docs/ADVANCED/CMDLET_REFERENCE.md @@ -18,7 +18,8 @@ Publish-AdfV2FromJson ` [-Stage ] ` [-Option ] ` [-Method ] ` - [-DryRun] + [-DryRun] ` + [-Plan] ``` **Parameters:** @@ -33,6 +34,7 @@ Publish-AdfV2FromJson ` | Option | AdfPublishOption | No | Publish options object (filtering, triggers, etc.) | | Method | String | No | Publishing method: 'AzDataFactory' or 'AzResource' (default) | | DryRun | Switch | No | Validate without deploying | +| Plan | Switch | No | Alias behavior of DryRun with terraform-like naming | **Returns:** AdfPublishOption (deployment result) diff --git a/docs/GUIDE/PUBLISHING.md b/docs/GUIDE/PUBLISHING.md index d3ef583..68a4670 100644 --- a/docs/GUIDE/PUBLISHING.md +++ b/docs/GUIDE/PUBLISHING.md @@ -33,9 +33,12 @@ Publish-AdfV2FromJson ` [-Stage] ` [-Option] ` [-Method] ` - [-DryRun] + [-DryRun] ` + [-Plan] ``` +`-Plan` is an alias behavior of `-DryRun` and prints a terraform-like plan without deploying. + ### Complete Example ```powershell diff --git a/docs/GUIDE/PUBLISH_OPTIONS.md b/docs/GUIDE/PUBLISH_OPTIONS.md index c9b474b..412b14b 100644 --- a/docs/GUIDE/PUBLISH_OPTIONS.md +++ b/docs/GUIDE/PUBLISH_OPTIONS.md @@ -67,6 +67,11 @@ $opt = New-AdfPublishOption - **Default:** true - **Purpose:** Fail deployment if config path doesn't exist (vs. warning only) +#### IgnoreLackOfReferencedObject +- **Type:** Boolean +- **Default:** false +- **Purpose:** Continue deployment when a referenced object is missing and emit a warning instead of failing + ### Trigger Control Options #### StopStartTriggers (see also: TriggerStopMethod, TriggerStartMethod) @@ -155,7 +160,15 @@ $opt.FailsWhenConfigItemNotFound = $false # Warnings will be printed instead of failing ``` -### Example 6: Incremental Deployment +### Example 6: Ignore Missing Referenced Objects + +```powershell +$opt = New-AdfPublishOption +$opt.IgnoreLackOfReferencedObject = $true +# Referenced objects that are not present will warn instead of failing +``` + +### Example 7: Incremental Deployment ```powershell $opt = New-AdfPublishOption @@ -163,7 +176,7 @@ $opt.IncrementalDeployment = $true $opt.IncrementalDeploymentStorageUri = 'https://storageaccount.file.core.windows.net/adftools' ``` -### Example 7: Selective Deployment with Smart Triggers +### Example 8: Selective Deployment with Smart Triggers ```powershell $opt = New-AdfPublishOption diff --git a/private/AdfPSObjects.class.ps1 b/private/AdfPSObjects.class.ps1 new file mode 100644 index 0000000..f2ca37c --- /dev/null +++ b/private/AdfPSObjects.class.ps1 @@ -0,0 +1,40 @@ + +# Lightweight wrapper classes for ADF objects retrieved via REST API. +# Class names follow the AdfPS pattern so that Get-SimplifiedType +# strips the "AdfPS" prefix and returns the correct simplified type (e.g. "Dataset"). +# These are used as a fallback when Get-AzDataFactoryV2* cmdlets fail to deserialize +# objects whose types are not yet known to the installed Az.DataFactory module version. + +class AdfPSDataset { + [String] $Name + AdfPSDataset([String] $name) { $this.Name = $name } +} + +class AdfPSDataFlow { + [String] $Name + AdfPSDataFlow([String] $name) { $this.Name = $name } +} + +class AdfPSPipeline { + [String] $Name + AdfPSPipeline([String] $name) { $this.Name = $name } +} + +class AdfPSLinkedService { + [String] $Name + AdfPSLinkedService([String] $name) { $this.Name = $name } +} + +class AdfPSIntegrationRuntime { + [String] $Name + AdfPSIntegrationRuntime([String] $name) { $this.Name = $name } +} + +class AdfPSTrigger { + [String] $Name + [String] $RuntimeState + AdfPSTrigger([String] $name, [String] $runtimeState) { + $this.Name = $name + $this.RuntimeState = $runtimeState + } +} diff --git a/private/ApplyExclusionOptions.ps1 b/private/ApplyExclusionOptions.ps1 index 4042a3c..be91f8b 100644 --- a/private/ApplyExclusionOptions.ps1 +++ b/private/ApplyExclusionOptions.ps1 @@ -49,7 +49,9 @@ function ApplyExclusionOptions { function ToBeDeployedStat { param( - [Parameter(Mandatory=$True)] [Adf] $adf + [Parameter(Mandatory=$True)] [Adf] $adf, + [Parameter(Mandatory=$False)] $targetAdfInstance, + [switch] $TerraformStyle ) $ToBeDeployedList = ($adf.AllObjects() | Where-Object { $_.ToBeDeployed -eq $true } | ToArray) @@ -59,4 +61,166 @@ function ToBeDeployedStat { Write-Host "- $($_.FullName($true))" } + if (!$TerraformStyle) { + return $null + } + + $plan = Get-DryRunPlan -adf $adf -targetAdfInstance $targetAdfInstance + Write-DryRunPlan -plan $plan + return $plan + +} + +function ConvertTo-CanonicalAdfType { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] + [String] $Type + ) + + $t = $Type.ToLowerInvariant() + switch ($t) + { + 'pipeline' { return 'pipeline' } + 'dataset' { return 'dataset' } + 'dataflow' { return 'dataflow' } + 'linkedservice' { return 'linkedService' } + 'integrationruntime' { return 'integrationRuntime' } + 'trigger' { return 'trigger' } + 'factory' { return 'factory' } + 'managedvirtualnetwork' { return 'managedVirtualNetwork' } + 'managedprivateendpoint' { return 'managedPrivateEndpoint' } + 'credential' { return 'credential' } + default { return $Type } + } +} + +function Get-AdfObjectFullName { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] + $Object + ) + + $name = $Object.Name + $simtype = '' + + if ($Object -is [AdfObject]) { + $simtype = Get-SimplifiedType -Type $Object.Type + if ($Object.Type -like 'Microsoft.DataFactory/factories*') { + $simtype = ConvertTo-AdfType -AzType $Object.Type + $simtype = Get-SimplifiedType -Type $simtype + } + } + else { + $simtype = Get-SimplifiedType -Type $Object.GetType().Name + } + + $simtype = ConvertTo-CanonicalAdfType -Type $simtype + return "$simtype.$name" +} + +function Get-DryRunPlan { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] [Adf] $adf, + [parameter(Mandatory = $false)] $targetAdfInstance + ) + + $create = [System.Collections.Generic.List[string]]::new() + $update = [System.Collections.Generic.List[string]]::new() + $delete = [System.Collections.Generic.List[string]]::new() + $unchanged = [System.Collections.Generic.List[string]]::new() + + $sourceByName = @{} + $adf.AllObjects() | ForEach-Object { + $fullName = Get-AdfObjectFullName -Object $_ + $sourceByName[$fullName] = $_ + } + + $targetByName = @{} + if ($null -ne $targetAdfInstance) { + $targetAdfInstance.AllObjects() | ForEach-Object { + $fullName = Get-AdfObjectFullName -Object $_ + $targetByName[$fullName] = $_ + } + } + + $adf.AllObjects() | ForEach-Object { + $fullName = Get-AdfObjectFullName -Object $_ + if ($_.ToBeDeployed -eq $false) { + $unchanged.Add($fullName) | Out-Null + } + elseif ($targetByName.ContainsKey($fullName)) { + $update.Add($fullName) | Out-Null + } + else { + $create.Add($fullName) | Out-Null + } + } + + if ($null -ne $targetAdfInstance -and $adf.PublishOptions.DeleteNotInSource) { + $targetByName.Keys | ForEach-Object { + $fullName = $_ + if (!$sourceByName.ContainsKey($fullName)) { + $oname = [AdfObjectName]::new($fullName) + $canDelete = $true + if ($adf.PublishOptions.DoNotDeleteExcludedObjects) { + $canDelete = !($oname.IsNameExcluded($adf.PublishOptions)) + } + if ($canDelete) { + $delete.Add($fullName) | Out-Null + } + } + } + } + + $create = @($create | Sort-Object) + $update = @($update | Sort-Object) + $delete = @($delete | Sort-Object) + $unchanged = @($unchanged | Sort-Object) + + return [PSCustomObject]@{ + Create = @($create) + Update = @($update) + Delete = @($delete) + Unchanged = @($unchanged) + } +} + +function Write-DryRunPlan { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] + $plan + ) + + Write-Host "===================================================================================" + Write-Host "Terraform-like plan (DryRun):" + Write-Host "Plan: $($plan.Create.Count) to add, $($plan.Update.Count) to change, $($plan.Delete.Count) to destroy." + + if ($plan.Create.Count -gt 0) { + Write-Host "" + Write-Host " + Create" + $plan.Create | ForEach-Object { Write-Host " + $_" } + } + + if ($plan.Update.Count -gt 0) { + Write-Host "" + Write-Host " ~ Update" + $plan.Update | ForEach-Object { Write-Host " ~ $_" } + } + + if ($plan.Delete.Count -gt 0) { + Write-Host "" + Write-Host " - Delete" + $plan.Delete | ForEach-Object { Write-Host " - $_" } + } + + if ($plan.Unchanged.Count -gt 0) { + Write-Host "" + Write-Host " = Unchanged / skipped" + $plan.Unchanged | ForEach-Object { Write-Host " = $_" } + } + } \ No newline at end of file diff --git a/private/Deploy-AdfObject.ps1 b/private/Deploy-AdfObject.ps1 index 9b67c54..d6b0ce4 100644 --- a/private/Deploy-AdfObject.ps1 +++ b/private/Deploy-AdfObject.ps1 @@ -31,7 +31,7 @@ function Deploy-AdfObject { if ($adf.PublishOptions.IgnoreLackOfReferencedObject -eq $true) { Write-Warning "ADFT0006: Referenced object [$type].[$name] was not found. No error raised as user wanted to carry on." } else { - Write-Error "ADFT0005: Referenced object [$type].[$name] was not found." + throw "ADFT0005: Referenced object [$type].[$name] was not found." } } elseif ($type -notin [AdfObject]::IgnoreTypes) { Deploy-AdfObject -obj $depobj diff --git a/private/Deploy-AdfObjectOnly.ps1 b/private/Deploy-AdfObjectOnly.ps1 index 1398035..0d9af2e 100644 --- a/private/Deploy-AdfObjectOnly.ps1 +++ b/private/Deploy-AdfObjectOnly.ps1 @@ -73,7 +73,7 @@ function Deploy-AdfObjectOnly { -Force | Out-Null } else { - Write-Error "ADFT0012: Deployment for this kind of Integration Runtime is not supported yet." + throw "ADFT0012: Deployment for this kind of Integration Runtime is not supported yet." } } 'linkedService' @@ -155,7 +155,7 @@ function Deploy-AdfObjectOnly { } default { - Write-Error "ADFT0013: Type $($obj.Type) is not supported." + throw "ADFT0013: Type $($obj.Type) is not supported." } } diff --git a/private/Get-AdfObjectsFromServiceRestAPI.ps1 b/private/Get-AdfObjectsFromServiceRestAPI.ps1 new file mode 100644 index 0000000..e27f563 --- /dev/null +++ b/private/Get-AdfObjectsFromServiceRestAPI.ps1 @@ -0,0 +1,41 @@ +function Get-AdfObjectsFromServiceRestAPI { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] $adfi, + [parameter(Mandatory = $true)] [String] $typePlural, + [parameter(Mandatory = $true)] [String] $simpleType + ) + + Write-Debug "BEGIN: Get-AdfObjectsFromServiceRestAPI(typePlural=$typePlural, simpleType=$simpleType)" + + $url = "$($script:BaseApiUrl)$($adfi.DataFactoryId)/$typePlural`?api-version=2018-06-01" + + [System.Collections.ArrayList] $items = @() + do { + $r = Invoke-AzRestMethod -Method 'GET' -Uri $url + if ($r.StatusCode -ne 200) { + Write-Error -Message "Unexpected response code: $($r.StatusCode) from REST API when listing $typePlural." + return $null + } + $content = $r.Content | ConvertFrom-Json + $url = if ($content.PSObject.Properties['nextLink']) { $content.nextLink } else { $null } + $content.value | ForEach-Object { + $item = $_ + $name = $item.name + $obj = $null + switch ($simpleType) { + 'Dataset' { $obj = [AdfPSDataset]::New($name) } + 'DataFlow' { $obj = [AdfPSDataFlow]::New($name) } + 'Pipeline' { $obj = [AdfPSPipeline]::New($name) } + 'LinkedService' { $obj = [AdfPSLinkedService]::New($name) } + 'IntegrationRuntime' { $obj = [AdfPSIntegrationRuntime]::New($name) } + 'Trigger' { $obj = [AdfPSTrigger]::New($name, [string]$item.properties.runtimeState) } + default { Write-Warning "Get-AdfObjectsFromServiceRestAPI: unsupported type '$simpleType'" } + } + if ($null -ne $obj) { $items.Add($obj) | Out-Null } + } + } while ($url) + + Write-Debug "END: Get-AdfObjectsFromServiceRestAPI()" + return $items +} diff --git a/private/ObjectProperty.ps1 b/private/ObjectProperty.ps1 index 2c7996f..010b95d 100644 --- a/private/ObjectProperty.ps1 +++ b/private/ObjectProperty.ps1 @@ -25,9 +25,14 @@ function Update-ObjectProperty { $exp = "`$obj.$path = `$$value" } elseif ($fieldType -eq [DateTime]) { Write-Debug "Setting as DateTime value" - $datevalue = [DateTime]$value - $utcvalue = Get-Date $datevalue -Format "yyyy-MM-ddTHH:mm:ss.fffZ" - $exp = "`$obj.$path = `"$utcvalue`"" + # Keep explicit timezone suffixes unchanged to avoid timezone shifts (Issue #402). + if ($value -match '(?i)(Z|[+-][0-9]{2}:[0-9]{2})$') { + $exp = "`$obj.$path = `"$value`"" + } else { + $date_value = [DateTime]$value + $utc_value = Get-Date $date_value -Format "yyyy-MM-ddTHH:mm:ss.fffZ" + $exp = "`$obj.$path = `"$utc_value`"" + } } elseif ($fieldType -eq [System.Management.Automation.PSCustomObject]) { Write-Debug "Setting as json value" $jvalue = ConvertFrom-Json $value diff --git a/private/Update-PropertiesFromFile.ps1 b/private/Update-PropertiesFromFile.ps1 index 542675a..f62f900 100644 --- a/private/Update-PropertiesFromFile.ps1 +++ b/private/Update-PropertiesFromFile.ps1 @@ -72,7 +72,7 @@ function Update-PropertiesFromFile { if ($option.FailsWhenConfigItemNotFound -eq $false) { Write-Warning "Could not find object: $type.$name, skipping..." } else { - Write-Error -Message "ADFT0007: Could not find object: $type.$name" + throw "ADFT0007: Could not find object: $type.$name" } } else { Write-Verbose "- Performing: $action for object(path): $type.$name(properties.$path)" @@ -131,7 +131,7 @@ function Update-PropertiesForObject { Write-Warning "Wrong path defined in config for object(path): $type.$name(properties.$path), skipping..." } else { $exc = ([System.Data.DataException]::new("ADFT0010: Wrong path defined in config for object(path): $type.$name(properties.$path)")) - Write-Error -Exception $exc + throw $exc } } diff --git a/public/Get-AdfFromService.ps1 b/public/Get-AdfFromService.ps1 index 40476c9..0e577a9 100644 --- a/public/Get-AdfFromService.ps1 +++ b/public/Get-AdfFromService.ps1 @@ -37,17 +37,47 @@ function Get-AdfFromService { $adf.Id = $adfi.DataFactoryId $adf.Location = $adfi.Location - $adf.DataSets = Get-AzDataFactoryV2Dataset -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.DataSets = Get-AzDataFactoryV2Dataset -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray + } catch { + Write-Warning "Failed to list Datasets via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.DataSets = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'datasets' -simpleType 'Dataset' | ToArray + } Write-Host ("DataSets: {0} object(s) loaded." -f $adf.DataSets.Count) - $adf.IntegrationRuntimes = Get-AzDataFactoryV2IntegrationRuntime -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.IntegrationRuntimes = Get-AzDataFactoryV2IntegrationRuntime -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray + } catch { + Write-Warning "Failed to list IntegrationRuntimes via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.IntegrationRuntimes = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'integrationruntimes' -simpleType 'IntegrationRuntime' | ToArray + } Write-Host ("IntegrationRuntimes: {0} object(s) loaded." -f $adf.IntegrationRuntimes.Count) - $adf.LinkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.LinkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray + } catch { + Write-Warning "Failed to list LinkedServices via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.LinkedServices = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'linkedservices' -simpleType 'LinkedService' | ToArray + } Write-Host ("LinkedServices: {0} object(s) loaded." -f $adf.LinkedServices.Count) - $adf.Pipelines = Get-AzDataFactoryV2Pipeline -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.Pipelines = Get-AzDataFactoryV2Pipeline -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray + } catch { + Write-Warning "Failed to list Pipelines via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.Pipelines = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'pipelines' -simpleType 'Pipeline' | ToArray + } Write-Host ("Pipelines: {0} object(s) loaded." -f $adf.Pipelines.Count) - $adf.DataFlows = Get-AzDataFactoryV2DataFlow -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.DataFlows = Get-AzDataFactoryV2DataFlow -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray + } catch { + Write-Warning "Failed to list DataFlows via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.DataFlows = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'dataflows' -simpleType 'DataFlow' | ToArray + } Write-Host ("DataFlows: {0} object(s) loaded." -f $adf.DataFlows.Count) - $adf.Triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" | ToArray + try { + $adf.Triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$FactoryName" -ErrorAction Stop | ToArray + } catch { + Write-Warning "Failed to list Triggers via Az cmdlet (possible deserialization issue). Falling back to REST API. Error: $_" + $adf.Triggers = Get-AdfObjectsFromServiceRestAPI -adfi $adfi -typePlural 'triggers' -simpleType 'Trigger' | ToArray + } Write-Host ("Triggers: {0} object(s) loaded." -f $adf.Triggers.Count) $adf.Credentials = Get-AzDFV2Credential -adfi $adfi | ToArray Write-Host ("Credentials: {0} object(s) loaded." -f $adf.Credentials.Count) diff --git a/public/Publish-AdfV2FromJson.ps1 b/public/Publish-AdfV2FromJson.ps1 index 0f079e0..410ac85 100644 --- a/public/Publish-AdfV2FromJson.ps1 +++ b/public/Publish-AdfV2FromJson.ps1 @@ -35,6 +35,9 @@ AzResource method has been introduced due to bugs in Az.DataFactory PS module. Optional switch parameter. When provided, process will not make any changes to target data factory but instead return the ADF object that would be used in deployment. +.PARAMETER Plan +Optional switch parameter. Alias for DryRun with terraform-style intent. Behaves the same as DryRun. + .EXAMPLE # Publish entire ADF $ResourceGroupName = 'rg-devops-factory' @@ -74,6 +77,10 @@ Publish-AdfV2FromJson -RootFolder "$RootFolder" -ResourceGroupName "$ResourceGro # Execute dry run of intended publishing changes Publish-AdfV2FromJson -RootFolder "$RootFolder" -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$DataFactoryName" -Location "$Location" -Stage "UAT" -DryRun +.EXAMPLE +# Execute plan (alias of DryRun) to preview intended changes +Publish-AdfV2FromJson -RootFolder "$RootFolder" -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$DataFactoryName" -Location "$Location" -Stage "UAT" -Plan + .LINK Online version: https://github.com/SQLPlayer/azure.datafactory.tools/ #> @@ -104,9 +111,16 @@ function Publish-AdfV2FromJson { [String]$Method = 'AzResource', [parameter(Mandatory = $false)] - [switch]$DryRun + [switch]$DryRun, + + [parameter(Mandatory = $false)] + [switch]$Plan ) + if ($Plan.IsPresent) { + $DryRun = $true + } + $m = Get-Module -Name "azure.datafactory.tools" $verStr = $m.Version.ToString(2) + "." + $m.Version.Build.ToString("000"); Write-Host "======================================================================================"; @@ -123,6 +137,7 @@ function Publish-AdfV2FromJson { Write-Host "Options provided: $($null -ne $Option)"; Write-Host "Publishing method: $Method"; Write-Host "Is Dry Run?: $($DryRun.IsPresent)"; + Write-Host "Is Plan Mode?: $($Plan.IsPresent)"; Write-Host "======================================================================================"; if ($null -ne $Option) { Write-Host "Options:" @@ -248,10 +263,29 @@ function Publish-AdfV2FromJson { } Write-Host "Found $unchanged_count unchanged object(s)." } - ToBeDeployedStat -adf $adf + + $dryRunPlan = $null + if ($DryRun.IsPresent) { + $targetAdfInstance = $null + try { + Write-Host "DRY RUN: Loading target ADF objects to classify plan changes..." + $targetAdfInstance = Get-AdfFromService -FactoryName "$DataFactoryName" -ResourceGroupName "$ResourceGroupName" + } + catch { + Write-Warning "DRY RUN: Unable to load target ADF state for full plan classification. Falling back to local-only plan. Details: $_" + } + + $dryRunPlan = ToBeDeployedStat -adf $adf -targetAdfInstance $targetAdfInstance -TerraformStyle + } + else { + ToBeDeployedStat -adf $adf + } if ($DryRun.IsPresent) { Write-Host "DRY RUN: Terminating script pre-deployment - inspect returned object to review planned changes." + if ($null -ne $dryRunPlan) { + $adf | Add-Member -MemberType NoteProperty -Name DryRunPlan -Value $dryRunPlan -Force + } Write-Host "==================================================================================="; return $adf } @@ -272,16 +306,16 @@ function Publish-AdfV2FromJson { $adf.Factories[0].ToBeDeployed = $false } } - $adf.AllObjects() | ForEach-Object { - Deploy-AdfObject -obj $_ + foreach ($obj in $adf.AllObjects()) { + Deploy-AdfObject -obj $obj } Write-Host "==================================================================================="; Write-Host "STEP: Deleting objects not in source ..." if ($opt.DeleteNotInSource -eq $true) { $adfIns = Get-AdfFromService -FactoryName "$DataFactoryName" -ResourceGroupName "$ResourceGroupName" - $adfIns.AllObjects() | ForEach-Object { - Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $_ -adfInstance $adfIns + foreach ($targetObj in $adfIns.AllObjects()) { + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $targetObj -adfInstance $adfIns } Write-Host "Deleted $($adf.DeletedObjectNames.Count) objects from ADF service." } else { diff --git a/test/!RunAllTests.ps1 b/test/!RunAllTests.ps1 index ba01f7c..1c4922d 100644 --- a/test/!RunAllTests.ps1 +++ b/test/!RunAllTests.ps1 @@ -51,6 +51,7 @@ if ($InstallModules) { #Install-Module 'Az.DataFactory' -Force -MinimumVersion 1.16.0 -Repository 'PSGallery' #Install-Module 'PSScriptAnalyzer' -Force #Install-Module 'Pester' -Force -MinimumVersion 5.1.1 + Install-Module 'Az.Storage' -Force -AllowClobber -Repository 'PSGallery' [Environment]::SetEnvironmentVariable("azure.datafactory.tools.unitTestInstalledModules", $true, 'Process'); } } else { @@ -58,6 +59,7 @@ if ($InstallModules) { } Import-Module 'Pester' -MinimumVersion 5.3.3 Import-Module 'PSScriptAnalyzer' +Import-Module 'Az.Storage' Import-Module "$folder\azure.datafactory.tools.psd1" Write-Host "=============== Modules ================" @@ -96,4 +98,5 @@ try { Pop-Location } -# . ".\test\!RunAllTests.ps1" -folder '.\test' -TestFilenameFilter '*' -MajorRelease:$true -InstallModules:$false +# Connect-AzAccount +# . ".\test\!RunAllTests.ps1" -folder '.\test' -TestFilenameFilter 'Inc*' -MajorRelease:$true -InstallModules:$false diff --git a/test/Incremental-Deployment.Tests.ps1 b/test/Incremental-Deployment.Tests.ps1 index 5382161..d5824c2 100644 --- a/test/Incremental-Deployment.Tests.ps1 +++ b/test/Incremental-Deployment.Tests.ps1 @@ -12,6 +12,9 @@ InModuleScope azure.datafactory.tools { $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' Import-Module -Name $testHelperPath -Force + $m = Get-Module -Name 'azure.datafactory.tools' + $script:verStr = $m.Version.ToString(2) + "." + $m.Version.Build.ToString("000") + # $VerbosePreference = 'Continue' # $DebugPreference = 'Continue' @@ -29,7 +32,7 @@ InModuleScope azure.datafactory.tools { $opt.IncrementalDeployment = $true $opt.StopStartTriggers = $false $script:gp = "" - $script:dstate = [AdfDeploymentState]::new($verStr) + $script:dstate = [AdfDeploymentState]::new($script:verStr) $script:dstate.LastUpdate = [System.DateTime]::UtcNow $script:dstateJson = $script:dstate | ConvertTo-Json $script:StorageUri= "https://sqlplayer2020.blob.core.windows.net" diff --git a/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 b/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 new file mode 100644 index 0000000..70514ae --- /dev/null +++ b/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 @@ -0,0 +1,117 @@ +BeforeDiscovery { + $ModuleRootPath = $PSScriptRoot | Split-Path -Parent + $moduleManifestName = 'azure.datafactory.tools.psd1' + $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName + + Import-Module -Name $moduleManifestPath -Force -Verbose:$false + $m = Get-Module -Name 'azure.datafactory.tools' + $script:verStr = $m.Version.ToString(2) + "." + $m.Version.Build.ToString("000") +} + +InModuleScope azure.datafactory.tools { + $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' + Import-Module -Name $testHelperPath -Force + + $m = Get-Module -Name 'azure.datafactory.tools' + $script:verStr = $m.Version.ToString(2) + "." + $m.Version.Build.ToString("000") + + $script:RootFolder = Join-Path $PSScriptRoot 'adf2' + $script:ResourceGroupName = 'rg-test' + $script:DataFactoryName = 'adf-test' + $script:Location = 'West Europe' + + Describe 'Publish-AdfV2FromJson DryRun incremental deployment' -Tag 'Unit' { + BeforeEach { + Mock Get-StateFromStorage { + [AdfDeploymentState]::new($script:verStr) + } + + Mock Set-StateToStorage { + } + + Mock Get-AdfFromService { + $null + } + } + + It 'should still load deployment state from storage when DryRun is enabled' { + $opt = New-AdfPublishOption + $opt.IncrementalDeployment = $true + $opt.IncrementalDeploymentStorageUri = 'https://example.blob.core.windows.net/adftools/folder' + $opt.StopStartTriggers = $false + $opt.DeleteNotInSource = $false + + { + Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName $script:ResourceGroupName ` + -DataFactoryName $script:DataFactoryName ` + -Location $script:Location ` + -Option $opt ` + -DryRun + } | Should -Not -Throw + + Should -Invoke -CommandName Get-StateFromStorage -Times 1 -Exactly + Should -Invoke -CommandName Set-StateToStorage -Times 0 -Exactly + } + } + + Describe 'DryRun terraform-like plan output' -Tag 'Unit' { + It 'should classify add, change, destroy and expose DryRunPlan on returned object' { + $opt = New-AdfPublishOption + $opt.StopStartTriggers = $false + $opt.DeleteNotInSource = $true + $opt.DoNotDeleteExcludedObjects = $false + $opt.Includes.Add('pipeline.PL_ExecSparkJob', '') + + Mock Get-AdfFromService { + $target = New-Object -TypeName AdfInstance + $target.Pipelines = @( + [AdfPSPipeline]::new('PL_ExecSparkJob'), + [AdfPSPipeline]::new('PL_Obsolete') + ) + $target + } + + $result = Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName $script:ResourceGroupName ` + -DataFactoryName $script:DataFactoryName ` + -Location $script:Location ` + -Option $opt ` + -DryRun + + $result | Should -Not -BeNullOrEmpty + $result.PSObject.Properties.Name | Should -Contain 'DryRunPlan' + + $plan = $result.DryRunPlan + $plan | Should -Not -BeNullOrEmpty + $plan.Update | Should -Contain 'pipeline.PL_ExecSparkJob' + $plan.Delete | Should -Contain 'pipeline.PL_Obsolete' + $plan.Unchanged | Should -Contain 'dataset.DS_Json' + $plan.Create.Count | Should -Be 0 + } + + It 'should support -Plan as an alias behavior of -DryRun' { + $opt = New-AdfPublishOption + $opt.StopStartTriggers = $false + + Mock Get-AdfFromService { + $target = New-Object -TypeName AdfInstance + $target + } + + $result = Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName $script:ResourceGroupName ` + -DataFactoryName $script:DataFactoryName ` + -Location $script:Location ` + -Option $opt ` + -Plan + + $result | Should -Not -BeNullOrEmpty + $result.PSObject.Properties.Name | Should -Contain 'DryRunPlan' + $result.DryRunPlan | Should -Not -BeNullOrEmpty + } + } +} \ No newline at end of file diff --git a/test/Publish-AdfV2FromJson-ErrorHandling.Tests.ps1 b/test/Publish-AdfV2FromJson-ErrorHandling.Tests.ps1 new file mode 100644 index 0000000..9a4a3a2 --- /dev/null +++ b/test/Publish-AdfV2FromJson-ErrorHandling.Tests.ps1 @@ -0,0 +1,141 @@ +BeforeDiscovery { + $ModuleRootPath = $PSScriptRoot | Split-Path -Parent + $moduleManifestName = 'azure.datafactory.tools.psd1' + $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName + Import-Module -Name $moduleManifestPath -Force -Verbose:$false +} + +InModuleScope azure.datafactory.tools { + $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' + Import-Module -Name $testHelperPath -Force + + $script:SrcFolder = "$PSScriptRoot\BigFactorySample2" + $script:TmpFolder = (New-TemporaryDirectory).FullName + $script:RootFolder = Join-Path -Path $script:TmpFolder -ChildPath (Split-Path -Path $script:SrcFolder -Leaf) + Copy-Item -Path "$SrcFolder" -Destination "$TmpFolder" -Filter "*.*" -Recurse:$true -Force + + AfterAll { + Remove-Item -Path $script:TmpFolder -Recurse -Force -ErrorAction SilentlyContinue + } + + # ------------------------------------------------------------------------- + # Issue #472 – Scenario 1: wrong property path in config (ADFT0010) + # + # config-c004-wrongpath.csv contains "typeProperties:baseUrl" (colon + # separator instead of dot), which causes Write-Error "ADFT0010" inside + # Update-PropertiesForObject. The error is non-terminating, so the + # ForEach-Object loop in Update-PropertiesFromFile swallows it and + # Publish-AdfV2FromJson continues to print "deployed successfully". + # + # Expected fix: the error must propagate as a terminating error so the + # caller observes a failure regardless of $ErrorActionPreference scope. + # ------------------------------------------------------------------------- + Describe 'Publish-AdfV2FromJson - Error propagation for wrong config path (Issue #472)' -Tag 'Unit' { + + Context 'When stage config has a wrong property path and FailsWhenPathNotFound is true (default)' { + + It 'Should throw when Publish-AdfV2FromJson is called with -DryRun' { + # BUG REPRODUCTION: currently does NOT throw (Write-Error is swallowed). + # After fix this test must pass. + $o = New-AdfPublishOption + # FailsWhenPathNotFound defaults to $true + { + Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName 'rg-test' ` + -DataFactoryName 'adf-test' ` + -Stage 'c004-wrongpath' ` + -Option $o ` + -DryRun + } | Should -Throw + } + } + + Context 'When stage config has a wrong property path and FailsWhenPathNotFound is false' { + + It 'Should not throw (wrong path is only a warning)' { + $o = New-AdfPublishOption + $o.FailsWhenPathNotFound = $false + { + Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName 'rg-test' ` + -DataFactoryName 'adf-test' ` + -Stage 'c004-wrongpath' ` + -Option $o ` + -DryRun + } | Should -Not -Throw + } + } + } + + # ------------------------------------------------------------------------- + # Issue #472 – Scenario 2: ADF object deployment fails at runtime + # + # When Deploy-AdfObjectOnly emits a non-terminating Write-Error (e.g. an + # Azure Policy block: "RequestDisallowedByPolicy"), the ForEach-Object + # pipeline in Publish-AdfV2FromJson absorbs the error and the function + # continues to report "deployed successfully". + # + # Expected fix: the deployment loop must propagate errors so the function + # terminates and the caller sees a failure. + # ------------------------------------------------------------------------- + Describe 'Publish-AdfV2FromJson - Error propagation for deployment failure (Issue #472)' -Tag 'Unit' { + + Context 'When an ADF object deployment produces a terminating error' { + + BeforeAll { + Mock Get-AzDataFactoryV2 { + [PSCustomObject]@{ Name = 'adf-test'; ResourceGroupName = 'rg-test' } + } + # Azure cmdlets (e.g. New-AzResource) throw terminating errors for + # serious failures like Azure Policy blocks. In PowerShell 5.1 the + # ForEach-Object pipeline was absorbing these terminating errors and + # allowing the loop to continue, so the function reported success. + # The fix replaces ForEach-Object with a foreach statement, which + # propagates terminating errors correctly in all PS versions. + Mock Deploy-AdfObjectOnly { + throw 'Simulated: RequestDisallowedByPolicy - resource was disallowed by policy' + } + } + + It 'Should throw when any object deployment fails' { + # BUG REPRODUCTION: previously did NOT throw because ForEach-Object + # absorbed the terminating error in PS 5.1. + # After fix (foreach loop) this test must pass. + $o = New-AdfPublishOption + $o.StopStartTriggers = $false + $o.DeleteNotInSource = $false + { + Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName 'rg-test' ` + -DataFactoryName 'adf-test' ` + -Option $o + } | Should -Throw + } + + It 'Should throw even when the caller has not set $ErrorActionPreference to Stop' { + # BUG REPRODUCTION: the module scope isolates $ErrorActionPreference + # from the caller's script scope, so setting it externally has no effect. + # The fix must make the function fail independently of the caller's EAP. + $o = New-AdfPublishOption + $o.StopStartTriggers = $false + $o.DeleteNotInSource = $false + $threw = $false + # Deliberately NOT using Should -Throw so Pester does not alter EAP + try { + Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName 'rg-test' ` + -DataFactoryName 'adf-test' ` + -Option $o + } + catch { + $threw = $true + } + $threw | Should -BeTrue -Because 'deployment failures must propagate to the caller' + } + } + } +} diff --git a/test/Update-PropertiesFromFile.Tests.ps1 b/test/Update-PropertiesFromFile.Tests.ps1 index 7701999..8ebea0c 100644 --- a/test/Update-PropertiesFromFile.Tests.ps1 +++ b/test/Update-PropertiesFromFile.Tests.ps1 @@ -375,6 +375,33 @@ InModuleScope azure.datafactory.tools { } + Describe 'Update-PropertiesFromFile - Issue #402' -Tag 'Unit','private','issue402' { + Context 'When trigger recurrence timestamps contain explicit timezone offsets' { + BeforeAll { + $script:adf = Import-AdfFromFolder -FactoryName "xyz" -RootFolder "$RootFolder" + $script:adf.PublishOptions = New-AdfPublishOption + + $script:configFilePath = Join-Path -Path $script:DeploymentFolder -ChildPath "config-issue-402-timezone.csv" + @( + 'type,name,path,value' + 'trigger,TR_RunEveryDay,"$.properties.typeProperties.recurrence.startTime","2020-06-01T23:22:11.000+10:00"' + 'trigger,TR_RunEveryDay,"$.properties.typeProperties.recurrence.endTime","2020-06-05T23:22:11.000+10:00"' + ) | Set-Content -Path $script:configFilePath -Encoding UTF8 + + Update-PropertiesFromFile -adf $script:adf -stage $script:configFilePath + $script:trigger = Get-AdfObjectByName -adf $script:adf -name "TR_RunEveryDay" -type "trigger" + } + + It 'Should keep recurrence.startTime offset unchanged' { + $script:trigger.Body.properties.typeProperties.recurrence.startTime | Should -Be "2020-06-01T23:22:11.000+10:00" + } + + It 'Should keep recurrence.endTime offset unchanged' { + $script:trigger.Body.properties.typeProperties.recurrence.endTime | Should -Be "2020-06-05T23:22:11.000+10:00" + } + } + } + Describe 'Publish-AdfV2FromJson with DryRun' -Tag 'Unit','private' { $cases = @( @{ configFile = 'config-endpoint.csv' }, @{ configFile = 'config-endpoint2.json' } ) diff --git a/test/zGet-AdfFromService.Tests.ps1 b/test/zGet-AdfFromService.Tests.ps1 index 82f7b8a..a919338 100644 --- a/test/zGet-AdfFromService.Tests.ps1 +++ b/test/zGet-AdfFromService.Tests.ps1 @@ -10,21 +10,353 @@ InModuleScope azure.datafactory.tools { $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' Import-Module -Name $testHelperPath -Force - # Variables for use in tests - $t = Get-TargetEnv 'adf2' - $script:adfName = $t.DataFactoryName - $script:rg = $t.ResourceGroupName - + # Shared factory stub + $script:fakeAdfi = [PSCustomObject]@{ + DataFactoryId = '/subscriptions/sub-123/resourceGroups/rg-test/providers/Microsoft.DataFactory/factories/adf-test' + Location = 'northeurope' + } + + # --------------------------------------------------------------------------- Describe 'Get-AdfFromService' -Tag 'Unit' { + It 'Should exist' { { Get-Command -Name Get-AdfFromService -ErrorAction Stop } | Should -Not -Throw } - } + Context 'When all Az cmdlets succeed (happy path)' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + Mock Get-AzDataFactoryV2Dataset { @() } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { @() } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { @() } + Mock Get-AzDFV2Credential { @() } + } + + It 'Should return an AdfInstance with the correct factory name' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.Name | Should -Be 'adf-test' + } + + It 'Should not call Invoke-AzRestMethod when Az cmdlets succeed' { + Mock Invoke-AzRestMethod { throw 'Should not be called' } + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + } + + Context 'When Get-AzDataFactoryV2Dataset throws a deserialization error (issue #480)' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + Mock Get-AzDataFactoryV2Dataset { throw 'Unable to deserialize the response.' } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { @() } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { @() } + Mock Get-AzDFV2Credential { @() } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ds_adls_csv","properties":{}},{"name":"ds_servicenow_v2","properties":{}}]}' + } + } + } + + It 'Should not throw' { + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + + It 'Should fall back to REST API and return all datasets by name' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.DataSets.Count | Should -Be 2 + $result.DataSets.Name | Should -Contain 'ds_adls_csv' + $result.DataSets.Name | Should -Contain 'ds_servicenow_v2' + } + + It 'Should return datasets as AdfPSDataset wrapper objects' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.DataSets | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSDataset' } + } + + It 'Should call Invoke-AzRestMethod with the correct datasets URL' { + Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly -ParameterFilter { + $Uri -like '*/datasets?api-version=2018-06-01' + } + } + } + + # This context specifically tests the real-world failure mode from issue #480: + # The Az.DataFactory SDK writes a *non-terminating* Write-Error (not a throw) when + # it cannot deserialize an unsupported object type such as ServiceNow V2. + # Without -ErrorAction Stop on the cmdlet call the try-catch never fires; with it, + # the non-terminating error is promoted to terminating and the catch fires correctly. + Context 'When Get-AzDataFactoryV2Dataset emits a non-terminating deserialization error (real Az.DataFactory behaviour)' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + # Simulate the actual Az.DataFactory SDK behaviour: Write-Error (non-terminating), + # NOT throw. The cmdlet call in Get-AdfFromService uses -ErrorAction Stop which + # must promote this to a terminating error so the catch block fires. + Mock Get-AzDataFactoryV2Dataset { Write-Error 'Unable to deserialize the response.' } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { @() } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { @() } + Mock Get-AzDFV2Credential { @() } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ds_adls_csv","properties":{}},{"name":"ds_servicenow_v2","properties":{}}]}' + } + } + } + + It 'Should not throw even when the cmdlet only writes a non-terminating error' { + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + + It 'Should activate the REST API fallback and return all datasets' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.DataSets.Count | Should -Be 2 + $result.DataSets.Name | Should -Contain 'ds_adls_csv' + $result.DataSets.Name | Should -Contain 'ds_servicenow_v2' + } + + It 'Should return AdfPSDataset wrapper objects from the REST fallback' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.DataSets | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSDataset' } + } + } + + Context 'When Get-AzDataFactoryV2LinkedService throws a deserialization error' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + Mock Get-AzDataFactoryV2Dataset { @() } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { throw 'Unable to deserialize the response.' } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { @() } + Mock Get-AzDFV2Credential { @() } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ls_adls","properties":{}},{"name":"ls_servicenow","properties":{}}]}' + } + } + } + + It 'Should not throw' { + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + + It 'Should return linked services via REST fallback as AdfPSLinkedService objects' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.LinkedServices.Count | Should -Be 2 + $result.LinkedServices | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSLinkedService' } + } + } + + Context 'When Get-AzDataFactoryV2Trigger throws a deserialization error' { + BeforeEach { + Mock Get-AzDataFactoryV2 { $script:fakeAdfi } + Mock Get-AzDataFactoryV2Dataset { @() } + Mock Get-AzDataFactoryV2IntegrationRuntime { @() } + Mock Get-AzDataFactoryV2LinkedService { @() } + Mock Get-AzDataFactoryV2Pipeline { @() } + Mock Get-AzDataFactoryV2DataFlow { @() } + Mock Get-AzDataFactoryV2Trigger { throw 'Unable to deserialize the response.' } + Mock Get-AzDFV2Credential { @() } + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"tr_daily","properties":{"runtimeState":"Started"}},{"name":"tr_hourly","properties":{"runtimeState":"Stopped"}}]}' + } + } + } + + It 'Should not throw' { + { Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' } | Should -Not -Throw + } + + It 'Should return triggers as AdfPSTrigger objects with correct RuntimeState' { + $result = Get-AdfFromService -FactoryName 'adf-test' -ResourceGroupName 'rg-test' + $result.Triggers.Count | Should -Be 2 + $result.Triggers | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSTrigger' } + ($result.Triggers | Where-Object Name -eq 'tr_daily').RuntimeState | Should -Be 'Started' + ($result.Triggers | Where-Object Name -eq 'tr_hourly').RuntimeState | Should -Be 'Stopped' + } + } + } + + # --------------------------------------------------------------------------- + Describe 'Get-AdfObjectsFromServiceRestAPI' -Tag 'Unit' { + + It 'Should exist' { + { Get-Command -Name Get-AdfObjectsFromServiceRestAPI -ErrorAction Stop } | Should -Not -Throw + } + + Context 'When the API returns an empty list' { + BeforeEach { + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ StatusCode = 200; Content = '{"value":[]}' } + } + } + + It 'Should return an empty collection' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + @($result).Count | Should -Be 0 + } + + It 'Should call Invoke-AzRestMethod exactly once' { + Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly + } + } + + Context 'When the API returns a single page of datasets' { + BeforeEach { + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ds_one","properties":{}},{"name":"ds_two","properties":{}},{"name":"ds_three","properties":{}}]}' + } + } + } + + It 'Should return all items from the page' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $result.Count | Should -Be 3 + } + + It 'Should return AdfPSDataset wrapper objects' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $result | ForEach-Object { $_.GetType().Name | Should -Be 'AdfPSDataset' } + } + + It 'Should preserve the dataset names' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $result.Name | Should -Contain 'ds_one' + $result.Name | Should -Contain 'ds_two' + $result.Name | Should -Contain 'ds_three' + } + + It 'Should call Invoke-AzRestMethod exactly once (no extra pages)' { + Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly + } + } + + Context 'When the API returns two pages (pagination)' { + BeforeEach { + $script:page2Url = 'https://management.azure.com/subscriptions/sub-123/.../datasets?api-version=2018-06-01&skipToken=page2' + $script:callCount = 0 + Mock Invoke-AzRestMethod { + $script:callCount++ + if ($script:callCount -eq 1) { + return [PSCustomObject]@{ + StatusCode = 200 + Content = "{`"value`":[{`"name`":`"ds_p1_a`",`"properties`":{}},{`"name`":`"ds_p1_b`",`"properties`":{}}],`"nextLink`":`"$($script:page2Url)`"}" + } + } else { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"ds_p2_a","properties":{}}]}' + } + } + } + } + + It 'Should return items from all pages combined' { + $result = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $result.Count | Should -Be 3 + $result.Name | Should -Contain 'ds_p1_a' + $result.Name | Should -Contain 'ds_p1_b' + $result.Name | Should -Contain 'ds_p2_a' + } + + It 'Should call Invoke-AzRestMethod twice (once per page)' { + Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + Assert-MockCalled Invoke-AzRestMethod -Times 2 -Exactly + } + + It 'Should use the nextLink URL for the second request' { + Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + Assert-MockCalled Invoke-AzRestMethod -Times 1 -Exactly -ParameterFilter { + $Uri -eq $script:page2Url + } + } + } + + Context 'When called for each supported object type' { + BeforeEach { + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ + StatusCode = 200 + Content = '{"value":[{"name":"obj1","properties":{"runtimeState":"Stopped"}}]}' + } + } + } + + It 'Should return AdfPSDataset for simpleType Dataset' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' + $r[0].GetType().Name | Should -Be 'AdfPSDataset' + } + + It 'Should return AdfPSPipeline for simpleType Pipeline' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'pipelines' -simpleType 'Pipeline' + $r[0].GetType().Name | Should -Be 'AdfPSPipeline' + } + + It 'Should return AdfPSLinkedService for simpleType LinkedService' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'linkedservices' -simpleType 'LinkedService' + $r[0].GetType().Name | Should -Be 'AdfPSLinkedService' + } + + It 'Should return AdfPSIntegrationRuntime for simpleType IntegrationRuntime' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'integrationruntimes' -simpleType 'IntegrationRuntime' + $r[0].GetType().Name | Should -Be 'AdfPSIntegrationRuntime' + } + + It 'Should return AdfPSDataFlow for simpleType DataFlow' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'dataflows' -simpleType 'DataFlow' + $r[0].GetType().Name | Should -Be 'AdfPSDataFlow' + } + + It 'Should return AdfPSTrigger for simpleType Trigger with RuntimeState' { + $r = Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'triggers' -simpleType 'Trigger' + $r[0].GetType().Name | Should -Be 'AdfPSTrigger' + $r[0].RuntimeState | Should -Be 'Stopped' + } + } + + Context 'When the API returns a non-200 status code' { + BeforeEach { + Mock Invoke-AzRestMethod { + return [PSCustomObject]@{ StatusCode = 403; Content = '{"error":{"code":"AuthorizationFailed"}}' } + } + } + + It 'Should throw when ErrorAction Stop is used' { + { Get-AdfObjectsFromServiceRestAPI -adfi $script:fakeAdfi -typePlural 'datasets' -simpleType 'Dataset' -ErrorAction Stop } | Should -Throw + } + } + } + + # --------------------------------------------------------------------------- Describe 'Get-AdfFromService' -Tag 'Integration' { + # Variables for use in integration tests + $t = Get-TargetEnv 'adf2' + $script:adfName = $t.DataFactoryName + $script:rg = $t.ResourceGroupName + It 'Should execute' { Get-AdfFromService -FactoryName $adfName -ResourceGroupName $rg } - } + } } +