diff --git a/README.md b/README.md index 1d9610e..7688171 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,8 @@ The main advantage of the module is the ability to publish all the Azure Data Fa - [Stage value as full path to CSV config file](#stage-value-as-full-path-to-csv-config-file) - [JSON format of Config file](#json-format-of-config-file) - [Step: Deployment Plan](#step-deployment-plan) - - [Step: Stoping triggers](#step-stoping-triggers) + - [DryRun / Plan output and CI usage](#dryrun--plan-output-and-ci-usage) + - [Step: Stopping triggers](#step-stopping-triggers) - [Step: Deployment of ADF objects](#step-deployment-of-adf-objects) - [Step: Save deployment state](#step-save-deployment-state) - [Step: Deleting objects not in source](#step-deleting-objects-not-in-source) @@ -185,6 +186,7 @@ Publish-AdfV2FromJson [-Option] [-Method] [-DryRun] + [-Plan] ``` Assuming your ADF is named ```SQLPlayerDemo``` and the code is located in ```c:\GitHub\AdfName\```, replace the values for *SubscriptionName*, *ResourceGroupName*, *DataFactoryName* and run the following command using PowerShell CLI: @@ -622,8 +624,45 @@ If you prefer using JSON rather than CSV for setting up configuration - JSON fil This step identifies objects to be deployed using `Includes` and `Excludes` list provided in *Publish Options*. Afterwards, if `IncrementalDeployment = true`, it excludes objects by comparing hashes from **Deployment State** to hashed of awaiting objects. +### DryRun / Plan output and CI usage -## Step: Stoping triggers +Use `-DryRun` or `-Plan` to preview intended changes without deploying. The process prints a terraform-like summary: + +```text +Terraform-like plan (DryRun): +Plan: 0 to add, 1 to change, 1 to destroy. + + ~ Update + ~ pipeline.PL_ExecSparkJob + + - Delete + - pipeline.PL_Obsolete + + = Unchanged / skipped + = dataset.DS_Json +``` + +The returned object contains a `DryRunPlan` property, so CI can enforce policy checks before deployment: + +```powershell +$result = Publish-AdfV2FromJson ` + -RootFolder $RootFolder ` + -ResourceGroupName $ResourceGroupName ` + -DataFactoryName $DataFactoryName ` + -Location $Location ` + -Option $opt ` + -Plan + +if ($result.DryRunPlan.Delete.Count -gt 0) { + Write-Error "Plan contains destroy actions. Review required before deployment." + exit 1 +} + +Write-Host "Plan accepted. Add=$($result.DryRunPlan.Create.Count); Change=$($result.DryRunPlan.Update.Count); Destroy=$($result.DryRunPlan.Delete.Count)" +``` + + +## Step: Stopping triggers 💬 In log you'll see line: `STEP: Stopping triggers...` diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index af6b098..287169e 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -12,7 +12,7 @@ RootModule = 'azure.datafactory.tools.psm1' # Version number of this module. - ModuleVersion = '1.17.1' + ModuleVersion = '1.18.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/changelog.md b/changelog.md index 14dce27..1855d2d 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.18.0] - 2026-07-01 +### Added +* `Publish-AdfV2FromJson` now supports terraform-like planning via `-DryRun`/`-Plan`, including structured `DryRunPlan` output (Create/Update/Delete/Unchanged) #360 + ## [1.17.1] - 2026-07-01 ### Fixed * Further fixes of `DeleteNotInSource` with deserialization issue #480 diff --git a/docs/ADVANCED/CMDLET_REFERENCE.md b/docs/ADVANCED/CMDLET_REFERENCE.md index 431ec2a..e334681 100644 --- a/docs/ADVANCED/CMDLET_REFERENCE.md +++ b/docs/ADVANCED/CMDLET_REFERENCE.md @@ -18,7 +18,8 @@ Publish-AdfV2FromJson ` [-Stage ] ` [-Option ] ` [-Method ] ` - [-DryRun] + [-DryRun] ` + [-Plan] ``` **Parameters:** @@ -33,6 +34,7 @@ Publish-AdfV2FromJson ` | Option | AdfPublishOption | No | Publish options object (filtering, triggers, etc.) | | Method | String | No | Publishing method: 'AzDataFactory' or 'AzResource' (default) | | DryRun | Switch | No | Validate without deploying | +| Plan | Switch | No | Alias behavior of DryRun with terraform-like naming | **Returns:** AdfPublishOption (deployment result) diff --git a/docs/GUIDE/PUBLISHING.md b/docs/GUIDE/PUBLISHING.md index d3ef583..68a4670 100644 --- a/docs/GUIDE/PUBLISHING.md +++ b/docs/GUIDE/PUBLISHING.md @@ -33,9 +33,12 @@ Publish-AdfV2FromJson ` [-Stage] ` [-Option] ` [-Method] ` - [-DryRun] + [-DryRun] ` + [-Plan] ``` +`-Plan` is an alias behavior of `-DryRun` and prints a terraform-like plan without deploying. + ### Complete Example ```powershell diff --git a/private/ApplyExclusionOptions.ps1 b/private/ApplyExclusionOptions.ps1 index 4042a3c..be91f8b 100644 --- a/private/ApplyExclusionOptions.ps1 +++ b/private/ApplyExclusionOptions.ps1 @@ -49,7 +49,9 @@ function ApplyExclusionOptions { function ToBeDeployedStat { param( - [Parameter(Mandatory=$True)] [Adf] $adf + [Parameter(Mandatory=$True)] [Adf] $adf, + [Parameter(Mandatory=$False)] $targetAdfInstance, + [switch] $TerraformStyle ) $ToBeDeployedList = ($adf.AllObjects() | Where-Object { $_.ToBeDeployed -eq $true } | ToArray) @@ -59,4 +61,166 @@ function ToBeDeployedStat { Write-Host "- $($_.FullName($true))" } + if (!$TerraformStyle) { + return $null + } + + $plan = Get-DryRunPlan -adf $adf -targetAdfInstance $targetAdfInstance + Write-DryRunPlan -plan $plan + return $plan + +} + +function ConvertTo-CanonicalAdfType { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] + [String] $Type + ) + + $t = $Type.ToLowerInvariant() + switch ($t) + { + 'pipeline' { return 'pipeline' } + 'dataset' { return 'dataset' } + 'dataflow' { return 'dataflow' } + 'linkedservice' { return 'linkedService' } + 'integrationruntime' { return 'integrationRuntime' } + 'trigger' { return 'trigger' } + 'factory' { return 'factory' } + 'managedvirtualnetwork' { return 'managedVirtualNetwork' } + 'managedprivateendpoint' { return 'managedPrivateEndpoint' } + 'credential' { return 'credential' } + default { return $Type } + } +} + +function Get-AdfObjectFullName { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] + $Object + ) + + $name = $Object.Name + $simtype = '' + + if ($Object -is [AdfObject]) { + $simtype = Get-SimplifiedType -Type $Object.Type + if ($Object.Type -like 'Microsoft.DataFactory/factories*') { + $simtype = ConvertTo-AdfType -AzType $Object.Type + $simtype = Get-SimplifiedType -Type $simtype + } + } + else { + $simtype = Get-SimplifiedType -Type $Object.GetType().Name + } + + $simtype = ConvertTo-CanonicalAdfType -Type $simtype + return "$simtype.$name" +} + +function Get-DryRunPlan { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] [Adf] $adf, + [parameter(Mandatory = $false)] $targetAdfInstance + ) + + $create = [System.Collections.Generic.List[string]]::new() + $update = [System.Collections.Generic.List[string]]::new() + $delete = [System.Collections.Generic.List[string]]::new() + $unchanged = [System.Collections.Generic.List[string]]::new() + + $sourceByName = @{} + $adf.AllObjects() | ForEach-Object { + $fullName = Get-AdfObjectFullName -Object $_ + $sourceByName[$fullName] = $_ + } + + $targetByName = @{} + if ($null -ne $targetAdfInstance) { + $targetAdfInstance.AllObjects() | ForEach-Object { + $fullName = Get-AdfObjectFullName -Object $_ + $targetByName[$fullName] = $_ + } + } + + $adf.AllObjects() | ForEach-Object { + $fullName = Get-AdfObjectFullName -Object $_ + if ($_.ToBeDeployed -eq $false) { + $unchanged.Add($fullName) | Out-Null + } + elseif ($targetByName.ContainsKey($fullName)) { + $update.Add($fullName) | Out-Null + } + else { + $create.Add($fullName) | Out-Null + } + } + + if ($null -ne $targetAdfInstance -and $adf.PublishOptions.DeleteNotInSource) { + $targetByName.Keys | ForEach-Object { + $fullName = $_ + if (!$sourceByName.ContainsKey($fullName)) { + $oname = [AdfObjectName]::new($fullName) + $canDelete = $true + if ($adf.PublishOptions.DoNotDeleteExcludedObjects) { + $canDelete = !($oname.IsNameExcluded($adf.PublishOptions)) + } + if ($canDelete) { + $delete.Add($fullName) | Out-Null + } + } + } + } + + $create = @($create | Sort-Object) + $update = @($update | Sort-Object) + $delete = @($delete | Sort-Object) + $unchanged = @($unchanged | Sort-Object) + + return [PSCustomObject]@{ + Create = @($create) + Update = @($update) + Delete = @($delete) + Unchanged = @($unchanged) + } +} + +function Write-DryRunPlan { + [CmdletBinding()] + param ( + [parameter(Mandatory = $true)] + $plan + ) + + Write-Host "===================================================================================" + Write-Host "Terraform-like plan (DryRun):" + Write-Host "Plan: $($plan.Create.Count) to add, $($plan.Update.Count) to change, $($plan.Delete.Count) to destroy." + + if ($plan.Create.Count -gt 0) { + Write-Host "" + Write-Host " + Create" + $plan.Create | ForEach-Object { Write-Host " + $_" } + } + + if ($plan.Update.Count -gt 0) { + Write-Host "" + Write-Host " ~ Update" + $plan.Update | ForEach-Object { Write-Host " ~ $_" } + } + + if ($plan.Delete.Count -gt 0) { + Write-Host "" + Write-Host " - Delete" + $plan.Delete | ForEach-Object { Write-Host " - $_" } + } + + if ($plan.Unchanged.Count -gt 0) { + Write-Host "" + Write-Host " = Unchanged / skipped" + $plan.Unchanged | ForEach-Object { Write-Host " = $_" } + } + } \ No newline at end of file diff --git a/public/Publish-AdfV2FromJson.ps1 b/public/Publish-AdfV2FromJson.ps1 index 3d687e7..410ac85 100644 --- a/public/Publish-AdfV2FromJson.ps1 +++ b/public/Publish-AdfV2FromJson.ps1 @@ -35,6 +35,9 @@ AzResource method has been introduced due to bugs in Az.DataFactory PS module. Optional switch parameter. When provided, process will not make any changes to target data factory but instead return the ADF object that would be used in deployment. +.PARAMETER Plan +Optional switch parameter. Alias for DryRun with terraform-style intent. Behaves the same as DryRun. + .EXAMPLE # Publish entire ADF $ResourceGroupName = 'rg-devops-factory' @@ -74,6 +77,10 @@ Publish-AdfV2FromJson -RootFolder "$RootFolder" -ResourceGroupName "$ResourceGro # Execute dry run of intended publishing changes Publish-AdfV2FromJson -RootFolder "$RootFolder" -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$DataFactoryName" -Location "$Location" -Stage "UAT" -DryRun +.EXAMPLE +# Execute plan (alias of DryRun) to preview intended changes +Publish-AdfV2FromJson -RootFolder "$RootFolder" -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$DataFactoryName" -Location "$Location" -Stage "UAT" -Plan + .LINK Online version: https://github.com/SQLPlayer/azure.datafactory.tools/ #> @@ -104,9 +111,16 @@ function Publish-AdfV2FromJson { [String]$Method = 'AzResource', [parameter(Mandatory = $false)] - [switch]$DryRun + [switch]$DryRun, + + [parameter(Mandatory = $false)] + [switch]$Plan ) + if ($Plan.IsPresent) { + $DryRun = $true + } + $m = Get-Module -Name "azure.datafactory.tools" $verStr = $m.Version.ToString(2) + "." + $m.Version.Build.ToString("000"); Write-Host "======================================================================================"; @@ -123,6 +137,7 @@ function Publish-AdfV2FromJson { Write-Host "Options provided: $($null -ne $Option)"; Write-Host "Publishing method: $Method"; Write-Host "Is Dry Run?: $($DryRun.IsPresent)"; + Write-Host "Is Plan Mode?: $($Plan.IsPresent)"; Write-Host "======================================================================================"; if ($null -ne $Option) { Write-Host "Options:" @@ -248,10 +263,29 @@ function Publish-AdfV2FromJson { } Write-Host "Found $unchanged_count unchanged object(s)." } - ToBeDeployedStat -adf $adf + + $dryRunPlan = $null + if ($DryRun.IsPresent) { + $targetAdfInstance = $null + try { + Write-Host "DRY RUN: Loading target ADF objects to classify plan changes..." + $targetAdfInstance = Get-AdfFromService -FactoryName "$DataFactoryName" -ResourceGroupName "$ResourceGroupName" + } + catch { + Write-Warning "DRY RUN: Unable to load target ADF state for full plan classification. Falling back to local-only plan. Details: $_" + } + + $dryRunPlan = ToBeDeployedStat -adf $adf -targetAdfInstance $targetAdfInstance -TerraformStyle + } + else { + ToBeDeployedStat -adf $adf + } if ($DryRun.IsPresent) { Write-Host "DRY RUN: Terminating script pre-deployment - inspect returned object to review planned changes." + if ($null -ne $dryRunPlan) { + $adf | Add-Member -MemberType NoteProperty -Name DryRunPlan -Value $dryRunPlan -Force + } Write-Host "==================================================================================="; return $adf } diff --git a/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 b/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 index 1da24b1..70514ae 100644 --- a/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 +++ b/test/Publish-AdfV2FromJson-DryRun.Tests.ps1 @@ -28,6 +28,10 @@ InModuleScope azure.datafactory.tools { Mock Set-StateToStorage { } + + Mock Get-AdfFromService { + $null + } } It 'should still load deployment state from storage when DryRun is enabled' { @@ -51,4 +55,63 @@ InModuleScope azure.datafactory.tools { Should -Invoke -CommandName Set-StateToStorage -Times 0 -Exactly } } + + Describe 'DryRun terraform-like plan output' -Tag 'Unit' { + It 'should classify add, change, destroy and expose DryRunPlan on returned object' { + $opt = New-AdfPublishOption + $opt.StopStartTriggers = $false + $opt.DeleteNotInSource = $true + $opt.DoNotDeleteExcludedObjects = $false + $opt.Includes.Add('pipeline.PL_ExecSparkJob', '') + + Mock Get-AdfFromService { + $target = New-Object -TypeName AdfInstance + $target.Pipelines = @( + [AdfPSPipeline]::new('PL_ExecSparkJob'), + [AdfPSPipeline]::new('PL_Obsolete') + ) + $target + } + + $result = Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName $script:ResourceGroupName ` + -DataFactoryName $script:DataFactoryName ` + -Location $script:Location ` + -Option $opt ` + -DryRun + + $result | Should -Not -BeNullOrEmpty + $result.PSObject.Properties.Name | Should -Contain 'DryRunPlan' + + $plan = $result.DryRunPlan + $plan | Should -Not -BeNullOrEmpty + $plan.Update | Should -Contain 'pipeline.PL_ExecSparkJob' + $plan.Delete | Should -Contain 'pipeline.PL_Obsolete' + $plan.Unchanged | Should -Contain 'dataset.DS_Json' + $plan.Create.Count | Should -Be 0 + } + + It 'should support -Plan as an alias behavior of -DryRun' { + $opt = New-AdfPublishOption + $opt.StopStartTriggers = $false + + Mock Get-AdfFromService { + $target = New-Object -TypeName AdfInstance + $target + } + + $result = Publish-AdfV2FromJson ` + -RootFolder $script:RootFolder ` + -ResourceGroupName $script:ResourceGroupName ` + -DataFactoryName $script:DataFactoryName ` + -Location $script:Location ` + -Option $opt ` + -Plan + + $result | Should -Not -BeNullOrEmpty + $result.PSObject.Properties.Name | Should -Contain 'DryRunPlan' + $result.DryRunPlan | Should -Not -BeNullOrEmpty + } + } } \ No newline at end of file