-
Notifications
You must be signed in to change notification settings - Fork 10
Bugfix conversion of single activities in pipeline json files #49
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
Alex-Swann
wants to merge
10
commits into
Azure-Player:main
Choose a base branch
from
Alex-Swann:bugfix/pipeline-notebook-deployments
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
10 commits
Select commit
Hold shift + click to select a range
36564d2
Bugfix conversion of single activities in pipeline json files when ha…
Alex-Swann 2944e18
Fix PSObject.Properties null-safe property checks
Alex-Swann 621c683
Add unit tests to demonstrate ConvertTo-JSON activities bug
Alex-Swann b600647
Fix auth token: check Az.Accounts module version (>=3.0.0) for Secure…
Alex-Swann 5a3ae6a
Add null-safety to Az.Accounts version check and unit tests for Get-R…
Alex-Swann 49023ba
Use -AsPlainText instead of BSTR marshal for Az.Accounts >= 3.0.0 tok…
Alex-Swann fc008f7
Fix SecureString unwrap on Linux: use CoTaskMemUnicode+PtrToStringUni…
Alex-Swann bcd3bb9
Fallback to REST API when Get-AzSynapsePipeline fails on single-activ…
Alex-Swann 626a7df
Add unit tests for Get-SynapseFromService pipeline REST fallback
Alex-Swann 2c1d4cd
Fix Get-SynapseFromService tests: stub Az cmdlets, use type name chec…
Alex-Swann File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| BeforeDiscovery { | ||
| $ModuleRootPath = $PSScriptRoot | Split-Path -Parent | ||
| $moduleManifestName = 'azure.synapse.tools.psd1' | ||
| $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName | ||
|
|
||
| Import-Module -Name $moduleManifestPath -Force -Verbose:$false | ||
| } | ||
|
|
||
| InModuleScope azure.synapse.tools { | ||
| $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' | ||
| Import-Module -Name $testHelperPath -Force | ||
|
|
||
| Describe 'Get-RequestHeader' -Tag 'Unit' { | ||
|
|
||
| It 'Should exist' { | ||
| { Get-Command -Name Get-RequestHeader -ErrorAction Stop } | Should -Not -Throw | ||
| } | ||
|
|
||
| Context 'Az.Accounts >= 3.0.0 (SecureString token - cross-platform unicode unwrap)' { | ||
| BeforeAll { | ||
| $plainToken = 'eyJhbGciOiJSUzI1NiJ9.payload.sig' | ||
| $secureToken = ConvertTo-SecureString $plainToken -AsPlainText -Force | ||
|
|
||
| Mock Get-Module { | ||
| [PSCustomObject]@{ Version = [version]'3.0.0' } | ||
| } -ParameterFilter { $Name -eq 'Az.Accounts' } -ModuleName azure.synapse.tools | ||
|
|
||
| Mock Get-AzAccessToken { | ||
| [PSCustomObject]@{ Token = $secureToken } | ||
| } -ModuleName azure.synapse.tools | ||
|
|
||
| $script:header = Get-RequestHeader | ||
| } | ||
|
|
||
| It 'Should return a Bearer token header' { | ||
| $script:header['Authorization'] | Should -Match '^Bearer ' | ||
| } | ||
|
|
||
| It 'Should correctly unwrap the SecureString to a plain-text token' { | ||
| $script:header['Authorization'] | Should -Be "Bearer $plainToken" | ||
| } | ||
|
|
||
| It 'Should set Content-Type to application/json' { | ||
| $script:header['Content-Type'] | Should -Be 'application/json' | ||
| } | ||
| } | ||
|
|
||
| Context 'Az.Accounts < 3.0.0 (plain string token)' { | ||
| BeforeAll { | ||
| $plainToken = 'eyJhbGciOiJSUzI1NiJ9.payload.sig' | ||
|
|
||
| Mock Get-Module { | ||
| [PSCustomObject]@{ Version = [version]'2.9.0' } | ||
| } -ParameterFilter { $Name -eq 'Az.Accounts' } -ModuleName azure.synapse.tools | ||
|
|
||
| Mock Get-AzAccessToken { | ||
| [PSCustomObject]@{ token = $plainToken } | ||
| } -ModuleName azure.synapse.tools | ||
|
|
||
| $script:header = Get-RequestHeader | ||
| } | ||
|
|
||
| It 'Should return a Bearer token header' { | ||
| $script:header['Authorization'] | Should -Match '^Bearer ' | ||
| } | ||
|
|
||
| It 'Should use the plain-text token directly' { | ||
| $script:header['Authorization'] | Should -Be "Bearer $plainToken" | ||
| } | ||
|
|
||
| It 'Should set Content-Type to application/json' { | ||
| $script:header['Content-Type'] | Should -Be 'application/json' | ||
| } | ||
| } | ||
|
|
||
| Context 'Az.Accounts module not loaded (null version fallback)' { | ||
| BeforeAll { | ||
| $plainToken = 'eyJhbGciOiJSUzI1NiJ9.payload.sig' | ||
|
|
||
| Mock Get-Module { | ||
| $null | ||
| } -ParameterFilter { $Name -eq 'Az.Accounts' } -ModuleName azure.synapse.tools | ||
|
|
||
| # Plain string token (old Az.Accounts < 3.0.0) uses .token property | ||
| Mock Get-AzAccessToken { | ||
| [PSCustomObject]@{ token = $plainToken } | ||
| } -ModuleName azure.synapse.tools | ||
|
|
||
| $script:header = Get-RequestHeader | ||
| } | ||
|
|
||
| It 'Should not throw' { | ||
| { Get-RequestHeader } | Should -Not -Throw | ||
| } | ||
|
|
||
| It 'Should still return a valid Authorization header' { | ||
| $script:header['Authorization'] | Should -Be "Bearer $plainToken" | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| BeforeDiscovery { | ||
| $ModuleRootPath = $PSScriptRoot | Split-Path -Parent | ||
| $moduleManifestName = 'azure.synapse.tools.psd1' | ||
| $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName | ||
|
|
||
| Import-Module -Name $moduleManifestPath -Force -Verbose:$false | ||
| } | ||
|
|
||
| # Stub out Az.Synapse cmdlets so Pester can mock them without Az.Synapse being installed | ||
| foreach ($cmd in @( | ||
| 'Get-AzSynapseWorkspace', 'Get-AzSynapseNotebook', 'Get-AzSynapseDataset', | ||
| 'Get-AzSynapseIntegrationRuntime', 'Get-AzSynapseLinkedService', | ||
| 'Get-AzSynapsePipeline', 'Get-AzSynapseDataFlow', 'Get-AzSynapseTrigger' | ||
| )) { | ||
| if (-not (Get-Command $cmd -ErrorAction SilentlyContinue)) { | ||
| $null = New-Item -Path "Function:Global:$cmd" -Value { } | ||
| } | ||
| } | ||
|
|
||
| InModuleScope azure.synapse.tools { | ||
| $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' | ||
| Import-Module -Name $testHelperPath -Force | ||
|
|
||
| Describe 'Get-SynapseFromService' -Tag 'Unit' { | ||
|
|
||
| It 'Should exist' { | ||
| { Get-Command -Name Get-SynapseFromService -ErrorAction Stop } | Should -Not -Throw | ||
| } | ||
|
|
||
| BeforeAll { | ||
| $workspaceName = 'synws-test-01' | ||
|
|
||
| Mock Get-AzSynapseWorkspace { | ||
| [PSCustomObject]@{ Id = '/subscriptions/00000000/resourceGroups/rg/providers/Microsoft.Synapse/workspaces/synws-test-01'; Location = 'uksouth' } | ||
| } -ModuleName azure.synapse.tools | ||
|
|
||
| Mock Get-AzSynapseNotebook { @() } -ModuleName azure.synapse.tools | ||
| Mock Get-AzSynapseDataset { @() } -ModuleName azure.synapse.tools | ||
| Mock Get-AzSynapseIntegrationRuntime { @() } -ModuleName azure.synapse.tools | ||
| Mock Get-AzSynapseLinkedService { @() } -ModuleName azure.synapse.tools | ||
| Mock Get-AzSynapseDataFlow { @() } -ModuleName azure.synapse.tools | ||
| Mock Get-AzSynapseTrigger { @() } -ModuleName azure.synapse.tools | ||
| } | ||
|
|
||
| Context 'Get-AzSynapsePipeline succeeds' { | ||
| BeforeAll { | ||
| Mock Get-AzSynapsePipeline { | ||
| @( | ||
| [PSCustomObject]@{ Name = 'pl_pipeline_a' }, | ||
| [PSCustomObject]@{ Name = 'pl_pipeline_b' } | ||
| ) | ||
| } -ModuleName azure.synapse.tools | ||
|
|
||
| $script:result = Get-SynapseFromService -WorkspaceName $workspaceName | ||
| } | ||
|
|
||
| It 'Should return a Synapse object' { | ||
| $script:result.GetType().Name | Should -Be 'Synapse' | ||
| } | ||
|
|
||
| It 'Should populate Pipelines from Az cmdlet' { | ||
| @($script:result.Pipelines).Count | Should -Be 2 | ||
| } | ||
|
|
||
| It 'Should contain the expected pipeline names' { | ||
| $script:result.Pipelines.Name | Should -Contain 'pl_pipeline_a' | ||
| $script:result.Pipelines.Name | Should -Contain 'pl_pipeline_b' | ||
| } | ||
| } | ||
|
|
||
| Context 'Get-AzSynapsePipeline fails (single-activity object deserialization bug)' { | ||
| BeforeAll { | ||
| Mock Get-AzSynapsePipeline { | ||
| throw "The requested operation requires an element of type 'Array', but the target element has type 'Object'." | ||
| } -ModuleName azure.synapse.tools | ||
|
|
||
| Mock Get-RequestHeader { | ||
| @{ 'Authorization' = 'Bearer dummy'; 'Content-Type' = 'application/json' } | ||
| } -ModuleName azure.synapse.tools | ||
|
|
||
| Mock Invoke-RestMethod { | ||
| [PSCustomObject]@{ | ||
| value = @( | ||
| [PSCustomObject]@{ name = 'pl_single_activity_pipeline' }, | ||
| [PSCustomObject]@{ name = 'pl_another_pipeline' } | ||
| ) | ||
| } | ||
| } -ModuleName azure.synapse.tools | ||
|
|
||
| $script:result = Get-SynapseFromService -WorkspaceName $workspaceName | ||
| } | ||
|
|
||
| It 'Should return a Synapse object despite the Az cmdlet failure' { | ||
| $script:result.GetType().Name | Should -Be 'Synapse' | ||
| } | ||
|
|
||
| It 'Should fall back to REST API and populate Pipelines' { | ||
| @($script:result.Pipelines).Count | Should -Be 2 | ||
| } | ||
|
|
||
| It 'Should produce PipelineResource objects so Get-SimplifiedType resolves to Pipeline' { | ||
| $script:result.Pipelines[0].GetType().Name | Should -Be 'PipelineResource' | ||
| } | ||
|
|
||
| It 'Should contain the expected pipeline names from REST response' { | ||
| $script:result.Pipelines.Name | Should -Contain 'pl_single_activity_pipeline' | ||
| $script:result.Pipelines.Name | Should -Contain 'pl_another_pipeline' | ||
| } | ||
|
|
||
| It 'Should call Invoke-RestMethod with the correct Synapse pipelines endpoint' { | ||
| # Verify via side-effect: if wrong URI or no call, pipelines would be empty | ||
| # The mock is URI-specific so if the wrong URI was used, Invoke-RestMethod | ||
| # would call through to the real implementation and fail | ||
| @($script:result.Pipelines).Count | Should -BeGreaterThan 0 | ||
| $script:result.Pipelines[0].Name | Should -Be 'pl_single_activity_pipeline' | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| BeforeDiscovery { | ||
| $ModuleRootPath = $PSScriptRoot | Split-Path -Parent | ||
| $moduleManifestName = 'azure.synapse.tools.psd1' | ||
| $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName | ||
|
|
||
| Import-Module -Name $moduleManifestPath -Force -Verbose:$false | ||
| } | ||
|
|
||
| InModuleScope azure.synapse.tools { | ||
| $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' | ||
| Import-Module -Name $testHelperPath -Force | ||
|
|
||
| Describe 'Save-SynapseObjectAsFile' -Tag 'Unit' { | ||
|
|
||
| BeforeAll { | ||
| $script:tmpDir = New-TemporaryDirectory | ||
| New-Item -ItemType Directory -Path (Join-Path $script:tmpDir.FullName 'pipeline') -Force | Out-Null | ||
| New-Item -ItemType Directory -Path (Join-Path $script:tmpDir.FullName 'dataset') -Force | Out-Null | ||
| } | ||
|
|
||
| AfterAll { | ||
| Remove-Item $script:tmpDir.FullName -Recurse -Force -ErrorAction SilentlyContinue | ||
| } | ||
|
|
||
| It 'Should exist' { | ||
| { Get-Command -Name Save-SynapseObjectAsFile -ErrorAction Stop } | Should -Not -Throw | ||
| } | ||
|
|
||
| Context 'Single top-level activity coerced to object by ConvertFrom-Json' { | ||
| BeforeAll { | ||
| $synapse = [Synapse]::new() | ||
| $synapse.Location = $script:tmpDir.FullName | ||
| $obj = [SynapseObject]::new() | ||
| $obj.Name = 'pl_single_activity' | ||
| $obj.Type = 'pipeline' | ||
| $obj.Synapse = $synapse | ||
| $obj.Body = '{"name":"pl_single_activity","properties":{"activities":{"name":"act1","type":"Copy","typeProperties":{}}}}' | ConvertFrom-Json | ||
| $script:result = Save-SynapseObjectAsFile -obj $obj | ||
| } | ||
| It 'Should serialise activities as a JSON array' { | ||
| $raw = Get-Content $script:result -Raw | ||
| $raw | Should -Match '"activities":\[' | ||
| $raw | Should -Not -Match '"activities":\{' | ||
| } | ||
| } | ||
|
|
||
| Context 'Multiple top-level activities already an array' { | ||
| BeforeAll { | ||
| $synapse = [Synapse]::new() | ||
| $synapse.Location = $script:tmpDir.FullName | ||
| $obj = [SynapseObject]::new() | ||
| $obj.Name = 'pl_multi_activity' | ||
| $obj.Type = 'pipeline' | ||
| $obj.Synapse = $synapse | ||
| $obj.Body = '{"name":"pl_multi_activity","properties":{"activities":[{"name":"act1","type":"Copy","typeProperties":{}},{"name":"act2","type":"Copy","typeProperties":{}}]}}' | ConvertFrom-Json | ||
| $script:result = Save-SynapseObjectAsFile -obj $obj | ||
| } | ||
| It 'Should preserve all activities and write them as a JSON array' { | ||
| $json = Get-Content $script:result -Raw | ConvertFrom-Json | ||
| @($json.properties.activities).Count | Should -Be 2 | ||
| } | ||
| } | ||
|
|
||
| Context 'ForEach nested single activity coerced to object by ConvertFrom-Json' { | ||
| BeforeAll { | ||
| $synapse = [Synapse]::new() | ||
| $synapse.Location = $script:tmpDir.FullName | ||
| $obj = [SynapseObject]::new() | ||
| $obj.Name = 'pl_foreach_single_nested_activity' | ||
| $obj.Type = 'pipeline' | ||
| $obj.Synapse = $synapse | ||
| $obj.Body = '{"name":"pl_foreach_single_nested_activity","properties":{"activities":[{"name":"forEach1","type":"ForEach","typeProperties":{"activities":{"name":"act1","type":"Copy","typeProperties":{}}}}]}}' | ConvertFrom-Json | ||
| $script:result = Save-SynapseObjectAsFile -obj $obj | ||
| } | ||
| It 'Should serialise nested typeProperties.activities as a JSON array' { | ||
| $raw = Get-Content $script:result -Raw | ||
| $raw | Should -Match '"typeProperties":\{"activities":\[' | ||
| $raw | Should -Not -Match '"typeProperties":\{"activities":\{' | ||
| } | ||
| } | ||
|
|
||
| Context 'Non-pipeline object' { | ||
| BeforeAll { | ||
| $synapse = [Synapse]::new() | ||
| $synapse.Location = $script:tmpDir.FullName | ||
| $obj = [SynapseObject]::new() | ||
| $obj.Name = 'ds_test' | ||
| $obj.Type = 'dataset' | ||
| $obj.Synapse = $synapse | ||
| $obj.Body = '{"name":"ds_test","properties":{"type":"AzureBlob"}}' | ConvertFrom-Json | ||
| $script:result = Save-SynapseObjectAsFile -obj $obj | ||
| } | ||
| It 'Should write the object to file without modification' { | ||
| $json = Get-Content $script:result -Raw | ConvertFrom-Json | ||
| $json.name | Should -Be 'ds_test' | ||
| $json.properties.type | Should -Be 'AzureBlob' | ||
| } | ||
| } | ||
| } | ||
| } |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Avoid using operators supported in PS 7.x+ only.