From b12a9188b7d9a3383d491322100981039460515b Mon Sep 17 00:00:00 2001 From: affonsov Date: Thu, 8 Aug 2024 12:33:37 -0700 Subject: [PATCH 1/3] Adding functionality to add shared to the connector functions --- .../tests/FunctionSharedSettings.json | 3 ++ testframework/tests/RunPQSDKTestSuites.ps1 | 48 ++++++++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 testframework/tests/FunctionSharedSettings.json diff --git a/testframework/tests/FunctionSharedSettings.json b/testframework/tests/FunctionSharedSettings.json new file mode 100644 index 0000000..7f14bb0 --- /dev/null +++ b/testframework/tests/FunctionSharedSettings.json @@ -0,0 +1,3 @@ +{ + "Functions": [] +} \ No newline at end of file diff --git a/testframework/tests/RunPQSDKTestSuites.ps1 b/testframework/tests/RunPQSDKTestSuites.ps1 index 946f986..0c80b90 100644 --- a/testframework/tests/RunPQSDKTestSuites.ps1 +++ b/testframework/tests/RunPQSDKTestSuites.ps1 @@ -3,7 +3,7 @@ Script: RunPQSDKTestSuites.ps1 Runs the pre-built PQ/PQOut format tests in Power Query SDK Test Framework using pqtest.exe compare command. - .DESCRIPTION + .DESCRIPTION This script will execute the PQ SDK PQ/PQOut tests present under Sanity, Standard & DataSourceSpecific folders. RunPQSDKTestSuitesSettings.json file is used provide configurations need to this script. Please review the template RunPQSDKTestSuitesSettingsTemplate.json for more info. Pre-Requisite: Ensure the credentials are setup for your connector following the instructions here: https://learn.microsoft.com/power-query/power-query-sdk-vs-code#set-credential @@ -51,6 +51,7 @@ param( [switch]$ValidateQueryFolding, [switch]$DetailedResults, [switch]$JSONResults, + [string]$FunctionSharedSettingsPath, [string]$RunPQSDKTestSuitesSettingsPath, [switch]$Pipeline ) @@ -63,6 +64,11 @@ if ($RunPQSDKTestSuitesSettingsPath) { $RunPQSDKTestSuitesSettings = Get-Content -Path $RunPQSDKTestSuitesSettingsPath | ConvertFrom-Json } +# Retrieving the settings for functions that need to be set to shared from the JSON settings file +if ($FunctionSharedSettingsPath) { + $FunctionSharedSettings = Get-Content -Path $FunctionSharedSettingsPath | ConvertFrom-Json +} + # Setting the PQTestExePath from settings object if not passed as an argument if (!$PQTestExePath) { $PQTestExePath = $RunPQSDKTestSuitesSettings.PQTestExePath } if ([string]::IsNullOrEmpty($PQTestExePath) -or !(Test-Path -Path $PQTestExePath)) { @@ -209,6 +215,42 @@ if (!$Pipeline) { } } +# Add shared to the functions and compile +if ($FunctionSharedSettings.Functions) { + $ProjectName = [System.IO.Path]::GetFileNameWithoutExtension($ExtensionPath) + $TempProjectPath = "temp\$ProjectName" + if (!(Test-Path -Path $TempProjectPath)) { + New-Item -ItemType Directory -Path $TempProjectPath + } + Copy-Item $ExtensionPath "$TempProjectPath\$ProjectName.zip" + Expand-Archive -Path "$TempProjectPath\$ProjectName.zip" -DestinationPath $TempProjectPath -Force + $PQFile = "$TempProjectPath\$ProjectName.pq" + Remove-Item "$TempProjectPath\$ProjectName.zip" + $PQFileContent = (Get-Content $PQFile) + foreach ($item in $FunctionSharedSettings.Functions) { + Write-Output ("Function replaced: " + $item ) + $PQFileContent = $PQFileContent.Replace("$item =", "shared $item =") + } + While ($True) { + try { + $PQFileContent | Set-Content $PQFile + break + } + catch { + Write-Output "Error in saving $PQFile" + Start-Sleep -Seconds 1 # wait for a seconds before next attempt. + } + } + Push-Location + Set-Location $TempProjectPath + # find the latest MakePQX.exe + $PQXMakePQXPath = Get-ChildItem "$home\.vscode\extensions" -recurse -include "MakePQX.exe" | Select-Object -last 1 | ForEach-Object { $_.FullName } + Invoke-expression "$PQXMakePQXPath compile" + Pop-Location + $ExtensionPath = Get-ChildItem -Path "$TempProjectPath\bin\AnyCPU\Debug\$ProjectName.mez" | ForEach-Object { $_.FullName } +} + + # Creating the DiagnosticFolderPath if ValidateQueryFolding is set to true if ($ValidateQueryFolding) { $DiagnosticFolderPath = Join-Path -Path (Get-Location) -ChildPath ("Diagnostics\" + (Get-Item $ExtensionPath).Basename) @@ -322,6 +364,10 @@ Write-Output("------------------------------------------------------------------ Write-Output("Total Tests: " + $TestCount + " | Passed: " + $Passed + " | Failed: " + $Failed + " | Total Duration: " + "{0:dd}d:{0:hh}h:{0:mm}m:{0:ss}s" -f (NEW-TIMESPAN -Start $TestExecStartTime -End $TestExecEndTime)) Write-Output("----------------------------------------------------------------------------------------------") +if ($FunctionSharedSettings.Functions) { + Remove-Item temp -Recurse -Force -Confirm:$false +} + if ($Failed -gt 0) { exit -1 } From 87ecb1976cf5c898ed0d11219a8c21f6fcb411c2 Mon Sep 17 00:00:00 2001 From: affonsov Date: Wed, 14 Aug 2024 14:33:23 -0700 Subject: [PATCH 2/3] Update - Added comments --- testframework/tests/RunPQSDKTestSuites.ps1 | 25 ++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/testframework/tests/RunPQSDKTestSuites.ps1 b/testframework/tests/RunPQSDKTestSuites.ps1 index 1ae5751..e5d3497 100644 --- a/testframework/tests/RunPQSDKTestSuites.ps1 +++ b/testframework/tests/RunPQSDKTestSuites.ps1 @@ -222,36 +222,52 @@ if (!$Pipeline) { # Add shared to the functions and compile if ($FunctionSharedSettings.Functions) { + # Get Project Name from the mez file provided by the user $ProjectName = [System.IO.Path]::GetFileNameWithoutExtension($ExtensionPath) + + # Create a temporary folder that will have the connector so we can add the shared to the functions $TempProjectPath = "temp\$ProjectName" if (!(Test-Path -Path $TempProjectPath)) { New-Item -ItemType Directory -Path $TempProjectPath } Copy-Item $ExtensionPath "$TempProjectPath\$ProjectName.zip" Expand-Archive -Path "$TempProjectPath\$ProjectName.zip" -DestinationPath $TempProjectPath -Force - $PQFile = "$TempProjectPath\$ProjectName.pq" Remove-Item "$TempProjectPath\$ProjectName.zip" + + # Get the content from the pq file to add the shared keyword to the functions + $PQFile = "$TempProjectPath\$ProjectName.pq" $PQFileContent = (Get-Content $PQFile) + + # Loop through the FunctionSharedSettings and add the shared keyword in the PQFile content foreach ($item in $FunctionSharedSettings.Functions) { Write-Output ("Function replaced: " + $item ) $PQFileContent = $PQFileContent.Replace("$item =", "shared $item =") } - While ($True) { + + # Try to save the PQfile + $Tentatives = 0 + While ($Tentatives -le 10) { try { $PQFileContent | Set-Content $PQFile + Write-Output "$PQFile saved successfully" break } catch { Write-Output "Error in saving $PQFile" Start-Sleep -Seconds 1 # wait for a seconds before next attempt. } - } + $Tentatives++ + } + Push-Location + # Change folder location Set-Location $TempProjectPath - # find the latest MakePQX.exe + # Find the latest MakePQX.exe $PQXMakePQXPath = Get-ChildItem "$home\.vscode\extensions" -recurse -include "MakePQX.exe" | Select-Object -last 1 | ForEach-Object { $_.FullName } Invoke-expression "$PQXMakePQXPath compile" Pop-Location + + # set the new mez file with shared function so testframework used this to run the tests $ExtensionPath = Get-ChildItem -Path "$TempProjectPath\bin\AnyCPU\Debug\$ProjectName.mez" | ForEach-Object { $_.FullName } } @@ -370,6 +386,7 @@ Write-Output("------------------------------------------------------------------ Write-Output("Total Tests: " + $TestCount + " | Passed: " + $Passed + " | Failed: " + $Failed + " | Total Duration: " + "{0:dd}d:{0:hh}h:{0:mm}m:{0:ss}s" -f (NEW-TIMESPAN -Start $TestExecStartTime -End $TestExecEndTime)) Write-Output("----------------------------------------------------------------------------------------------") +# Delte the temporary folder created to store the mez file with shared functions if ($FunctionSharedSettings.Functions) { Remove-Item temp -Recurse -Force -Confirm:$false } From fd04d29dcc138a8b8028889bed530acd2f41c8ae Mon Sep 17 00:00:00 2001 From: affonsov Date: Wed, 14 Aug 2024 14:47:42 -0700 Subject: [PATCH 3/3] adding empty line for FunctionSharedSettings.json --- testframework/tests/FunctionSharedSettings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testframework/tests/FunctionSharedSettings.json b/testframework/tests/FunctionSharedSettings.json index 7f14bb0..1088fba 100644 --- a/testframework/tests/FunctionSharedSettings.json +++ b/testframework/tests/FunctionSharedSettings.json @@ -1,3 +1,3 @@ { "Functions": [] -} \ No newline at end of file +}