1. Run Azure/powershell@v2
2. with:
3. azPSVersion: latest
4. failOnStandardError: true
5. errorActionPreference: stop
6. inlineScript: # Make sure that errors are caught and the script stops
7. $PSNativeCommandUseErrorActionPreference = $true
8. $ErrorActionPreference = 'Stop'
9. $SubscriptionId = "${env:subscriptionId}"
10. # Override the installed version of Az.Accounts, as it's out of date
11. $requiredAzAccountsVersion = $(Find-Package Az.ResourceGraph -IncludeDependencies -Source PSGallery | Where-Object { $_.Name -eq 'Az.Accounts' }).Version
12. Install-Module -Name Az.Accounts -RequiredVersion $requiredAzAccountsVersion -Force -AllowClobber
13. $azAccountsModulePath = (Get-module -ListAvailable Az.Accounts).path | Where-Object { $_.Contains($requiredAzAccountsVersion) }
14. Import-Module -Name $azAccountsModulePath -Force
15. # Install the module so we can make graph queries
16. Install-Module -Name Az.ResourceGraph -Scope CurrentUser -force -AllowClobber
17. Import-Module -Name Az.ResourceGraph
18. Set-AzContext -Subscription $SubscriptionId
19. # We have to remap the variables from the previous task for the Publish-AdfV2FromJson task to pick them up
20. $lims_sqldnsname = "${env:lims_sqldnsname}"
21. $lims_sqldbname = "${env:lims_sqldbname}"
22. $projectDatalakeResourceId = "${env:projectDatalakeResourceId}"
23. $commonDatalakeResourceId = "${env:commonDatalakeResourceId}"
24. $databricksWorkspaceUrl = "${env:databricksWorkspaceUrl}"
25. $databricksResourceId = "${env:databricksResourceId}"
26. $databricksMultinodeClusterId = "${env:databricksMultinodeClusterId}"
27. $databricksSinglenodeClusterId = "${env:databricksSinglenodeClusterId}"
28. $databricksMultinodeMlClusterId = "${env:databricksMultinodeMlClusterId}"
29. $databricksSinglenodeMlClusterId = "${env:databricksSinglenodeMlClusterId}"
30. Write-Host "##[debug]Installing DataFactory tools"
31. Install-Module -Name Az.DataFactory -Scope CurrentUser -force -AllowClobber
32. Import-Module -Name Az.DataFactory -SkipEditionCheck -Force
33. Install-Module -Name azure.datafactory.tools -Scope CurrentUser -force -AllowClobber
34. Import-Module -Name azure.datafactory.tools
35. Write-Host "##[debug]Creating options for Datafactory Deployment"
36. # See parameters here: https://github.com/Azure-Player/azure.datafactory.tools/blob/master/public/Publish-AdfV2FromJson.ps1
37. $opt = New-AdfPublishOption
38. $opt.DeleteNotInSource = $true
39. $opt.StopStartTriggers = $true
40. $opt.CreateNewInstance = $false
41. Write-Host "##[debug]Starting Datafactory Deployment"
42. # If factory-override/$proj-override-$env.csv exists, use that as the override file otherwise use factory/$proj-$env.csv
43. # If there's an override file for this specific project and environment, use it
44. if (Test-Path "/home/runner/work/platform/platform/platform/azure/qberd/factory_override/qberd-factory-tst.csv") {
45. $override_filepath = "/home/runner/work/platform/platform/platform/azure/qberd/factory_override/qberd-factory-tst.csv"
46. } else {
47. $override_filepath = "/home/runner/work/platform/platform/platform/azure/qberd/factory_override/qberd-factory.csv"
48. }
49. Publish-AdfV2FromJson -RootFolder '/home/runner/work/platform/platform/platform/azure/qberd/factory/' -ResourceGroupName "${env:resourceGroup}" -DataFactoryName "qberd-factory-tst" -Location "AustraliaEast" -Stage "$override_filepath" -Method "AzResource" -Option $opt
50. Write-Host "##[debug]Testing pipelines"
51. $flag = $false
52. /home/runner/work/platform/platform/platform/infra/test/test_ADF_linked_services.ps1 -factory_name qberd-factory-tst -rg ${env:resourceGroup} -subscription_id $SubscriptionId
53. $flag = $flag -or !$?
54. /home/runner/work/platform/platform/platform/infra/test/test_ADF_integration_runtimes.ps1 -factory_name qberd-factory-tst -rg ${env:resourceGroup} -subscription_id $SubscriptionId
55. $flag = $flag -or !$?
56. exit $flag
57.
58. githubToken: ***
59. env:
60. AZURE_HTTP_USER_AGENT:
61. AZUREPS_HOST_ENVIRONMENT:
62. shirId: /subscriptions//resourceGroups/rg-SharedSrvc-dev/providers/Microsoft.DataFactory/factories/sharedsrvc-factory-dev/integrationruntimes/sharedsrvc-shir
63. subscriptionId:
64. resourceGroup: rg-qberd-tst
65. QESDsubscriptionId:
66. targetEnv: tst
67. lims_sqldnsname: npd-des-limsdev-sqlserver.database.windows.net
68. lims_sqldbname: npd-des-limsdev-sqldb
69. projectDatalakeResourceId: /subscriptions//resourceGroups/rg-QBERD-tst/providers/Microsoft.Storage/storageAccounts/qberdlaketst
70. commonDatalakeResourceId: /subscriptions//resourceGroups/rg-QESDCommon-tst/providers/Microsoft.Storage/storageAccounts/qesdcommonlaketst
71. databricksWorkspaceUrl:
72. databricksResourceId: /subscriptions//resourceGroups/rg-QBERD-tst/providers/Microsoft.Databricks/workspaces/qberd-databricks-tst
73. databricksMultinodeClusterId: 0829-004316-j0f2en38
74. databricksSinglenodeClusterId: 0829-004318-pedwrmuy
75. databricksMultinodeXlClusterId: 0916-065918-m56czyyh
76. databricksSinglenodeXlClusterId: 0916-065918-b7o9c4za
77. databricksMultinodeMlClusterId: 0829-004314-9kcqk2ap
78. databricksSinglenodeMlClusterId: 0829-004313-hw20tx9e
79. Validating inputs
80. Module Az latest installed from hostedAgentFolder
81. Initializing Az Module
82. /usr/bin/pwsh -NoLogo -NoProfile -NonInteractive -Command Test-Path (Join-Path /usr/share az_*)
83. /usr/bin/pwsh -NoLogo -NoProfile -NonInteractive -Command $env:PSModulePath
84. /home/runner/.local/share/powershell/Modules:/usr/local/share/powershell/Modules:/opt/microsoft/powershell/7/Modules
85. /usr/bin/pwsh -NoLogo -NoProfile -NonInteractive -Command try {
86. $ErrorActionPreference = "Stop"
87. $WarningPreference = "SilentlyContinue"
88. $output = @{}
89. $data = Get-Module -Name Az -ListAvailable | Sort-Object Version -Descending | Select-Object -First 1
90. $output['AzVersion'] = $data.Version.ToString()
91. $output['Success'] = "true"
92. }
93. catch {
94. $output['Error'] = $_.exception.Message
95. }
96. return ConvertTo-Json $output
97. True
98. {
99. "AzVersion": "12.5.0",
100. "Success": "true"
101. }
102. /usr/bin/pwsh -NoLogo -NoProfile -NonInteractive -Command $env:PSModulePath
103. /usr/share:/home/runner/.local/share/powershell/Modules:/usr/local/share/powershell/Modules:/opt/microsoft/powershell/7/Modules
104. Initializing Az Module Complete
105. Running Az PowerShell Script
106. /usr/bin/pwsh -NoLogo -NoProfile -NonInteractive -Command /home/runner/work/_temp/.ps1
107.
108. WARNING: You're using Az version 12.5.0. The latest version of Az is 15.0.0. Upgrade your Az modules using the following commands:
109. Update-PSResource Az -WhatIf -- Simulate updating your Az modules.
110. Update-PSResource Az -- Update your Az modules.
111. There will be breaking changes from 12.5.0 to 15.0.0. Open https://go.microsoft/fwlink/?linkid=2241373 and check the details.
112. ======================================================================================
113. ### azure.datafactory.tools Version 1.14.000 ###
114. ======================================================================================
115. Invoking Publish-AdfV2FromJson (https://github.com/SQLPlayer/azure.datafactory.tools)
116. with the following parameters:
117. ======================================================================================
118. RootFolder: /home/runner/work/platform/platform/platform/azure/qberd/factory/
119. ResourceGroupName: rg-qberd-tst
120. DataFactoryName: qberd-factory-tst
121. Location: AustraliaEast
122. Stage: /home/runner/work/platform/platform/platform/azure/qberd/factory_override/qberd-factory.csv
123. Options provided: True
124. Publishing method: AzResource
125. Is Dry Run?: False
126. ======================================================================================
127. Options:
128. Includes : {}
129. Excludes : {}
130. DeleteNotInSource : True
131. StopStartTriggers : True
132. CreateNewInstance : False
133. DeployGlobalParams : True
134. FailsWhenConfigItemNotFound : True
135. FailsWhenPathNotFound : True
136. IgnoreLackOfReferencedObject : False
137. DoNotStopStartExcludedTriggers : False
138. DoNotDeleteExcludedObjects : True
139. IncrementalDeployment : False
140. IncrementalDeploymentStorageUri :
141. TriggerStopMethod : AllEnabled
142. TriggerStartMethod : BasedOnSourceCode
143.
144. ======================================================================================
145. Publish options are provided.
146. STEP: Verifying whether ADF exists...
147. Azure Data Factory exists.
148. ===================================================================================
149. STEP: Reading Azure Data Factory from JSON files...
150. IntegrationRuntimes: 1 object(s) loaded.
151. LinkedServices: 13 object(s) loaded.
152. Pipelines: 21 object(s) loaded.
153. DataSets: 38 object(s) loaded.
154. DataFlows: 0 object(s) loaded.
155. Triggers: 3 object(s) loaded.
156. Managed VNet: 0 object(s) loaded.
157. Managed Private Endpoints: 0 object(s) loaded.
158. Credentials: 0 object(s) loaded.
159. Factories: 0 object(s) loaded.
160. ===================================================================================
161. STEP: Replacing all properties environment-related...
162. Config file: /home/runner/work/platform/platform/platform/azure/qberd/factory_override/qberd-factory.csv
163. Validation of config file completed.
164. Tenant: ***
165. SubscriptionName SubscriptionId Account
166. ---------------- -------------- -------
167. DES-QESD_QBERD-npd
168. Write-Error: /home/runner/.local/share/powershell/Modules/azure.datafactory.tools/1.14.0/private/Update-PropertiesFromFile.ps1:80
169. Line |
170. 80 | … $null = Update-PropertiesForObject -o $_ -action $action -path $p …
171. | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
172. | ADFT0010: Wrong path defined in config for object(path):
173. | linkedService.ls_qberd_dynamic_sql(properties.properties.parameters.url.defaultValue)
174. *** Properties modification report ***
175.
176. Updated Removed Added
177. ------- ------- -----
178. 14 0 0
179.
180. ===================================================================================
181. STEP: Determining the objects to be deployed...
182. # Number of objects marked as to be deployed: 76/76
183. - [linkedService].[Is_wildnet_api]
184. - [linkedService].[ls_arcgis_http_survey123_r8]
185. - [linkedService].[ls_arcgis_http]
186. - [linkedService].[ls_qbeis_datalake]
187. - [linkedService].[ls_qberd_databricks_ml_multinode]
188. - [linkedService].[ls_qberd_databricks_multinode]
189. - [linkedService].[ls_qberd_databricks_singlenode]
190. - [linkedService].[ls_qberd_datalake]
191. - [linkedService].[ls_qberd_dynamic_sql]
192. - [linkedService].[ls_qberd_keyvault]
193. - [linkedService].[ls_qberd_synapse]
194. - [linkedService].[test_dan]
195. - [linkedService].[test]
196. - [pipeline].[0_arcgis_orchestrate]
197. - [pipeline].[0_arcgis_survey_123_orchestrate]
198. - [pipeline].[0_qbeis_to_qberd_orchestrate]
199. - [pipeline].[0_qberd_export_orchestrate]
200. - [pipeline].[0_wildnet_api_export_orchestrate]
201. - [pipeline].[1_arcgis_survey_123_to_raw]
202. - [pipeline].[1_arcgis_to_raw]
203. - [pipeline].[1_qbeis_to_qberd]
204. - [pipeline].[1_qberd_sql_to_lake_curated]
205. - [pipeline].[1_wildnet_api_to_lake_landing_raw]
206. - [pipeline].[2_1_wildnet_api_superseded_to_lake_landing_raw]
207. - [pipeline].[2_synapse_survey_123_to_qberd_sql]
208. - [pipeline].[2_wildnet_api_superseded_to_lake_landing_raw]
209. - [pipeline].[3_1_wildnet_api_json_archive_delete]
210. - [pipeline].[3_wildnet_api_json_archive_retention_check]
211. - [pipeline].[4_1_synapse_to_qberd_sql_notification]
212. - [pipeline].[4_synapse_to_qberd_sql]
213. - [pipeline].[5_1_superseded_update_qberd_sql_notificaiton]
214. - [pipeline].[5_superseded_update_qberd_sql]
215. - [pipeline].[9_custom_notification]
216. - [pipeline].[9_on_demand_notification]
217. - [dataset].[ds_arcgis_datalake_json]
218. - [dataset].[ds_arcgis_datalake_parquet]
219. - [dataset].[ds_arcgis_json]
220. - [dataset].[ds_arcgis_rapid_assessment_8_datalake_json_archive_survey]
221. - [dataset].[ds_arcgis_rapid_assessment_8_datalake_json_archive]
222. - [dataset].[ds_arcgis_rapid_assessment_8_datalake_json]
223. - [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet_source_survey]
224. - [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet_source]
225. - [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet_survey_attachments]
226. - [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet_survey]
227. - [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet]
228. - [dataset].[ds_arcgis_rapid_assessment_8_json]
229. - [dataset].[ds_arcgis_rapid_assessment_8_merged_survey]
230. - [dataset].[ds_qbeis_s3_contents]
231. - [dataset].[ds_qberd_binary_raw]
232. - [dataset].[ds_qberd_curated_parquet]
233. - [dataset].[ds_qberd_raw_parquet]
234. - [dataset].[ds_qberd_s3_landing]
235. - [dataset].[ds_qberd_sql_BioCon_temp]
236. - [dataset].[ds_qberd_sql_BioCon]
237. - [dataset].[ds_qberd_sql_disturbance_temp]
238. - [dataset].[ds_qberd_sql_disturbance]
239. - [dataset].[ds_qberd_sql_temp]
240. - [dataset].[ds_qberd_sql]
241. - [dataset].[ds_qberd_synapse]
242. - [dataset].[ds_synapse_od]
243. - [dataset].[ds_wildnet_api__landing_csv]
244. - [dataset].[ds_wildnet_api__landing_dummy]
245. - [dataset].[ds_wildnet_api_json_landing_archive]
246. - [dataset].[ds_wildnet_api_json_landing_dummy]
247. - [dataset].[ds_wildnet_api_json_landing_supersed]
248. - [dataset].[ds_wildnet_api_json_landing]
249. - [dataset].[ds_wildnet_api_landing_Supersed_csv]
250. - [dataset].[ds_wildnet_api_landing_supersed]
251. - [dataset].[ds_wildnet_api_parquet_raw_superseded]
252. - [dataset].[ds_wildnet_api_parquet_raw]
253. - [dataset].[ds_wildnet_api_species_search]
254. - [dataset].[ds_wildnet_api_supersededBy]
255. - [trigger].[qbeis_daily_import_s3_container]
256. - [trigger].[Qberd export - weekly run]
257. - [trigger].[Wildnet API export - weekly run]
258. - [IntegrationRuntime].[sharedsrvc-shir]
259. ===================================================================================
260. STEP: Stopping triggers...
261. Getting triggers...
262. The number of active triggers: 0 (out of 3)
263. TriggerStopMethod = AllEnabled
264. The number of triggers to stop: 0
265. ===================================================================================
266. STEP: Deployment of all ADF objects...
267. Start deploying object: [linkedService].[Is_wildnet_api] (1 dependency/ies)
268. Start deploying object: [IntegrationRuntime].[sharedsrvc-shir] (0 dependency/ies)
269. New-AzResource: /home/runner/.local/share/powershell/Modules/azure.datafactory.tools/1.14.0/private/Deploy-AdfObjectOnly.ps1:134
270. Line |
271. 134 | New-AzResource `
272. | ~~~~~~~~~~~~~~~~
273. | RequestDisallowedByPolicy : Resource 'sharedsrvc-shir' was disallowed by
274. | policy. Policy identifiers:
275. | '[{"policyAssignment":{"name":"ITP-Policy-Deny-DataFactoryIntegrationRuntimes","id":"/providers/Microsoft.Management/managementGroups/NonProdPublicAccessFirewallUnRestricted/providers/Microsoft.Authorization/policyAssignments/fd0de06dc3bf10b1fdafd47c"},"policyDefinition":{"name":"ITP-Policy-Deny-DataFactoryIntegrationRuntimes","id":"/providers/Microsoft.Management/managementGroups/***/providers/Microsoft.Authorization/policyDefinitions/80a0a353-b614-4ade-8243-36a5c26ae559","version":"1.0.0"}}]'. CorrelationId: 5ef502ea-a36f-479f-a1b3-883c47be4e7c
276. Finished deploying object: [IntegrationRuntime].[sharedsrvc-shir]
277. Finished deploying object: [linkedService].[Is_wildnet_api]
278. Start deploying object: [linkedService].[ls_arcgis_http_survey123_r8] (0 dependency/ies)
279. Finished deploying object: [linkedService].[ls_arcgis_http_survey123_r8]
280. Start deploying object: [linkedService].[ls_arcgis_http] (0 dependency/ies)
281. Finished deploying object: [linkedService].[ls_arcgis_http]
282. Start deploying object: [linkedService].[ls_qbeis_datalake] (0 dependency/ies)
283. Finished deploying object: [linkedService].[ls_qbeis_datalake]
284. Start deploying object: [linkedService].[ls_qberd_databricks_ml_multinode] (1 dependency/ies)
285. Finished deploying object: [linkedService].[ls_qberd_databricks_ml_multinode]
286. Start deploying object: [linkedService].[ls_qberd_databricks_multinode] (1 dependency/ies)
287. Finished deploying object: [linkedService].[ls_qberd_databricks_multinode]
288. Start deploying object: [linkedService].[ls_qberd_databricks_singlenode] (1 dependency/ies)
289. Finished deploying object: [linkedService].[ls_qberd_databricks_singlenode]
290. Start deploying object: [linkedService].[ls_qberd_datalake] (0 dependency/ies)
291. Finished deploying object: [linkedService].[ls_qberd_datalake]
292. Start deploying object: [linkedService].[ls_qberd_dynamic_sql] (1 dependency/ies)
293. Finished deploying object: [linkedService].[ls_qberd_dynamic_sql]
294. Start deploying object: [linkedService].[ls_qberd_keyvault] (0 dependency/ies)
295. Finished deploying object: [linkedService].[ls_qberd_keyvault]
296. Start deploying object: [linkedService].[ls_qberd_synapse] (0 dependency/ies)
297. Finished deploying object: [linkedService].[ls_qberd_synapse]
298. Start deploying object: [linkedService].[test_dan] (0 dependency/ies)
299. Finished deploying object: [linkedService].[test_dan]
300. Start deploying object: [linkedService].[test] (0 dependency/ies)
301. Finished deploying object: [linkedService].[test]
302. Start deploying object: [pipeline].[0_arcgis_orchestrate] (1 dependency/ies)
303. Start deploying object: [pipeline].[1_arcgis_to_raw] (4 dependency/ies)
304. Start deploying object: [dataset].[ds_arcgis_json] (1 dependency/ies)
305. Finished deploying object: [dataset].[ds_arcgis_json]
306. Start deploying object: [dataset].[ds_arcgis_datalake_json] (1 dependency/ies)
307. Finished deploying object: [dataset].[ds_arcgis_datalake_json]
308. Start deploying object: [dataset].[ds_synapse_od] (1 dependency/ies)
309. Finished deploying object: [dataset].[ds_synapse_od]
310. Start deploying object: [dataset].[ds_arcgis_datalake_parquet] (1 dependency/ies)
311. Finished deploying object: [dataset].[ds_arcgis_datalake_parquet]
312. Finished deploying object: [pipeline].[1_arcgis_to_raw]
313. Finished deploying object: [pipeline].[0_arcgis_orchestrate]
314. Start deploying object: [pipeline].[0_arcgis_survey_123_orchestrate] (3 dependency/ies)
315. Start deploying object: [pipeline].[1_arcgis_survey_123_to_raw] (11 dependency/ies)
316. Start deploying object: [dataset].[ds_arcgis_rapid_assessment_8_json] (1 dependency/ies)
317. Finished deploying object: [dataset].[ds_arcgis_rapid_assessment_8_json]
318. Start deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_json] (1 dependency/ies)
319. Finished deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_json]
320. Start deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet] (1 dependency/ies)
321. Finished deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet]
322. Start deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_json_archive] (1 dependency/ies)
323. Finished deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_json_archive]
324. Start deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet_source] (1 dependency/ies)
325. Finished deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet_source]
326. Start deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet_survey] (1 dependency/ies)
327. Finished deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet_survey]
328. Start deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_json_archive_survey] (1 dependency/ies)
329. Finished deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_json_archive_survey]
330. Start deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet_source_survey] (1 dependency/ies)
331. Finished deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet_source_survey]
332. Start deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet_survey_attachments] (1 dependency/ies)
333. Finished deploying object: [dataset].[ds_arcgis_rapid_assessment_8_datalake_parquet_survey_attachments]
334. Finished deploying object: [pipeline].[1_arcgis_survey_123_to_raw]
335. Start deploying object: [pipeline].[9_on_demand_notification] (0 dependency/ies)
336. Finished deploying object: [pipeline].[9_on_demand_notification]
337. Start deploying object: [pipeline].[2_synapse_survey_123_to_qberd_sql] (7 dependency/ies)
338. Start deploying object: [dataset].[ds_qberd_sql] (1 dependency/ies)
339. Finished deploying object: [dataset].[ds_qberd_sql]
340. Start deploying object: [dataset].[ds_qberd_synapse] (1 dependency/ies)
341. Finished deploying object: [dataset].[ds_qberd_synapse]
342. Start deploying object: [dataset].[ds_qberd_sql_temp] (1 dependency/ies)
343. Finished deploying object: [dataset].[ds_qberd_sql_temp]
344. Start deploying object: [dataset].[ds_qberd_sql_BioCon_temp] (1 dependency/ies)
345. Finished deploying object: [dataset].[ds_qberd_sql_BioCon_temp]
346. Start deploying object: [dataset].[ds_qberd_sql_disturbance_temp] (1 dependency/ies)
347. Finished deploying object: [dataset].[ds_qberd_sql_disturbance_temp]
348. Start deploying object: [dataset].[ds_qberd_sql_BioCon] (1 dependency/ies)
349. Finished deploying object: [dataset].[ds_qberd_sql_BioCon]
350. Start deploying object: [dataset].[ds_qberd_sql_disturbance] (1 dependency/ies)
351. Finished deploying object: [dataset].[ds_qberd_sql_disturbance]
352. Finished deploying object: [pipeline].[2_synapse_survey_123_to_qberd_sql]
353. Finished deploying object: [pipeline].[0_arcgis_survey_123_orchestrate]
354. Start deploying object: [pipeline].[0_qbeis_to_qberd_orchestrate] (1 dependency/ies)
355. Start deploying object: [pipeline].[1_qbeis_to_qberd] (2 dependency/ies)
356. Start deploying object: [dataset].[ds_qbeis_s3_contents] (1 dependency/ies)
357. Finished deploying object: [dataset].[ds_qbeis_s3_contents]
358. Start deploying object: [dataset].[ds_qberd_s3_landing] (1 dependency/ies)
359. Finished deploying object: [dataset].[ds_qberd_s3_landing]
360. Finished deploying object: [pipeline].[1_qbeis_to_qberd]
361. Finished deploying object: [pipeline].[0_qbeis_to_qberd_orchestrate]
362. Start deploying object: [pipeline].[0_qberd_export_orchestrate] (2 dependency/ies)
363. Start deploying object: [pipeline].[1_qberd_sql_to_lake_curated] (4 dependency/ies)
364. Start deploying object: [pipeline].[9_custom_notification] (0 dependency/ies)
365. Finished deploying object: [pipeline].[9_custom_notification]
366. Start deploying object: [dataset].[ds_qberd_curated_parquet] (1 dependency/ies)
367. Finished deploying object: [dataset].[ds_qberd_curated_parquet]
368. Finished deploying object: [pipeline].[1_qberd_sql_to_lake_curated]
369. Finished deploying object: [pipeline].[0_qberd_export_orchestrate]
370. Start deploying object: [pipeline].[0_wildnet_api_export_orchestrate] (6 dependency/ies)
371. Start deploying object: [pipeline].[1_wildnet_api_to_lake_landing_raw] (8 dependency/ies)
372. Start deploying object: [dataset].[ds_wildnet_api_species_search] (1 dependency/ies)
373. Finished deploying object: [dataset].[ds_wildnet_api_species_search]
374. Start deploying object: [dataset].[ds_wildnet_api_json_landing] (1 dependency/ies)
375. Finished deploying object: [dataset].[ds_wildnet_api_json_landing]
376. Start deploying object: [dataset].[ds_wildnet_api_landing_Supersed_csv] (1 dependency/ies)
377. Finished deploying object: [dataset].[ds_wildnet_api_landing_Supersed_csv]
378. Start deploying object: [dataset].[ds_wildnet_api_parquet_raw] (1 dependency/ies)
379. Finished deploying object: [dataset].[ds_wildnet_api_parquet_raw]
380. Start deploying object: [dataset].[ds_wildnet_api_json_landing_archive] (1 dependency/ies)
381. Finished deploying object: [dataset].[ds_wildnet_api_json_landing_archive]
382. Finished deploying object: [pipeline].[1_wildnet_api_to_lake_landing_raw]
383. Start deploying object: [pipeline].[2_wildnet_api_superseded_to_lake_landing_raw] (6 dependency/ies)
384. Start deploying object: [dataset].[ds_wildnet_api_json_landing_supersed] (1 dependency/ies)
385. Finished deploying object: [dataset].[ds_wildnet_api_json_landing_supersed]
386. Start deploying object: [dataset].[ds_wildnet_api_json_landing_dummy] (1 dependency/ies)
387. Finished deploying object: [dataset].[ds_wildnet_api_json_landing_dummy]
388. Start deploying object: [pipeline].[2_1_wildnet_api_superseded_to_lake_landing_raw] (5 dependency/ies)
389. Start deploying object: [dataset].[ds_wildnet_api_parquet_raw_superseded] (1 dependency/ies)
390. Finished deploying object: [dataset].[ds_wildnet_api_parquet_raw_superseded]
391. Finished deploying object: [pipeline].[2_1_wildnet_api_superseded_to_lake_landing_raw]
392. Finished deploying object: [pipeline].[2_wildnet_api_superseded_to_lake_landing_raw]
393. Start deploying object: [pipeline].[3_wildnet_api_json_archive_retention_check] (3 dependency/ies)
394. Start deploying object: [pipeline].[3_1_wildnet_api_json_archive_delete] (1 dependency/ies)
395. Finished deploying object: [pipeline].[3_1_wildnet_api_json_archive_delete]
396. Finished deploying object: [pipeline].[3_wildnet_api_json_archive_retention_check]
397. Start deploying object: [pipeline].[4_synapse_to_qberd_sql] (6 dependency/ies)
398. Start deploying object: [pipeline].[4_1_synapse_to_qberd_sql_notification] (1 dependency/ies)
399. Finished deploying object: [pipeline].[4_1_synapse_to_qberd_sql_notification]
400. Finished deploying object: [pipeline].[4_synapse_to_qberd_sql]
401. Start deploying object: [pipeline].[5_superseded_update_qberd_sql] (6 dependency/ies)
402. Start deploying object: [pipeline].[5_1_superseded_update_qberd_sql_notificaiton] (1 dependency/ies)
403. Finished deploying object: [pipeline].[5_1_superseded_update_qberd_sql_notificaiton]
404. Finished deploying object: [pipeline].[5_superseded_update_qberd_sql]
405. Finished deploying object: [pipeline].[0_wildnet_api_export_orchestrate]
406. Start deploying object: [dataset].[ds_arcgis_rapid_assessment_8_merged_survey] (1 dependency/ies)
407. Finished deploying object: [dataset].[ds_arcgis_rapid_assessment_8_merged_survey]
408. Start deploying object: [dataset].[ds_qberd_binary_raw] (1 dependency/ies)
409. Finished deploying object: [dataset].[ds_qberd_binary_raw]
410. Start deploying object: [dataset].[ds_qberd_raw_parquet] (1 dependency/ies)
411. Finished deploying object: [dataset].[ds_qberd_raw_parquet]
412. Start deploying object: [dataset].[ds_wildnet_api__landing_csv] (1 dependency/ies)
413. Finished deploying object: [dataset].[ds_wildnet_api__landing_csv]
414. Start deploying object: [dataset].[ds_wildnet_api__landing_dummy] (1 dependency/ies)
415. Finished deploying object: [dataset].[ds_wildnet_api__landing_dummy]
416. Start deploying object: [dataset].[ds_wildnet_api_landing_supersed] (1 dependency/ies)
417. Finished deploying object: [dataset].[ds_wildnet_api_landing_supersed]
418. Start deploying object: [dataset].[ds_wildnet_api_supersededBy] (1 dependency/ies)
419. Finished deploying object: [dataset].[ds_wildnet_api_supersededBy]
420. Start deploying object: [trigger].[qbeis_daily_import_s3_container] (1 dependency/ies)
421. Finished deploying object: [trigger].[qbeis_daily_import_s3_container]
422. Start deploying object: [trigger].[Qberd export - weekly run] (1 dependency/ies)
423. Finished deploying object: [trigger].[Qberd export - weekly run]
424. Start deploying object: [trigger].[Wildnet API export - weekly run] (1 dependency/ies)
425. Finished deploying object: [trigger].[Wildnet API export - weekly run]
426. ===================================================================================
427. STEP: Deleting objects not in source ...
428. Azure Data Factory (instance) loaded.
429. DataSets: 38 object(s) loaded.
430. IntegrationRuntimes: 1 object(s) loaded.
431. LinkedServices: 13 object(s) loaded.
432. Pipelines: 21 object(s) loaded.
433. DataFlows: 0 object(s) loaded.
434. Triggers: 3 object(s) loaded.
435. Credentials: 0 object(s) loaded.
436. Deleted 0 objects from ADF service.
437. ===================================================================================
438. STEP: Updating (incremental) deployment state...
439. Incremental Deployment State will not be saved as publish option 'IncrementalDeployment' = false
440. Try this new feature to speed up the deployment process. Check out more in documentation.
441. ===================================================================================
442. STEP: Starting triggers...
443. TriggerStartMethod = BasedOnSourceCode
444. The number of triggers to start: 1
445. Starting triggers:
446. - Enabling trigger: qbeis_daily_import_s3_container
447. Complete starting triggers.
448. ==============================================================================
449. ***** Azure Data Factory files have been deployed successfully. *****
450.
451. Data Factory name: qberd-factory-tst
452. Resource Group name: rg-qberd-tst
453. Region (Location): AustraliaEast
454. Elapsed time: 0:00:41.402
455.
456. ==============================================================================
457.
458. Name : qberd-factory-tst
459. ResourceGroupName : rg-qberd-tst
460. Region : AustraliaEast
461. Pipelines : {0_arcgis_orchestrate, 0_arcgis_survey_123_orchestrat
462. e, 0_qbeis_to_qberd_orchestrate, 0_qberd_export_orche
463. strate…}
464. LinkedServices : {Is_wildnet_api, ls_arcgis_http_survey123_r8, ls_arcg
465. is_http, ls_qbeis_datalake…}
466. DataSets : {ds_arcgis_datalake_json, ds_arcgis_datalake_parquet,
467. ds_arcgis_json, ds_arcgis_rapid_assessment_8_datalak
468. e_json_archive_survey…}
469. DataFlows : {}
470. Triggers : {qbeis_daily_import_s3_container, Qberd export - week
471. ly run, Wildnet API export - weekly run}
472. IntegrationRuntimes : {sharedsrvc-shir}
473. Factories : {}
474. ManagedVirtualNetwork : {}
475. ManagedPrivateEndpoints : {}
476. Credentials : {}
477. Location : /home/runner/work/platform/platform/platform/azure/qb
478. erd/factory/
479. GlobalFactory : AdfGlobalProp
480. PublishOptions : AdfPublishOption
481. ArmTemplateJson :
482. ActiveTriggers : {}
483. StoppedTriggerNames : {}
484. TargetTriggerNames : {@{Name=qbeis_daily_import_s3_container; RuntimeState
485. =Stopped}, @{Name=Qberd export - weekly run; RuntimeS
486. tate=Stopped}, @{Name=Wildnet API export - weekly run
487. ; RuntimeState=Stopped}}
488. DeletedObjectNames : {}
489.
490. WARNING: Preview version of extension is disabled by default for extension installation, enabled for modules without stable versions.
491. WARNING: Please run 'az config set extension.dynamic_install_allow_preview=true or false' to config it specifically.
492. WARNING: The command requires the extension datafactory. It will be installed first.
493. LinkedServiceName: ls_qberd_databricks_ml_multinode DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
494.
495. succeeded : True
496.
497. LinkedServiceName: ls_qberd_databricks_multinode DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
498.
499. succeeded : True
500.
501. LinkedServiceName: ls_qberd_datalake DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
502.
503. succeeded : True
504.
505. LinkedServiceName: ls_qberd_keyvault DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
506.
507. succeeded : True
508.
509. LinkedServiceName: ls_qberd_synapse DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
510.
511. succeeded : True
512.
513. LinkedServiceName: ls_arcgis_http DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
514.
515. succeeded : True
516.
517. LinkedServiceName: Is_wildnet_api DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
518. Skipped, linked service has user-defined parameters or is an Azure ML Service/Function App
519. LinkedServiceName: ls_qberd_databricks_singlenode DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
520.
521. succeeded : True
522.
523. LinkedServiceName: ls_arcgis_http_survey123_r8 DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
524.
525. succeeded : True
526.
527. LinkedServiceName: test_dan DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
528.
529. succeeded : True
530.
531. LinkedServiceName: test DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
532.
533. succeeded : True
534.
535. LinkedServiceName: ls_qberd_dynamic_sql DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
536. Skipped, linked service has user-defined parameters or is an Azure ML Service/Function App
537. LinkedServiceName: ls_qbeis_datalake DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst
538.
539. succeeded : True
540.
541. Checking Runtime sharedsrvc-shir DataFactoryName: qberd-factory-tst ResourceGroupName: rg-qberd-tst Subscription: f1334b3f-99c0-49d4-8ab2-3a588e905af4
542. Integration runtime sharedsrvc-shir is online.
543.
544. Script execution Complete
Originally posted by @mariariveraaraya in #471