Dev/mmitche/release10 v4 publishing backport

eng/Publishing.props

@@ -8,6 +8,7 @@
    <PropertyGroup>
      <!-- Do not auto-generate symbol packages. Arcade embeds all PDBs so it's just waste of space -->
      <AutoGenerateSymbolPackages>false</AutoGenerateSymbolPackages>
+     <PublishingVersion>4</PublishingVersion>
    </PropertyGroup>
 
 </Project>

eng/build.yml

@@ -22,15 +22,17 @@ stages:
         publish:
           artifacts: true
           logs: true
-          manifests: true
       enableMicrobuild: true
       microbuildUseESRP: ${{ parameters.microbuildUseESRP }}
+      enablePublishBuildAssets: true
       enableSourceIndex: true
       enableSourceBuild: true
+      publishingVersion: 4
       workspace:
         clean: all
       jobs:
       - job: Windows_NT
+        enablePublishing: true
         timeoutInMinutes: 90
         strategy:
           matrix:
@@ -62,7 +64,7 @@ stages:
 
 - template: /eng/common/templates-official/post-build/post-build.yml@self
   parameters:
-    publishingInfraVersion: 3
+    publishingInfraVersion: 4
     # signing validation will not run, even if the below value is 'true', if the 'PostBuildSign' variable is set to 'true'
     enableSigningValidation: false
     # Sourcelink validation isn't passing for Arcade due to some regressions. This should be

eng/common-variables.yml

@@ -28,11 +28,9 @@ variables:
     - group: Publish-Build-Assets
     - group: DotNet-HelixApi-Access
     - group: SDL_Settings
-    # DotNetPublishUsingPipelines can be removed when Arcade itself consumes a new Arcade version.
     - name: _InternalBuildArgs
       value: /p:DotNetSignType=${{ parameters.signType }}
         /p:TeamName=$(_TeamName)
-        /p:DotNetPublishUsingPipelines=true
         /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
     - name: PostBuildSign
       value: true

eng/common/core-templates/job/job.yml

@@ -24,12 +24,12 @@ parameters:
   enablePublishBuildArtifacts: false
   enablePublishBuildAssets: false
   enablePublishTestResults: false
+  enablePublishing: false
   enableBuildRetry: false
   mergeTestResults: false
   testRunTitle: ''
   testResultsFormat: ''
   name: ''
-  componentGovernanceSteps: []
   preSteps: []
   artifactPublishSteps: []
   runAsPublic: false
@@ -146,9 +146,6 @@ jobs:
   - ${{ each step in parameters.steps }}:
     - ${{ step }}
 
-  - ${{ each step in parameters.componentGovernanceSteps }}:
-    - ${{ step }}
-
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - template: /eng/common/core-templates/steps/cleanup-microbuild.yml
       parameters:

eng/common/core-templates/job/publish-build-assets.yml

@@ -172,17 +172,18 @@ jobs:
             targetPath: '$(Build.ArtifactStagingDirectory)/MergedManifest.xml'
             artifactName: AssetManifests
             displayName: 'Publish Merged Manifest'
-            retryCountOnTaskFailure: 10 # for any logs being locked
-            sbomEnabled: false  # we don't need SBOM for logs
+            retryCountOnTaskFailure: 10 # for any files being locked
+            isProduction: false # just metadata for publishing
 
-    - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+    - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
       parameters:
         is1ESPipeline: ${{ parameters.is1ESPipeline }}
         args:
           displayName: Publish ReleaseConfigs Artifact
-          pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs'
-          publishLocation: Container
+          targetPath: '$(Build.StagingDirectory)/ReleaseConfigs'
           artifactName: ReleaseConfigs
+          retryCountOnTaskFailure: 10 # for any files being locked
+          isProduction: false # just metadata for publishing
 
     - ${{ if or(eq(parameters.publishAssetsImmediately, 'true'), eq(parameters.isAssetlessBuild, 'true')) }}:
       - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
@@ -218,4 +219,5 @@ jobs:
       - template: /eng/common/core-templates/steps/publish-logs.yml
         parameters:
           is1ESPipeline: ${{ parameters.is1ESPipeline }}
-          JobLabel: 'Publish_Artifacts_Logs'     
+          StageLabel: 'BuildAssetRegistry'
+          JobLabel: 'Publish_Artifacts_Logs'

eng/common/core-templates/jobs/jobs.yml

@@ -43,6 +43,10 @@ parameters:
 
   artifacts: {}
   is1ESPipeline: ''
+
+  # Publishing version w/default.
+  publishingVersion: 3
+
   repositoryAlias: self
   officialBuildId: ''
 
@@ -102,6 +106,7 @@ jobs:
       parameters:
         is1ESPipeline: ${{ parameters.is1ESPipeline }}
         continueOnError: ${{ parameters.continueOnError }}
+        publishingVersion: ${{ parameters.publishingVersion }}
         dependsOn:
         - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
           - ${{ each job in parameters.publishBuildAssetsDependsOn }}:

eng/common/core-templates/post-build/post-build.yml

@@ -9,6 +9,7 @@ parameters:
     default: 3
     values:
     - 3
+    - 4
 
   - name: BARBuildId
     displayName: BAR Build Id
@@ -140,16 +141,30 @@ stages:
             PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
             is1ESPipeline: ${{ parameters.is1ESPipeline }}
 
-        - task: DownloadBuildArtifacts@0
-          displayName: Download Package Artifacts
-          inputs:
-            buildType: specific
-            buildVersionToDownload: specific
-            project: $(AzDOProjectName)
-            pipeline: $(AzDOPipelineId)
-            buildId: $(AzDOBuildId)
-            artifactName: PackageArtifacts
-            checkDownloadedFiles: true
+        - ${{ if ne(parameters.publishingInfraVersion, 4) }}:
+          - task: DownloadBuildArtifacts@0
+            displayName: Download Package Artifacts
+            inputs:
+              buildType: specific
+              buildVersionToDownload: specific
+              project: $(AzDOProjectName)
+              pipeline: $(AzDOPipelineId)
+              buildId: $(AzDOBuildId)
+              artifactName: PackageArtifacts
+              checkDownloadedFiles: true
+        - ${{ if eq(parameters.publishingInfraVersion, 4) }}:
+          - task: DownloadPipelineArtifact@2
+            displayName: Download Pipeline Artifacts (V4)
+            inputs:
+              itemPattern: '*/packages/**/*.nupkg'
+              targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload'
+          - task: CopyFiles@2
+            displayName: Flatten packages to PackageArtifacts
+            inputs:
+              SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload'
+              Contents: '**/*.nupkg'
+              TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts'
+              flattenFolders: true
 
         - task: PowerShell@2
           displayName: Validate
@@ -183,16 +198,30 @@ stages:
             PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
             is1ESPipeline: ${{ parameters.is1ESPipeline }}
 
-        - task: DownloadBuildArtifacts@0
-          displayName: Download Package Artifacts
-          inputs:
-            buildType: specific
-            buildVersionToDownload: specific
-            project: $(AzDOProjectName)
-            pipeline: $(AzDOPipelineId)
-            buildId: $(AzDOBuildId)
-            artifactName: PackageArtifacts
-            checkDownloadedFiles: true
+        - ${{ if ne(parameters.publishingInfraVersion, 4) }}:
+          - task: DownloadBuildArtifacts@0
+            displayName: Download Package Artifacts
+            inputs:
+              buildType: specific
+              buildVersionToDownload: specific
+              project: $(AzDOProjectName)
+              pipeline: $(AzDOPipelineId)
+              buildId: $(AzDOBuildId)
+              artifactName: PackageArtifacts
+              checkDownloadedFiles: true
+        - ${{ if eq(parameters.publishingInfraVersion, 4) }}:
+          - task: DownloadPipelineArtifact@2
+            displayName: Download Pipeline Artifacts (V4)
+            inputs:
+              itemPattern: '*/packages/**/*.nupkg'
+              targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload'
+          - task: CopyFiles@2
+            displayName: Flatten packages to PackageArtifacts
+            inputs:
+              SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload'
+              Contents: '**/*.nupkg'
+              TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts'
+              flattenFolders: true
 
         # This is necessary whenever we want to publish/restore to an AzDO private feed
         # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
@@ -244,16 +273,30 @@ stages:
             PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
             is1ESPipeline: ${{ parameters.is1ESPipeline }}
 
-        - task: DownloadBuildArtifacts@0
-          displayName: Download Blob Artifacts
-          inputs:
-            buildType: specific
-            buildVersionToDownload: specific
-            project: $(AzDOProjectName)
-            pipeline: $(AzDOPipelineId)
-            buildId: $(AzDOBuildId)
-            artifactName: BlobArtifacts
-            checkDownloadedFiles: true
+        - ${{ if ne(parameters.publishingInfraVersion, 4) }}:
+          - task: DownloadBuildArtifacts@0
+            displayName: Download Blob Artifacts
+            inputs:
+              buildType: specific
+              buildVersionToDownload: specific
+              project: $(AzDOProjectName)
+              pipeline: $(AzDOPipelineId)
+              buildId: $(AzDOBuildId)
+              artifactName: BlobArtifacts
+              checkDownloadedFiles: true
+        - ${{ if eq(parameters.publishingInfraVersion, 4) }}:
+          - task: DownloadPipelineArtifact@2
+            displayName: Download Pipeline Artifacts (V4)
+            inputs:
+              itemPattern: '*/assets/**'
+              targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload'
+          - task: CopyFiles@2
+            displayName: Flatten assets to BlobArtifacts
+            inputs:
+              SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload'
+              Contents: '**/*'
+              TargetFolder: '$(Build.ArtifactStagingDirectory)/BlobArtifacts'
+              flattenFolders: true
 
         - task: PowerShell@2
           displayName: Validate
@@ -328,7 +371,7 @@ stages:
             scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1
             arguments: >
               -BuildId $(BARBuildId)
-              -PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
+              -PublishingInfraVersion 3
               -AzdoToken '$(System.AccessToken)'
               -WaitPublishingFinish true
               -RequireDefaultChannels ${{ parameters.requireDefaultChannels }}

eng/common/core-templates/post-build/setup-maestro-vars.yml

@@ -8,12 +8,11 @@ steps:
     - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
 
   - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}:
-    - task: DownloadBuildArtifacts@0
+    - task: DownloadPipelineArtifact@2
       displayName: Download Release Configs
       inputs:
-        buildType: current
         artifactName: ReleaseConfigs
-        checkDownloadedFiles: true
+        targetPath: '$(Build.StagingDirectory)/ReleaseConfigs'
 
   - task: AzureCLI@2
     name: setReleaseVars

eng/common/core-templates/steps/generate-sbom.yml

@@ -1,54 +1,14 @@
-# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated.
-# PackageName - The name of the package this SBOM represents.
-# PackageVersion - The version of the package this SBOM represents. 
-# ManifestDirPath - The path of the directory where the generated manifest files will be placed
-# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector.
-
 parameters:
-  PackageVersion: 10.0.0
-  BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts'
-  PackageName: '.NET'
-  ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom
-  IgnoreDirectories: ''
-  sbomContinueOnError: true
-  is1ESPipeline: false
-  # disable publishArtifacts if some other step is publishing the artifacts (like job.yml).
-  publishArtifacts: true
+  PackageVersion: unused
+  BuildDropPath: unused
+  PackageName: unused
+  ManifestDirPath: unused
+  IgnoreDirectories: unused
+  sbomContinueOnError: unused
+  is1ESPipeline: unused
+  publishArtifacts: unused
 
 steps:
-- task: PowerShell@2 
-  displayName: Prep for SBOM generation in (Non-linux)
-  condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin'))
-  inputs: 
-    filePath: ./eng/common/generate-sbom-prep.ps1
-    arguments: ${{parameters.manifestDirPath}}
-
-# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461
 - script: |
-    chmod +x ./eng/common/generate-sbom-prep.sh
-    ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}}
-  displayName: Prep for SBOM generation in (Linux)
-  condition: eq(variables['Agent.Os'], 'Linux')
-  continueOnError: ${{ parameters.sbomContinueOnError }}
-
-- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
-  displayName: 'Generate SBOM manifest'
-  continueOnError: ${{ parameters.sbomContinueOnError }}
-  inputs:
-      PackageName: ${{ parameters.packageName }}
-      BuildDropPath: ${{ parameters.buildDropPath }}
-      PackageVersion: ${{ parameters.packageVersion }}
-      ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME)
-      ${{ if ne(parameters.IgnoreDirectories, '') }}:
-        AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}'
-
-- ${{ if eq(parameters.publishArtifacts, 'true')}}:
-  - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
-    parameters:
-      is1ESPipeline: ${{ parameters.is1ESPipeline }}
-      args:
-        displayName: Publish SBOM manifest
-        continueOnError: ${{parameters.sbomContinueOnError}}
-        targetPath: '${{ parameters.manifestDirPath }}'
-        artifactName: $(ARTIFACT_NAME)
-
+    echo "##vso[task.logissue type=warning]Including generate-sbom.yml is deprecated, SBOM generation is handled 1ES PT now. Remove this include."
+  displayName: Issue generate-sbom.yml deprecation warning

eng/common/core-templates/steps/publish-logs.yml

@@ -50,13 +50,14 @@ steps:
     TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs'
   condition: always()
 
-- template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
   parameters:
     is1ESPipeline: ${{ parameters.is1ESPipeline }}
     args:
       displayName: Publish Logs
-      pathToPublish: '$(Build.ArtifactStagingDirectory)/PostBuildLogs'
-      publishLocation: Container
-      artifactName: PostBuildLogs
+      targetPath: '$(Build.ArtifactStagingDirectory)/PostBuildLogs'
+      artifactName: PostBuildLogs_${{ parameters.StageLabel }}_${{ parameters.JobLabel }}_Attempt$(System.JobAttempt)
       continueOnError: true
       condition: always()
+      retryCountOnTaskFailure: 10 # for any files being locked
+      isProduction: false # logs are non-production artifacts

eng/common/core-templates/steps/source-build.yml

@@ -62,4 +62,4 @@ steps:
       artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
       continueOnError: true
       condition: succeededOrFailed()
-      sbomEnabled: false  # we don't need SBOM for logs
+      isProduction: false # logs are non-production artifacts