Convert consolidated pipeline to a YAML based pipeline and enable nightly builds

eng/ci/consolidate-artifacts.yml

@@ -0,0 +1,80 @@
+schedules:
+- cron: "30 2 * * *"
+  displayName: Nightly Consolidation
+  always: true
+  branches:
+    include:
+      - main
+
+name: $(Build.SourceBranchName)_$(Build.Reason)
+
+trigger: none
+
+# This pipeline is triggered by the completion of scheduled builds
+# in the core-tools-inproc and core-tools-default pipelines
+resources:
+  repositories:
+  - repository: 1es
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+  - repository: eng
+    type: git
+    name: engineering
+    ref: refs/tags/release
+  pipelines:
+    - pipeline: core-tools-host
+      source: azure\azure-functions-core-tools\core-tools-host.official
+
+    - pipeline: core-tools-inproc
+      source: azure\azure-functions-core-tools\core-tools.official
+      branch: in-proc  # in-proc branch of core-tools.official
+      tags:
+      - nightly-build
+      trigger: 
+        branches:
+          include:
+            - in-proc
+
+    - pipeline: core-tools-default
+      source: azure\azure-functions-core-tools\core-tools.official
+      branch: main  # main branch of core-tools.official
+      tags:
+      - nightly-build
+      trigger:
+        branches:
+          include:
+            - main
+
+variables:
+- template: /ci/variables/cfs.yml@eng
+- name: archs
+  value: 'win-x64,win-x86,min.win-x64,osx-x64,linux-x64,osx-arm64,win-arm64,min.win-arm64,min.win-x86'
+
+extends:
+  template: v1/1ES.Official.PipelineTemplate.yml@1es
+  parameters:
+    featureFlags:
+      incrementalSDLBinaryAnalysis: true
+      incrementalSDLSourceAnalysis: true
+    pool:
+      name: 1es-pool-azfunc
+      image: 1es-windows-2022
+      os: windows
+    sdl:
+      codeql:
+         compiled:
+           enabled: true
+         runSourceLanguagesInSourceAnalysis: true
+
+    stages:
+    - stage: ConsolidateArticacts
+      displayName: "Assemble Artifacts"
+      jobs:
+      - template: /eng/ci/templates/official/jobs/process-nuget-package.yml@self
+
+      - ${{ each arch in split(variables.archs,',') }}:
+        - template: /eng/ci/templates/official/jobs/assemble-artifacts.yml@self
+          parameters:
+            arch: ${{ arch }}
+            displayName: ${{ replace(replace(arch, '-', ''), '.', '') }}

eng/ci/templates/official/jobs/assemble-artifacts.yml

@@ -0,0 +1,151 @@
+parameters:
+  - name: arch
+    type: string
+  - name: displayName
+    type: string
+
+jobs:
+- job: AssembleArtifact${{ parameters.displayName }}
+  displayName: "Assemble ${{ parameters.arch }}"
+  templateContext:
+    inputs:
+      - input: pipelineArtifact
+        pipeline: core-tools-host
+        artifactName: drop-coretools-host-linux-signed
+        targetPath: $(Pipeline.Workspace)\core-tools-host\drop-coretools-host-linux-signed
+      - input: pipelineArtifact
+        pipeline: core-tools-host
+        artifactName: drop-coretools-host-windows
+        targetPath: $(Pipeline.Workspace)\core-tools-host\drop-coretools-host-windows
+      - input: pipelineArtifact
+        pipeline: core-tools-inproc
+        artifactName: drop-inproc6
+        targetPath: $(Pipeline.Workspace)\core-tools-inproc\drop-inproc6
+      - input: pipelineArtifact
+        pipeline: core-tools-inproc
+        artifactName: drop-inproc8
+        targetPath: $(Pipeline.Workspace)\core-tools-inproc\drop-inproc8
+      - input: pipelineArtifact
+        pipeline: core-tools-default
+        artifactName: drop
+        targetPath: $(Pipeline.Workspace)\core-tools-default\drop
+
+    outputParentDirectory: $(Build.ArtifactStagingDirectory)
+    outputs:
+      - output: pipelineArtifact
+        displayName: Publish assembled artifact ${{ parameters.arch }}
+        targetPath: $(Build.ArtifactStagingDirectory)/core-tools
+        artifact: drop-${{ parameters.arch }}
+      - output: pipelineArtifact
+        displayName: Publish metadata.json
+        targetPath: $(Build.ArtifactStagingDirectory)/metadata.json
+        artifact: drop-metadata-json
+        condition: and(succeeded(), eq('${{ parameters.arch }}', 'min.win-x64'))
+
+  steps:
+  - checkout: self
+
+  - task: DotNetCoreCLI@2
+    displayName: "Run ArtifactAssembler"
+    inputs:
+      command: run
+      projects: "$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/Azure.Functions.ArtifactAssembler.csproj"
+      arguments: '-c release -- "Azure.Functions.Cli.${{ parameters.arch }}"'
+      workingDirectory: '$(Pipeline.Workspace)'
+
+  - ${{ if eq(parameters.arch, 'min.win-x64') }}:
+    - task: PowerShell@2
+      displayName: 'Generate metadata.json file'
+      inputs:
+        targetType: filePath
+        filePath: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/PipelineHelpers/generateMetadataFile.ps1'
+        arguments: '-StagingDirectory "$(Pipeline.Workspace)\staging"'
+        workingDirectory: '$(Pipeline.Workspace)'
+
+  - ${{ if startsWith(parameters.arch, 'win') }}:
+    - task: PowerShell@2
+      displayName: 'Generate MSI files'
+      inputs:
+        targetType: filePath
+        filePath: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/PipelineHelpers/generateMsiFiles.ps1'
+        arguments: '-ArtifactsPath "$(Pipeline.Workspace)\staging\coretools-cli"'
+        workingDirectory: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler'
+
+  - ${{ if startsWith(parameters.arch, 'win-x') }}:
+    - template: ci/sign-files.yml@eng
+      parameters:
+        displayName: Sign MSI files
+        folderPath: $(Pipeline.Workspace)\staging\coretools-cli
+        pattern: '*.msi'
+        signType: 'inline'
+        inlineOperation: |
+          [
+            {
+              "KeyCode": "CP-230012",
+              "OperationCode": "SigntoolSign",
+              "Parameters": {
+                "OpusName": "Microsoft",
+                "OpusInfo": "http://www.microsoft.com",
+                "FileDigest": "/fd \"SHA256\"",
+                "PageHash": "/NPH",
+                "TimeStamp": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256"
+              },
+              "ToolName": "sign",
+              "ToolVersion": "1.0"
+            },
+            {
+              "KeyCode": "CP-230012",
+              "OperationCode": "SigntoolVerify",
+              "Parameters": {},
+              "ToolName": "sign",
+              "ToolVersion": "1.0"
+            }
+          ]
+
+  - ${{ if eq(parameters.arch, 'min.win-x64') }}:
+    - task: PowerShell@2
+      displayName: 'Test Artifacts - Visual Studio'
+      inputs:
+        targetType: filePath
+        filePath: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/PipelineHelpers/testVsArtifacts.ps1'
+        arguments: '-StagingDirectory "$(Pipeline.Workspace)\staging\coretools-visualstudio"'
+        workingDirectory: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler'
+
+  - ${{ if startsWith(parameters.arch, 'win') }}:
+    - task: PowerShell@2
+      displayName: 'Test Artifacts'
+      inputs:
+        targetType: filePath
+        filePath: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/PipelineHelpers/testArtifacts.ps1'
+        arguments: '-StagingDirectory "$(Pipeline.Workspace)\staging\coretools-cli"'
+        workingDirectory: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler'
+
+  - task: DotNetCoreCLI@2
+    displayName: 'Zip Artifacts'
+    inputs:
+      command: run
+      projects: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/Azure.Functions.ArtifactAssembler.csproj'
+      arguments: '--no-build -c release -- zip'
+      workingDirectory: '$(Pipeline.Workspace)'
+
+  - task: PowerShell@2
+    displayName: 'Generate SHA files'
+    inputs:
+      targetType: filePath
+      filePath: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/PipelineHelpers/generateSha.ps1'
+      arguments: '-CurrentDirectory "$(Pipeline.Workspace)"'
+
+  - task: CopyFiles@2
+    displayName: 'Copy Files to Artifact Staging Directory'
+    inputs:
+      SourceFolder: '$(Pipeline.Workspace)/staging'
+      Contents: 'coretools-*/**'
+      TargetFolder: '$(Build.ArtifactStagingDirectory)/core-tools'
+
+  - ${{ if eq(parameters.arch, 'min.win-x64') }}:
+    - task: CopyFiles@2
+      displayName: 'Copy metadata.json to Artifact Staging Directory'
+      inputs:
+        SourceFolder: '$(Pipeline.Workspace)/staging'
+        Contents: 'metadata.json'
+        TargetFolder: '$(Build.ArtifactStagingDirectory)'

eng/ci/templates/official/jobs/build-test.yml

@@ -81,6 +81,10 @@ jobs:
   - pwsh: |
       .\check-vulnerabilities.ps1
     displayName: "Check for security vulnerabilities"
+  - ${{ if eq(variables['Build.Reason'], 'Schedule') }}:
+    - pwsh: |
+        # Check if this is a scheduled build
+        Write-Host "##vso[build.addbuildtag]nightly-build"
   - pwsh: |
       .\build.ps1
     env:

eng/ci/templates/official/jobs/process-nuget-package.yml

@@ -0,0 +1,23 @@
+jobs:
+- job: ProcessNuGetPackage
+  displayName: "Process NuGet Package"
+  templateContext:
+    inputs:
+      - input: pipelineArtifact
+        pipeline: core-tools-default
+        artifactName: drop
+        targetPath: $(Pipeline.Workspace)\core-tools-default\drop
+    outputs:
+      - output: pipelineArtifact
+        displayName: Publish nuget package
+        targetPath: $(Pipeline.Workspace)/nugetPackage
+        artifact: drop-nuget-package
+  steps:
+  - checkout: self
+
+  - task: PowerShell@2
+    displayName: 'Process and move nuget package to staging directory'
+    inputs:
+      targetType: filePath
+      filePath: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/PipelineHelpers/moveNugetPackage.ps1'
+      arguments: '-CurrentDirectory "$(Pipeline.Workspace)"'