From fb88dd60fb867aedc9fff1052c2e70d9d142e1c1 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 21 Mar 2024 00:02:45 +0000 Subject: [PATCH 01/16] Update dependencies from https://github.com/dotnet/arcade build Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Build.Tasks.Templating , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.RemoteExecutor From Version 8.0.0-beta.24113.2 -> To Version 8.0.0-beta.24165.4 --- eng/Version.Details.xml | 20 +- eng/Versions.props | 6 +- eng/common/SetupNugetSources.ps1 | 26 +- eng/common/templates-official/job/job.yml | 263 ++++++++++++++++ .../templates-official/job/onelocbuild.yml | 112 +++++++ .../job/publish-build-assets.yml | 155 ++++++++++ .../templates-official/job/source-build.yml | 67 ++++ .../job/source-index-stage1.yml | 68 +++++ .../templates-official/jobs/codeql-build.yml | 31 ++ eng/common/templates-official/jobs/jobs.yml | 97 ++++++ .../templates-official/jobs/source-build.yml | 46 +++ .../post-build/common-variables.yml | 22 ++ .../post-build/post-build.yml | 285 ++++++++++++++++++ .../post-build/setup-maestro-vars.yml | 70 +++++ .../post-build/trigger-subscription.yml | 13 + .../steps/add-build-to-channel.yml | 13 + .../templates-official/steps/build-reason.yml | 12 + .../steps/component-governance.yml | 13 + .../steps/execute-codeql.yml | 32 ++ .../templates-official/steps/execute-sdl.yml | 88 ++++++ .../steps/generate-sbom.yml | 48 +++ .../templates-official/steps/publish-logs.yml | 23 ++ .../templates-official/steps/retain-build.yml | 28 ++ .../steps/send-to-helix.yml | 91 ++++++ .../templates-official/steps/source-build.yml | 129 ++++++++ .../variables/pool-providers.yml | 45 +++ .../variables/sdl-variables.yml | 7 + eng/common/templates/job/job.yml | 4 + eng/common/templates/steps/generate-sbom.yml | 2 +- global.json | 4 +- 30 files changed, 1791 insertions(+), 29 deletions(-) create mode 100644 eng/common/templates-official/job/job.yml create mode 100644 eng/common/templates-official/job/onelocbuild.yml create mode 100644 eng/common/templates-official/job/publish-build-assets.yml create mode 100644 eng/common/templates-official/job/source-build.yml create mode 100644 eng/common/templates-official/job/source-index-stage1.yml create mode 100644 eng/common/templates-official/jobs/codeql-build.yml create mode 100644 eng/common/templates-official/jobs/jobs.yml create mode 100644 eng/common/templates-official/jobs/source-build.yml create mode 100644 eng/common/templates-official/post-build/common-variables.yml create mode 100644 eng/common/templates-official/post-build/post-build.yml create mode 100644 eng/common/templates-official/post-build/setup-maestro-vars.yml create mode 100644 eng/common/templates-official/post-build/trigger-subscription.yml create mode 100644 eng/common/templates-official/steps/add-build-to-channel.yml create mode 100644 eng/common/templates-official/steps/build-reason.yml create mode 100644 eng/common/templates-official/steps/component-governance.yml create mode 100644 eng/common/templates-official/steps/execute-codeql.yml create mode 100644 eng/common/templates-official/steps/execute-sdl.yml create mode 100644 eng/common/templates-official/steps/generate-sbom.yml create mode 100644 eng/common/templates-official/steps/publish-logs.yml create mode 100644 eng/common/templates-official/steps/retain-build.yml create mode 100644 eng/common/templates-official/steps/send-to-helix.yml create mode 100644 eng/common/templates-official/steps/source-build.yml create mode 100644 eng/common/templates-official/variables/pool-providers.yml create mode 100644 eng/common/templates-official/variables/sdl-variables.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 45bbb7c0029c..82bd68a70a68 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -376,26 +376,26 @@ https://github.com/dotnet/winforms abda8e3bfa78319363526b5a5f86863ec979940e - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + f311667e0587f19c3fa9553a909975662107a351 - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + f311667e0587f19c3fa9553a909975662107a351 - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + f311667e0587f19c3fa9553a909975662107a351 - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + f311667e0587f19c3fa9553a909975662107a351 - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + f311667e0587f19c3fa9553a909975662107a351 https://github.com/dotnet/extensions diff --git a/eng/Versions.props b/eng/Versions.props index 63beef18904f..6777a35a635a 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -161,9 +161,9 @@ 6.2.4 6.2.4 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 + 8.0.0-beta.24165.4 + 8.0.0-beta.24165.4 + 8.0.0-beta.24165.4 8.0.0-alpha.1.24168.2 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 6c65e81925f2..efa2fd72bfaa 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -48,12 +48,11 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern else { Write-Host "Package source $SourceName already present." } - - AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $password) { +function AddCredential($creds, $source, $username, $pwd) { # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -82,17 +81,18 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("key", "ClearTextPassword") $sourceElement.AppendChild($passwordElement) | Out-Null } - $passwordElement.SetAttribute("value", $Password) + + $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." ForEach ($PackageSource in $maestroPrivateSources) { Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd } } @@ -144,13 +144,13 @@ if ($disabledSources -ne $null) { $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password # 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $dotnetVersions = @('5','6','7','8') @@ -159,9 +159,9 @@ foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } } -$doc.Save($filename) +$doc.Save($filename) \ No newline at end of file diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml new file mode 100644 index 000000000000..a2709d10562c --- /dev/null +++ b/eng/common/templates-official/job/job.yml @@ -0,0 +1,263 @@ +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + templateContext: '' + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + enableBuildRetry: false + disableComponentGovernance: '' + componentGovernanceIgnoreDirectories: '' + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + preSteps: [] + runAsPublic: false +# Sbom related params + enableSbom: true + PackageVersion: 7.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@3 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: NuGetAuthenticate@1 + + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} + continueOnError: true + + - template: /eng/common/templates-official/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: 'artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: 'Publish logs' + continueOnError: true + condition: always() + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/templates-official/steps/generate-sbom.yml + parameters: + PackageVersion: ${{ parameters.packageVersion}} + BuildDropPath: ${{ parameters.buildDropPath }} + IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true \ No newline at end of file diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml new file mode 100644 index 000000000000..ba9ba4930329 --- /dev/null +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -0,0 +1,112 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml new file mode 100644 index 000000000000..53138622fe7a --- /dev/null +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -0,0 +1,155 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 + + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry + + variables: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/common-variables.yml + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(Build.BuildNumber) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force + $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + Add-Content -Path $filePath -Value $(BARBuildId) + Add-Content -Path $filePath -Value "$(DefaultChannels)" + Add-Content -Path $filePath -Value $(IsStableBuild) + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/templates-official/steps/publish-logs.yml + parameters: + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml new file mode 100644 index 000000000000..8aba3b44bb25 --- /dev/null +++ b/eng/common/templates-official/job/source-build.yml @@ -0,0 +1,67 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + image: 1es-mariner-2-pt + os: linux + + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates-official/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml new file mode 100644 index 000000000000..4b6337391708 --- /dev/null +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -0,0 +1,68 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + condition: '' + dependsOn: '' + pool: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET Core SDK 6 + inputs: + packageType: sdk + version: 6.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml new file mode 100644 index 000000000000..b68d3c2f3199 --- /dev/null +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -0,0 +1,31 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + +jobs: +- template: /eng/common/templates-official/jobs/jobs.yml + parameters: + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.109.0 + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml new file mode 100644 index 000000000000..857a0f8ba43e --- /dev/null +++ b/eng/common/templates-official/jobs/jobs.yml @@ -0,0 +1,97 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates-official/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - template: ../job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/templates-official/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml new file mode 100644 index 000000000000..08e5db9bb116 --- /dev/null +++ b/eng/common/templates-official/jobs/source-build.yml @@ -0,0 +1,46 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates-official/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml new file mode 100644 index 000000000000..c24193acfc98 --- /dev/null +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -0,0 +1,22 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro-prod.westus2.cloudapp.azure.com" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml new file mode 100644 index 000000000000..5c98fe1c0f3a --- /dev/null +++ b/eng/common/templates-official/post-build/post-build.yml @@ -0,0 +1,285 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: NuGet Validation + condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml new file mode 100644 index 000000000000..0c87f149a4ad --- /dev/null +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -0,0 +1,70 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + +steps: + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates-official/post-build/trigger-subscription.yml b/eng/common/templates-official/post-build/trigger-subscription.yml new file mode 100644 index 000000000000..da669030daf6 --- /dev/null +++ b/eng/common/templates-official/post-build/trigger-subscription.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml new file mode 100644 index 000000000000..f67a210d62f3 --- /dev/null +++ b/eng/common/templates-official/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/build-reason.yml b/eng/common/templates-official/steps/build-reason.yml new file mode 100644 index 000000000000..eba58109b52c --- /dev/null +++ b/eng/common/templates-official/steps/build-reason.yml @@ -0,0 +1,12 @@ +# build-reason.yml +# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons +# to include steps (',' separated). +parameters: + conditions: '' + steps: [] + +steps: + - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: + - ${{ parameters.steps }} + - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: + - ${{ parameters.steps }} diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml new file mode 100644 index 000000000000..0ecec47b0c91 --- /dev/null +++ b/eng/common/templates-official/steps/component-governance.yml @@ -0,0 +1,13 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-codeql.yml b/eng/common/templates-official/steps/execute-codeql.yml new file mode 100644 index 000000000000..9b4a5ffa30a7 --- /dev/null +++ b/eng/common/templates-official/steps/execute-codeql.yml @@ -0,0 +1,32 @@ +parameters: + # Language that should be analyzed. Defaults to csharp + language: csharp + # Build Commands + buildCommands: '' + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth + # diagnosis of problems with specific tool configurations. + publishGuardianDirectoryToPipeline: false + # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL + # parameters rather than relying on YAML. It may be better to use a local script, because you can + # reproduce results locally without piecing together a command based on the YAML. + executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + # optional: determines whether to continue the build if the step errors; + sdlContinueOnError: false + +steps: +- template: /eng/common/templates-official/steps/execute-sdl.yml + parameters: + overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} + executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} + overrideParameters: ${{ parameters.overrideParameters }} + additionalParameters: '${{ parameters.additionalParameters }} + -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")' + publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml new file mode 100644 index 000000000000..07426fde05d8 --- /dev/null +++ b/eng/common/templates-official/steps/execute-sdl.yml @@ -0,0 +1,88 @@ +parameters: + overrideGuardianVersion: '' + executeAllSdlToolsScript: '' + overrideParameters: '' + additionalParameters: '' + publishGuardianDirectoryToPipeline: false + sdlContinueOnError: false + condition: '' + +steps: +- task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: GuardianConnect + +- task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + +- ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian (Overridden) + +- ${{ if eq(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian + +- ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} + displayName: Execute SDL (Overridden) + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianCliLocation $(GuardianCliLocation) + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) + - task: DeleteFiles@1 + displayName: Delete Guardian dependencies to avoid uploading + inputs: + SourceFolder: $(Agent.BuildDirectory)/.gdn + Contents: | + c + i + condition: succeededOrFailed() + + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() + + # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration + # with the "SARIF SAST Scans Tab" Azure DevOps extension + - task: CopyFiles@2 + displayName: Copy SARIF files + inputs: + flattenFolders: true + sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ + contents: '**/*.sarif' + targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs + condition: succeededOrFailed() + + # Use PublishBuildArtifacts because the SARIF extension only checks this case + # see microsoft/sarif-azuredevops-extension#4 + - task: PublishBuildArtifacts@1 + displayName: Publish SARIF files to CodeAnalysisLogs container + inputs: + pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs + artifactName: CodeAnalysisLogs + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml new file mode 100644 index 000000000000..1bf43bf807af --- /dev/null +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -0,0 +1,48 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + +parameters: + PackageVersion: 8.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + IgnoreDirectories: '' + sbomContinueOnError: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + inputs: + targetPath: '${{parameters.manifestDirPath}}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml new file mode 100644 index 000000000000..04012fed182a --- /dev/null +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -0,0 +1,23 @@ +parameters: + StageLabel: '' + JobLabel: '' + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml new file mode 100644 index 000000000000..83d97a26a01f --- /dev/null +++ b/eng/common/templates-official/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml new file mode 100644 index 000000000000..3eb7e2d5f840 --- /dev/null +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -0,0 +1,91 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml new file mode 100644 index 000000000000..829f17c34d11 --- /dev/null +++ b/eng/common/templates-official/steps/source-build.yml @@ -0,0 +1,129 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + assetManifestFileName=SourceBuild_RidSpecific.xml + if [ '${{ parameters.platform.name }}' != '' ]; then + assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + $baseOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true \ + /p:AssetManifestFileName=$assetManifestFileName + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/source-build/self/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (Exclude upstream cache) + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml new file mode 100644 index 000000000000..beab7d1bfba0 --- /dev/null +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -0,0 +1,45 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates-official/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) +# +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2022-pt + +variables: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml new file mode 100644 index 000000000000..dbdd66d4a4b3 --- /dev/null +++ b/eng/common/templates-official/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index e24ca2f46f98..8ec5c4f2d9f9 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -15,6 +15,7 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' + templateContext: '' # Job base template specific parameters # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md @@ -68,6 +69,9 @@ jobs: ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index a06373f38fa5..2b21eae42732 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -5,7 +5,7 @@ # IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. parameters: - PackageVersion: 7.0.0 + PackageVersion: 8.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' PackageName: '.NET' ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom diff --git a/global.json b/global.json index e697669c8b29..88ba69a6c220 100644 --- a/global.json +++ b/global.json @@ -27,7 +27,7 @@ }, "msbuild-sdks": { "Yarn.MSBuild": "1.22.19", - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24113.2", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24113.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24165.4", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24165.4" } } From 0e40ad5e578df6cafbdca8d6fb209d41e6ab0ce2 Mon Sep 17 00:00:00 2001 From: Igor Velikorossov Date: Thu, 21 Mar 2024 17:13:08 +1100 Subject: [PATCH 02/16] Start converting to 1es pipelines - public --- .azure/pipelines/ci-public.yml | 1168 +++++++++------------- .azure/pipelines/jobs/codesign-xplat.yml | 4 +- .azure/pipelines/jobs/default-build.yml | 639 ++++++++---- 3 files changed, 912 insertions(+), 899 deletions(-) diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml index b184f7cef685..482d68cb1a60 100644 --- a/.azure/pipelines/ci-public.yml +++ b/.azure/pipelines/ci-public.yml @@ -9,7 +9,6 @@ trigger: include: - main - release/* - - internal/release/* # Run PR validation on all branches # This doesn't have any path exclusions, even for things like docs, because @@ -21,17 +20,6 @@ pr: include: - '*' -schedules: -- cron: 0 9 * * 1 - displayName: "Run CodeQL3000 weekly, Monday at 2:00 AM PDT" - branches: - include: - - release/2.1 - - release/6.0 - - release/7.0 - - main - always: true - parameters: # Choose whether to skip tests when running pipeline manually. - name: skipTests @@ -40,12 +28,6 @@ parameters: type: boolean # Parameters below are ignored in public builds. # -# Choose whether to run the CodeQL3000 tasks. -# Manual builds align w/ official builds unless this parameter is true. -- name: runCodeQL3000 - default: false - displayName: Run CodeQL3000 tasks - type: boolean # Choose whether to enable binlogs when running pipeline manually. # Binary logs are enabled by default in public builds and aren't designed to be disabled there. - name: produceBinlogs @@ -67,372 +49,268 @@ variables: - name: PostBuildSign value: true - name: _UseHelixOpenQueues - value: ${{ ne(variables['System.TeamProject'], 'internal') }} -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - - name: enableSourceIndex - value: true -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: _BuildArgs - value: /p:TeamName=$(_TeamName) - /p:OfficialBuildId=$(Build.BuildNumber) - /p:SkipTestBuild=true - /p:PostBuildSign=$(PostBuildSign) - # DotNet-Blob-Feed provides: dotnetfeed-storage-access-key-1 - # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT - - group: Publish-Build-Assets - # The following extra properties are not set when testing. Use with final build.[cmd,sh] of asset-producing jobs. - - name: _PublishArgs - value: /p:Publish=true - /p:GenerateChecksums=true - /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) - - ${{ if ne(parameters.produceBinlogs, 'true') }}: - # Do not log most Windows steps in official builds; this is the slowest job. Site extensions step always logs. - - name: WindowsArm64LogArgs - value: -ExcludeCIBinaryLog - - name: Windows64LogArgs - value: -ExcludeCIBinaryLog - - name: Windows86LogArgs - value: -ExcludeCIBinaryLog - - name: WindowsSignLogArgs - value: -ExcludeCIBinaryLog - - name: WindowsInstallersLogArgs - value: -ExcludeCIBinaryLog - - name: WindowsArm64InstallersLogArgs - value: -ExcludeCIBinaryLog -- ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + value: 'true' +- ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - name: _BuildArgs value: '/p:SkipTestBuild=true /p:PostBuildSign=$(PostBuildSign)' - name: _PublishArgs value: '' -- ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'), eq(parameters.produceBinlogs, 'true')) }}: - # Write binary logs for all main Windows build steps except the x86 one in public and PR builds. - - name: WindowsArm64LogArgs - value: /bl:artifacts/log/Release/Build.arm64.binlog - - name: Windows64LogArgs - value: /bl:artifacts/log/Release/Build.x64.binlog - - name: Windows86LogArgs - value: -ExcludeCIBinaryLog - - name: WindowsSignLogArgs - value: /bl:artifacts/log/Release/Build.CodeSign.binlog - - name: WindowsInstallersLogArgs - value: /bl:artifacts/log/Release/Build.Installers.binlog - - name: WindowsArm64InstallersLogArgs - value: /bl:artifacts/log/Release/Build.Installers.Arm64.binlog -- ${{ if ne(variables['System.TeamProject'], 'internal') }}: - - name: _SignType - value: '' - - name: _InternalRuntimeDownloadArgs - value: '' - - name: _InternalRuntimeDownloadCodeSignArgs - value: '' -- ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBuilds storage account read tokens - - name: _InternalRuntimeDownloadArgs - value: -RuntimeSourceFeed https://dotnetbuilds.blob.core.windows.net/internal - -RuntimeSourceFeedKey $(dotnetbuilds-internal-container-read-token-base64) - /p:DotNetAssetRootAccessTokenSuffix='$(dotnetbuilds-internal-container-read-token-base64)' - # The code signing doesn't use the aspnet build scripts, so the msbuild parameters have to be passed directly. This - # is awkward but necessary because the eng/common/ build scripts don't add the msbuild properties automatically. - - name: _InternalRuntimeDownloadCodeSignArgs - value: $(_InternalRuntimeDownloadArgs) - /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal - /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - - group: DotNet-HelixApi-Access - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - - name: _SignType - value: real - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - name: _SignType - value: test -- name: runCodeQL3000 - value: ${{ or(eq(variables['Build.Reason'], 'Schedule'), and(eq(variables['Build.Reason'], 'Manual'), eq(parameters.runCodeQL3000, 'true'))) }} +# Write binary logs for all main Windows build steps except the x86 one in public and PR builds. +- name: WindowsArm64LogArgs + value: /bl:artifacts/log/Release/Build.arm64.binlog +- name: Windows64LogArgs + value: /bl:artifacts/log/Release/Build.x64.binlog +- name: Windows86LogArgs + value: -ExcludeCIBinaryLog +- name: WindowsSignLogArgs + value: /bl:artifacts/log/Release/Build.CodeSign.binlog +- name: WindowsInstallersLogArgs + value: /bl:artifacts/log/Release/Build.Installers.binlog +- name: WindowsArm64InstallersLogArgs + value: /bl:artifacts/log/Release/Build.Installers.Arm64.binlog +- name: _SignType + value: '' +- name: _InternalRuntimeDownloadArgs + value: '' +- name: _InternalRuntimeDownloadCodeSignArgs + value: '' - template: /eng/common/templates/variables/pool-providers.yml stages: - stage: build displayName: Build jobs: - - ${{ if and(ne(variables['System.TeamProject'], 'public'), eq(variables.runCodeQL3000, 'true')) }}: - - template: jobs/default-build.yml - parameters: - jobName: build - jobDisplayName: Build and run CodeQL3000 - agentOs: Windows - codeSign: false - # Component governance and SBOM creation are not needed here. Disable what Arcade would inject. - disableComponentGovernance: true - enableSbom: false - variables: - # Security analysis is included in normal runs. Disable its auto-injection. - - skipNugetSecurityAnalysis: true - # Do not let CodeQL3000 Extension gate scan frequency. - - Codeql.Cadence: 0 - # Enable CodeQL3000 unconditionally so it may be run on any branch. - - Codeql.Enabled: true - # Ignore the small amount of infrastructure Python code in this repo. - - Codeql.Language: cpp,csharp,java,javascript - - Codeql.ExcludePathPatterns: submodules - # Ignore test and infrastructure code. - - Codeql.SourceRoot: src - # CodeQL3000 needs this plumbed along as a variable to enable TSA. - - Codeql.TSAEnabled: ${{ eq(variables['Build.Reason'], 'Schedule') }} - # Default expects tsaoptions.json under SourceRoot. - - Codeql.TSAOptionsPath: '$(Build.SourcesDirectory)/.config/tsaoptions.json' - beforeBuild: - - task: CodeQL3000Init@0 - displayName: CodeQL Initialize - - script: "echo ##vso[build.addbuildtag]CodeQL3000" - displayName: 'Set CI CodeQL3000 tag' - condition: ne(variables.CODEQL_DIST,'') - steps: - - script: ./eng/build.cmd - -ci - -arch x64 - -all - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - /p:UseSharedCompilation=false - displayName: Build x64 - afterBuild: - - task: CodeQL3000Finalize@0 - displayName: CodeQL Finalize - artifacts: - - name: Build_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - - ${{ else }}: # regular build - # Code check - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual')) }}: - - template: jobs/default-build.yml - parameters: - jobName: Code_check - jobDisplayName: Code check - agentOs: Windows - steps: - - powershell: ./eng/scripts/CodeCheck.ps1 -ci $(_InternalRuntimeDownloadArgs) - displayName: Run eng/scripts/CodeCheck.ps1 - artifacts: - - name: Code_Check_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - # Build Windows (x64/x86/arm64) - - template: jobs/default-build.yml - parameters: - codeSign: true - jobName: Windows_build - jobDisplayName: "Build: Windows x64/x86/arm64" - agentOs: Windows - steps: - - ${{ if notIn(variables['Build.Reason'], 'PullRequest') }}: - - script: "echo ##vso[build.addbuildtag]daily-build" - displayName: 'Set CI daily-build tag' - - # !!! NOTE !!! Some of these steps have disabled code signing. - # This is intentional to workaround https://github.com/dotnet/arcade/issues/1957 which always re-submits for code-signing, even - # if they have already been signed. This results in slower builds due to re-submitting the same .nupkg many times for signing. - # The sign settings have been configured to - - script: ./eng/build.cmd - -ci - -arch x64 - -pack - -all - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - $(Windows64LogArgs) - displayName: Build x64 - - # Build the x86 shared framework - # This is going to actually build x86 native assets. - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -arch x86 - -pack - -all - -noBuildJava - -noBuildNative - /p:OnlyPackPlatformSpecificPackages=true - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - $(Windows86LogArgs) - displayName: Build x86 - - # Build the arm64 shared framework - - script: ./eng/build.cmd + # Code check + - template: jobs/default-build.yml + parameters: + jobName: Code_check + jobDisplayName: Code check + agentOs: Windows + steps: + - powershell: ./eng/scripts/CodeCheck.ps1 -ci $(_InternalRuntimeDownloadArgs) + displayName: Run eng/scripts/CodeCheck.ps1 + artifacts: + - name: Code_Check_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + + # Build Windows (x64/x86/arm64) + - template: jobs/default-build.yml + parameters: + codeSign: true + jobName: Windows_build + jobDisplayName: "Build: Windows x64/x86/arm64" + agentOs: Windows + steps: + - ${{ if notIn(variables['Build.Reason'], 'PullRequest') }}: + - script: "echo ##vso[build.addbuildtag]daily-build" + displayName: 'Set CI daily-build tag' + + # !!! NOTE !!! Some of these steps have disabled code signing. + # This is intentional to workaround https://github.com/dotnet/arcade/issues/1957 which always re-submits for code-signing, even + # if they have already been signed. This results in slower builds due to re-submitting the same .nupkg many times for signing. + # The sign settings have been configured to + - script: ./eng/build.cmd + -ci + -arch x64 + -pack + -all + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(Windows64LogArgs) + displayName: Build x64 + + # Build the x86 shared framework + # This is going to actually build x86 native assets. + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -arch x86 + -pack + -all + -noBuildJava + -noBuildNative + /p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(Windows86LogArgs) + displayName: Build x86 + + # Build the arm64 shared framework + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -arch arm64 + -sign + -pack + -noBuildJava + -noBuildNative + /p:DotNetSignType=$(_SignType) + /p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(WindowsArm64LogArgs) + displayName: Build ARM64 + + # Submit a manual build (in public or internal project) to validate changes to site extensions. + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - script: .\src\SiteExtensions\build.cmd -ci -noBuildRepoTasks - -arch arm64 - -sign -pack - -noBuildJava + -noBuildDeps -noBuildNative - /p:DotNetSignType=$(_SignType) - /p:OnlyPackPlatformSpecificPackages=true $(_BuildArgs) $(_InternalRuntimeDownloadArgs) - $(WindowsArm64LogArgs) - displayName: Build ARM64 - - # Submit a manual build (in public or internal project) to validate changes to site extensions. - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - script: .\src\SiteExtensions\build.cmd - -ci - -noBuildRepoTasks - -pack - -noBuildDeps - -noBuildNative - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - displayName: Build SiteExtension - - # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If - # https://github.com/dotnet/arcade/issues/1957 is resolved, consider running code-signing inline with the other - # previous steps. Sign check is disabled because it is run in a separate step below, after installers are built. - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -noBuildNative - -noBuild - -sign - /p:DotNetSignType=$(_SignType) - $(_BuildArgs) - $(WindowsSignLogArgs) - displayName: Code sign packages - - # Windows installers bundle x86/x64/arm64 assets - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -sign - -buildInstallers - -noBuildNative - /p:DotNetSignType=$(_SignType) - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - $(WindowsInstallersLogArgs) - displayName: Build Installers - - # Windows installers bundle and sharedfx msi for arm64 - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -arch arm64 - -sign - -buildInstallers - -noBuildNative - /p:DotNetSignType=$(_SignType) - /p:AssetManifestFileName=aspnetcore-win.xml - $(_BuildArgs) - $(_PublishArgs) - /p:PublishInstallerBaseVersion=true - $(_InternalRuntimeDownloadArgs) - $(WindowsArm64InstallersLogArgs) - displayName: Build ARM64 Installers - - artifacts: - - name: Windows_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Windows_Packages - path: artifacts/packages/ - - name: Windows_HostingBundle - path: artifacts/bin/WindowsHostingBundle - - name: Windows_ANCM_Msi - path: artifacts/bin/ANCMv2 - - name: Windows_ANCMIISExpress_Msi - path: artifacts/bin/AncmIISExpressV2 - - # Build MacOS arm64 - - template: jobs/default-build.yml - parameters: - jobName: MacOs_arm64_build - jobDisplayName: "Build: macOS arm64" - agentOs: macOs - buildArgs: - --arch arm64 - --pack - --all + displayName: Build SiteExtension + + # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If + # https://github.com/dotnet/arcade/issues/1957 is resolved, consider running code-signing inline with the other + # previous steps. Sign check is disabled because it is run in a separate step below, after installers are built. + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -noBuildNative + -noBuild + -sign + /p:DotNetSignType=$(_SignType) + $(_BuildArgs) + $(WindowsSignLogArgs) + displayName: Code sign packages + + # Windows installers bundle x86/x64/arm64 assets + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -sign + -buildInstallers + -noBuildNative + /p:DotNetSignType=$(_SignType) + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(WindowsInstallersLogArgs) + displayName: Build Installers + + # Windows installers bundle and sharedfx msi for arm64 + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -arch arm64 + -sign + -buildInstallers + -noBuildNative + /p:DotNetSignType=$(_SignType) + /p:AssetManifestFileName=aspnetcore-win.xml + $(_BuildArgs) + $(_PublishArgs) + /p:PublishInstallerBaseVersion=true + $(_InternalRuntimeDownloadArgs) + $(WindowsArm64InstallersLogArgs) + displayName: Build ARM64 Installers + + artifacts: + - name: Windows_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Windows_Packages + path: artifacts/packages/ + - name: Windows_HostingBundle + path: artifacts/bin/WindowsHostingBundle + - name: Windows_ANCM_Msi + path: artifacts/bin/ANCMv2 + - name: Windows_ANCMIISExpress_Msi + path: artifacts/bin/AncmIISExpressV2 + + # Build MacOS arm64 + - template: jobs/default-build.yml + parameters: + jobName: MacOs_arm64_build + jobDisplayName: "Build: macOS arm64" + agentOs: macOs + timeoutInMinutes: 90 + buildArgs: + --arch arm64 + --pack + --all --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-MacOS_arm64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-MacOS_arm64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) installNodeJs: false - artifacts: - - name: MacOS_arm64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: MacOS_arm64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: MacOS_arm64 - - # Build MacOS x64 - - template: jobs/default-build.yml + artifacts: + - name: MacOS_arm64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: MacOS_arm64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml parameters: - jobName: MacOs_x64_build - jobDisplayName: "Build: macOS x64" - agentOs: macOs - buildArgs: - --pack - --all + inputName: MacOS_arm64 + + # Build MacOS x64 + - template: jobs/default-build.yml + parameters: + jobName: MacOs_x64_build + jobDisplayName: "Build: macOS x64" + agentOs: macOs + timeoutInMinutes: 90 + buildArgs: + --pack + --all --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-MacOS_x64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-MacOS_x64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) installNodeJs: false - artifacts: - - name: MacOS_x64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: MacOS_x64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: MacOS_x64 - - # Build Linux x64 - - template: jobs/default-build.yml + artifacts: + - name: MacOS_x64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: MacOS_x64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml parameters: - jobName: Linux_x64_build - jobDisplayName: "Build: Linux x64" - agentOs: Linux - useHostedUbuntu: false - steps: - - script: ./eng/build.sh - --ci - --arch x64 - --pack - --all + inputName: MacOS_x64 + + # Build Linux x64 + - template: jobs/default-build.yml + parameters: + jobName: Linux_x64_build + jobDisplayName: "Build: Linux x64" + agentOs: Linux + useHostedUbuntu: false + steps: + - script: ./eng/build.sh + --ci + --arch x64 + --pack + --all --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - displayName: Run build.sh + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Run build.sh - script: git clean -xfd src/**/obj/; ./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=deb $(_BuildArgs) $(_InternalRuntimeDownloadArgs) - displayName: Build Debian installers + displayName: Build Debian installers - script: git clean -xfd src/**/obj/; ./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm @@ -440,70 +318,70 @@ stages: $(_BuildArgs) $(_PublishArgs) $(_InternalRuntimeDownloadArgs) - displayName: Build RPM installers + displayName: Build RPM installers installNodeJs: false - artifacts: - - name: Linux_x64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_x64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_x64 - - # Build Linux ARM - - template: jobs/default-build.yml + artifacts: + - name: Linux_x64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_x64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml parameters: - jobName: Linux_arm_build - jobDisplayName: "Build: Linux ARM" - agentOs: Linux - buildArgs: - --arch arm - --pack - --all + inputName: Linux_x64 + + # Build Linux ARM + - template: jobs/default-build.yml + parameters: + jobName: Linux_arm_build + jobDisplayName: "Build: Linux ARM" + agentOs: Linux + buildArgs: + --arch arm + --pack + --all --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-Linux_arm.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_arm.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) installNodeJs: false - artifacts: - - name: Linux_arm_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_arm_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_arm - - # Build Linux ARM64 - - template: jobs/default-build.yml + artifacts: + - name: Linux_arm_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_arm_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml parameters: - jobName: Linux_arm64_build - jobDisplayName: "Build: Linux ARM64" - agentOs: Linux - steps: - - script: ./eng/build.sh - --ci - --arch arm64 - --pack - --all + inputName: Linux_arm + + # Build Linux ARM64 + - template: jobs/default-build.yml + parameters: + jobName: Linux_arm64_build + jobDisplayName: "Build: Linux ARM64" + agentOs: Linux + steps: + - script: ./eng/build.sh + --ci + --arch arm64 + --pack + --all --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - displayName: Run build.sh + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Run build.sh - script: git clean -xfd src/**/obj/; ./dockerbuild.sh rhel --ci --nobl --arch arm64 --build-installers --no-build-deps --no-build-nodejs -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm @@ -511,296 +389,230 @@ stages: $(_BuildArgs) $(_PublishArgs) $(_InternalRuntimeDownloadArgs) - displayName: Build RPM installers + displayName: Build RPM installers installNodeJs: false - artifacts: - - name: Linux_arm64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_arm64_Packages - path: artifacts/packages/ + artifacts: + - name: Linux_arm64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_arm64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml + parameters: + inputName: Linux_arm64 - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_arm64 + # Build Linux Musl x64 + - template: jobs/default-build.yml + parameters: + jobName: Linux_musl_x64_build + jobDisplayName: "Build: Linux Musl x64" + agentOs: Linux + container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + buildArgs: + --arch x64 + --os-name linux-musl + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_musl_x64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false + disableComponentGovernance: true + artifacts: + - name: Linux_musl_x64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_musl_x64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml + parameters: + inputName: Linux_musl_x64 - # Build Linux Musl x64 - - template: jobs/default-build.yml + # Build Linux Musl ARM + - template: jobs/default-build.yml + parameters: + jobName: Linux_musl_arm_build + jobDisplayName: "Build: Linux Musl ARM" + agentOs: Linux + useHostedUbuntu: false + container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + buildArgs: + --arch arm + --os-name linux-musl + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_musl_arm.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false + artifacts: + - name: Linux_musl_arm_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_musl_arm_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml parameters: - jobName: Linux_musl_x64_build - jobDisplayName: "Build: Linux Musl x64" - agentOs: Linux - container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-WithNode - buildArgs: - --arch x64 - --os-name linux-musl - --pack - --all + inputName: Linux_musl_arm + + # Build Linux Musl ARM64 + - template: jobs/default-build.yml + parameters: + jobName: Linux_musl_arm64_build + jobDisplayName: "Build: Linux Musl ARM64" + agentOs: Linux + useHostedUbuntu: false + container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm64-alpine + buildArgs: + --arch arm64 + --os-name linux-musl + --pack + --all --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-Linux_musl_x64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_musl_arm64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) installNodeJs: false - disableComponentGovernance: true + artifacts: + - name: Linux_musl_arm64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_musl_arm64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml + parameters: + inputName: Linux_musl_arm64 + + - ${{ if ne(parameters.skipTests, 'true') }}: + # Test jobs + - template: jobs/default-build.yml + parameters: + jobName: Windows_Test + jobDisplayName: "Test: Windows Server x64" + agentOs: Windows + isAzDOTestingJob: true + # Just uploading artifacts/logs/ files can take 15 minutes. Doubling the cancel timeout for this job. + cancelTimeoutInMinutes: 30 + buildArgs: -all -pack -test -binaryLog /p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true + /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false /p:RunBlazorPlaywrightTemplateTests=true + $(_InternalRuntimeDownloadArgs) + beforeBuild: + - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1" + displayName: Setup IISExpress test certificates and schema artifacts: - - name: Linux_musl_x64_Logs + - name: Windows_Test_Logs path: artifacts/log/ publishOnError: true includeForks: true - - name: Linux_musl_x64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_musl_x64 + - name: Windows_Test_Results + path: artifacts/TestResults/ + publishOnError: true + includeForks: true - # Build Linux Musl ARM - template: jobs/default-build.yml parameters: - jobName: Linux_musl_arm_build - jobDisplayName: "Build: Linux Musl ARM" - agentOs: Linux - useHostedUbuntu: false - container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm-alpine - buildArgs: - --arch arm - --os-name linux-musl - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-Linux_musl_arm.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false + jobName: MacOS_Test + jobDisplayName: "Test: macOS" + agentOs: macOS + timeoutInMinutes: 90 + isAzDOTestingJob: true + buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) + beforeBuild: + - bash: "./eng/scripts/install-nginx-mac.sh" + displayName: Installing Nginx artifacts: - - name: Linux_musl_arm_Logs + - name: MacOS_Test_Logs path: artifacts/log/ publishOnError: true includeForks: true - - name: Linux_musl_arm_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_musl_arm + - name: MacOS_Test_Results + path: artifacts/TestResults/ + publishOnError: true + includeForks: true - # Build Linux Musl ARM64 - template: jobs/default-build.yml parameters: - jobName: Linux_musl_arm64_build - jobDisplayName: "Build: Linux Musl ARM64" + jobName: Linux_Test + jobDisplayName: "Test: Ubuntu x64" agentOs: Linux + isAzDOTestingJob: true useHostedUbuntu: false - container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-alpine - buildArgs: - --arch arm64 - --os-name linux-musl - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-Linux_musl_arm64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false + buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) + beforeBuild: + - bash: "./eng/scripts/install-nginx-linux.sh" + displayName: Installing Nginx + - bash: "echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p" + displayName: Increase inotify limit artifacts: - - name: Linux_musl_arm64_Logs + - name: Linux_Test_Logs path: artifacts/log/ publishOnError: true includeForks: true - - name: Linux_musl_arm64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_musl_arm64 - - - ${{ if and(ne(parameters.skipTests, 'true'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual'))) }}: - # Test jobs - - template: jobs/default-build.yml - parameters: - jobName: Windows_Test - jobDisplayName: "Test: Windows Server x64" - agentOs: Windows - isAzDOTestingJob: true - # Just uploading artifacts/logs/ files can take 15 minutes. Doubling the cancel timeout for this job. - cancelTimeoutInMinutes: 30 - buildArgs: -all -pack -test -binaryLog /p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true - /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false /p:RunBlazorPlaywrightTemplateTests=true - $(_InternalRuntimeDownloadArgs) - beforeBuild: - - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1" - displayName: Setup IISExpress test certificates and schema - artifacts: - - name: Windows_Test_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Windows_Test_Results - path: artifacts/TestResults/ - publishOnError: true - includeForks: true - - - template: jobs/default-build.yml - parameters: - jobName: MacOS_Test - jobDisplayName: "Test: macOS" - agentOs: macOS - timeoutInMinutes: 240 - isAzDOTestingJob: true - buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) - beforeBuild: - - bash: "./eng/scripts/install-nginx-mac.sh" - displayName: Installing Nginx - artifacts: - - name: MacOS_Test_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: MacOS_Test_Results - path: artifacts/TestResults/ - publishOnError: true - includeForks: true - - - template: jobs/default-build.yml - parameters: - jobName: Linux_Test - jobDisplayName: "Test: Ubuntu x64" - agentOs: Linux - isAzDOTestingJob: true - useHostedUbuntu: false - buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) - beforeBuild: - - bash: "./eng/scripts/install-nginx-linux.sh" - displayName: Installing Nginx - - bash: "echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p" - displayName: Increase inotify limit - artifacts: - - name: Linux_Test_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_Test_Results - path: artifacts/TestResults/ - publishOnError: true - includeForks: true - - # Helix x64 - - template: jobs/default-build.yml - parameters: - jobName: Helix_x64 - jobDisplayName: 'Tests: Helix x64' - agentOs: Windows - timeoutInMinutes: 240 - steps: - # Build the shared framework + - name: Linux_Test_Results + path: artifacts/TestResults/ + publishOnError: true + includeForks: true + + # Helix x64 + - template: jobs/default-build.yml + parameters: + jobName: Helix_x64 + jobDisplayName: 'Tests: Helix x64' + agentOs: Windows + timeoutInMinutes: 240 + steps: + # Build the shared framework - script: ./eng/build.cmd -ci -nobl -all -pack -arch x64 - /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) - displayName: Build shared fx - # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + displayName: Build shared fx + # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. - script: ./eng/build.cmd -ci -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test - -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true - /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) - displayName: Run build.cmd helix target - env: - HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues - SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops - - artifacts: - - name: Helix_logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - # Source build - - template: /eng/common/templates/job/source-build.yml - parameters: - platform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' - buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)' - skipPublishValidation: true - jobProperties: - timeoutInMinutes: 120 - variables: - # Log environment variables in binary logs to ease debugging - MSBUILDLOGALLENVIRONMENTVARIABLES: true - - - ${{ if eq(variables.enableSourceIndex, 'true') }}: - - template: /eng/common/templates/job/source-index-stage1.yml - parameters: - sourceIndexBuildCommand: ./eng/build.cmd -Configuration Release -ci -noBuildJava -binaryLog /p:OnlyPackPlatformSpecificPackages=true - binlogPath: artifacts/log/Release/Build.binlog - presteps: - - task: NodeTool@0 - displayName: Install Node 18.x - inputs: - versionSpec: 18.x - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2022 - - # Publish to the BAR and perform source indexing. Wait until everything else is done. - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/job/publish-build-assets.yml - parameters: - dependsOn: - - Windows_build - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - CodeSign_Xplat_MacOS_arm64 - - CodeSign_Xplat_MacOS_x64 - - CodeSign_Xplat_Linux_x64 - - CodeSign_Xplat_Linux_arm - - CodeSign_Xplat_Linux_arm64 - - CodeSign_Xplat_Linux_musl_x64 - - CodeSign_Xplat_Linux_musl_arm - - CodeSign_Xplat_Linux_musl_arm64 - - ${{ if eq(variables.PostBuildSign, 'true') }}: - - MacOs_arm64_build - - MacOs_x64_build - - Linux_x64_build - - Linux_arm_build - - Linux_arm64_build - - Linux_musl_x64_build - - Linux_musl_arm_build - - Linux_musl_arm64_build - # In addition to the dependencies above that provide assets, ensure the build was successful overall. - - ${{ if in(variables['Build.Reason'], 'Manual') }}: - - Code_check - - ${{ if ne(parameters.skipTests, 'true') }}: - - Windows_Test - - MacOS_Test - - Linux_Test - - Helix_x64 - - ${{ if eq(variables.enableSourceIndex, 'true') }}: - - SourceIndexStage1 - - Source_Build_Managed - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2019 - publishUsingPipelines: ${{ variables._PublishUsingPipelines }} - enablePublishBuildArtifacts: true # publish artifacts/log files - publishAssetsImmediately: true # Don't use a separate stage for darc publishing. - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.runCodeQL3000, 'true')) }}: - - template: /eng/common/templates/post-build/post-build.yml + -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + displayName: Run build.cmd helix target + env: + HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues + SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops + + artifacts: + - name: Helix_logs + path: artifacts/log/ + publishOnError: true + includeForks: true + + # Source build + - template: /eng/common/templates/job/source-build.yml parameters: - publishingInfraVersion: 3 - enableSymbolValidation: false - enableSigningValidation: false - enableNugetValidation: false - publishInstallersAndChecksums: true - publishAssetsImmediately: true + platform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)' + skipPublishValidation: true + jobProperties: + timeoutInMinutes: 120 + variables: + # Log environment variables in binary logs to ease debugging + MSBUILDLOGALLENVIRONMENTVARIABLES: true \ No newline at end of file diff --git a/.azure/pipelines/jobs/codesign-xplat.yml b/.azure/pipelines/jobs/codesign-xplat.yml index c96dc12f93ad..2be4f5b8fc77 100644 --- a/.azure/pipelines/jobs/codesign-xplat.yml +++ b/.azure/pipelines/jobs/codesign-xplat.yml @@ -3,7 +3,7 @@ parameters: inputName: '' jobs: -- template: default-build.yml +- template: default-build.yml@self parameters: codeSign: true dependsOn: @@ -15,7 +15,7 @@ jobs: installNodeJs: false installJdk: false steps: - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@2 displayName: Download ${{ parameters.inputName }} artifacts inputs: artifactName: ${{ parameters.inputName }}_Packages diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index 9c9b41d2415e..f37363f73344 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -79,156 +79,357 @@ parameters: cancelTimeoutInMinutes: 15 jobs: -- template: /eng/common/templates/job/job.yml - parameters: - name: ${{ coalesce(parameters.jobName, parameters.agentOs) }} - displayName: ${{ coalesce(parameters.jobDisplayName, parameters.agentOs) }} - dependsOn: ${{ parameters.dependsOn }} - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - ${{ if ne(parameters.enableRichCodeNavigation, '') }}: - enableRichCodeNavigation: true - richCodeNavigationLanguage: 'csharp,typescript,java' - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}: - enableMicrobuild: true - enablePublishBuildAssets: true - enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} - ${{ if and(eq(parameters.isAzDOTestingJob, true), ne(parameters.enablePublishTestResults, false)) }}: - enablePublishTestResults: true - testResultsFormat: xUnit # Have no vsTest results in any job. - mergeTestResults: true - testRunTitle: ${{ parameters.testRunTitle }} - enableSbom: ${{ parameters.enableSbom }} - enableTelemetry: true - helixRepo: dotnet/aspnetcore - helixType: build.product/ - workspace: - clean: all - # Map friendly OS names to the right queue - # See https://github.com/dotnet/arcade/blob/master/Documentation/ChoosingAMachinePool.md - pool: - ${{ if eq(parameters.agentOs, 'macOS') }}: - vmImage: macOS-11 - ${{ if eq(parameters.agentOs, 'Linux') }}: - ${{ if and(eq(parameters.useHostedUbuntu, true), or(ne(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'Manual', 'PullRequest', 'Schedule'))) }}: - vmImage: ubuntu-20.04 - ${{ if or(eq(parameters.useHostedUbuntu, false), and(eq(variables['System.TeamProject'], 'internal'), notin(variables['Build.Reason'], 'Manual', 'PullRequest', 'Schedule'))) }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: +- ${{ if ne(variables['System.TeamProject'], 'internal') }}: + - template: /eng/common/templates/job/job.yml@self + parameters: + name: ${{ coalesce(parameters.jobName, parameters.agentOs) }} + displayName: ${{ coalesce(parameters.jobDisplayName, parameters.agentOs) }} + dependsOn: ${{ parameters.dependsOn }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + ${{ if ne(parameters.enableRichCodeNavigation, '') }}: + enableRichCodeNavigation: true + richCodeNavigationLanguage: 'csharp,typescript,java' + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + ${{ if and(eq(parameters.isAzDOTestingJob, true), ne(parameters.enablePublishTestResults, false)) }}: + enablePublishTestResults: true + testResultsFormat: xUnit # Have no vsTest results in any job. + mergeTestResults: true + testRunTitle: ${{ parameters.testRunTitle }} + enableSbom: ${{ parameters.enableSbom }} + enableTelemetry: true + helixRepo: dotnet/aspnetcore + helixType: build.product/ + workspace: + clean: all + # Map friendly OS names to the right queue + # See https://github.com/dotnet/arcade/blob/master/Documentation/ChoosingAMachinePool.md + pool: + ${{ if eq(parameters.agentOs, 'macOS') }}: + vmImage: macOS-12 + ${{ if eq(parameters.agentOs, 'Linux') }}: + ${{ if and(eq(parameters.useHostedUbuntu, true), in(variables['Build.Reason'], 'Manual', 'PullRequest', 'Schedule')) }}: + vmImage: ubuntu-20.04 + ${{ if or(eq(parameters.useHostedUbuntu, false), notin(variables['Build.Reason'], 'Manual', 'PullRequest', 'Schedule')) }}: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals Build.Ubuntu.2004.Amd64.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals Build.Ubuntu.2004.Amd64 - ${{ if eq(parameters.agentOs, 'Windows') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: + ${{ if eq(parameters.agentOs, 'Windows') }}: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals 1es-windows-2022-open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + demands: ImageOverride -equals windows.vs2022preview.amd64.open + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + ${{ if ne(parameters.disableComponentGovernance, '') }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + variables: + - AgentOsName: ${{ parameters.agentOs }} + - ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping + - BuildScriptArgs: ${{ parameters.buildArgs }} + - _BuildConfig: ${{ parameters.configuration }} + - BuildConfiguration: ${{ parameters.configuration }} + - BuildDirectory: ${{ parameters.buildDirectory }} + - DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory) + - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + - TeamName: AspNetCore + - ${{ if eq(parameters.agentOs, 'Linux') }}: + - LC_ALL: 'en_US.UTF-8' + - LANG: 'en_US.UTF-8' + - LANGUAGE: 'en_US.UTF-8' + - ${{ if ne(parameters.codeSign, true) }}: + - _SignType: '' + - LC_ALL: 'en_US.UTF-8' + - LANG: 'en_US.UTF-8' + - LANGUAGE: 'en_US.UTF-8' + # Log environment variables in binary logs to ease debugging + - MSBUILDLOGALLENVIRONMENTVARIABLES: true + # Rely on task Arcade injects, not auto-injected build step. + - skipComponentGovernanceDetection: true + - ${{ each variable in parameters.variables }}: + # handle a variable list using "name" and "value" properties + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle name/value pairs (converting them into variable list entries) + # example: + # - [name]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + steps: + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: df -h + displayName: Disk size + - ${{ if eq(parameters.agentOs, 'macOS') }}: + - script: sudo xcode-select -s /Applications/Xcode_14.2.0.app/Contents/Developer + displayName: Use XCode 14.2.0 + - checkout: self + clean: true + - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: + - powershell: ./eng/scripts/InstallProcDump.ps1 + displayName: Install ProcDump + - powershell: ./eng/scripts/StartDumpCollectionForHangingBuilds.ps1 $(ProcDumpPath)procdump.exe artifacts/dumps/ (Get-Date).AddMinutes(160) dotnet + displayName: Start background dump collection + - ${{ if eq(parameters.installNodeJs, 'true') }}: + - task: NodeTool@0 + displayName: Install Node 20.x + inputs: + versionSpec: 20.x + - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: + - powershell: | + Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\" + ./eng/scripts/InstallGoogleChrome.ps1 + displayName: Install Chrome + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools" + displayName: Add dotnet tools to path + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: echo "##vso[task.prependpath]$(DOTNET_CLI_HOME)/.dotnet/tools" + displayName: Add dotnet tools to path + + - ${{ parameters.beforeBuild }} + + # Add COMPlus_* environment variables to build steps. + - ${{ if ne(parameters.steps, '')}}: + - ${{ each step in parameters.steps }}: + # Include all properties e.g. `task: CmdLine@2` or `displayName: Build x64` _except_ a provided `env:`. + # Aim here is to avoid having two `env:` properties in the expanded YAML. + - ${{ each pair in step }}: + ${{ if ne(pair.key, 'env') }}: + ${{ pair.key }}: ${{ pair.value }} + env: + # Include the variables we always want. + COMPlus_DbgEnableMiniDump: 1 + COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" + DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + # Expand provided `env:` properties, if any. + ${{ if step.env }}: + ${{ step.env }} + - ${{ if eq(parameters.steps, '')}}: + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - script: $(BuildDirectory)\build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) + /p:DotNetSignType=$(_SignType) + displayName: Run build.cmd + env: + COMPlus_DbgEnableMiniDump: 1 + COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" + DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: $(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs) + displayName: Run build.sh + env: + COMPlus_DbgEnableMiniDump: 1 + COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" + DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + + - ${{ parameters.afterBuild }} + + - ${{ if eq(parameters.agentOs, 'Linux') }}: + - script: df -h && du -h --threshold=50MB .. + displayName: Disk utilization + - ${{ if eq(parameters.agentOs, 'macOS') }}: + - script: df -h && du -h -d 3 .. + displayName: Disk utilization + + - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: + - powershell: ./eng/scripts/FinishDumpCollectionForHangingBuilds.ps1 artifacts/dumps/ + displayName: Finish background dump collection + continueOnError: true + condition: always() + + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - powershell: eng\scripts\KillProcesses.ps1 + displayName: Kill processes + continueOnError: true + condition: always() + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: eng/scripts/KillProcesses.sh + displayName: Kill processes + continueOnError: true + condition: always() + + - ${{ each artifact in parameters.artifacts }}: + - task: PublishBuildArtifacts@1 + displayName: Upload artifacts from ${{ artifact.path }} + condition: and(or(succeeded(), eq('${{ artifact.publishOnError }}', 'true')), or(eq(variables['system.pullrequest.isfork'], false), eq('${{ artifact.includeForks }}', 'true'))) + continueOnError: true + inputs: + # Assume runtime variable values are absolute paths already. + ${{ if startsWith(artifact.path, '$(') }}: + pathToPublish: ${{ artifact.path }} + ${{ if not(startsWith(artifact.path, '$(')) }}: + pathToPublish: $(Build.SourcesDirectory)/${{ artifact.path }} + ${{ if eq(artifact.name, '') }}: + artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration) + ${{ if ne(artifact.name, '') }}: + artifactName: ${{ artifact.name }} + artifactType: Container + parallel: true + + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - powershell: $(Build.SourcesDirectory)/eng/scripts/UploadCores.ps1 -ProcDumpOutputPath artifacts/dumps/ + condition: failed() + displayName: Upload cores + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: $(Build.SourcesDirectory)/eng/scripts/upload-cores.sh + condition: failed() + displayName: Upload cores + + - ${{ if and(eq(parameters.isAzDOTestingJob, true), ne(parameters.enablePublishTestResults, false)) }}: + - task: PublishTestResults@2 + displayName: Publish js test results + condition: always() + inputs: + testResultsFormat: JUnit + testResultsFiles: '**/artifacts/log/**/*.junit.xml' + testRunTitle: $(AgentOsName)-$(BuildConfiguration)-js + mergeTestResults: true + buildConfiguration: $(BuildConfiguration) + buildPlatform: $(AgentOsName) + +- ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - template: /eng/common/templates-official/job/job.yml@self + parameters: + name: ${{ coalesce(parameters.jobName, parameters.agentOs) }} + displayName: ${{ coalesce(parameters.jobDisplayName, parameters.agentOs) }} + dependsOn: ${{ parameters.dependsOn }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + ${{ if ne(parameters.enableRichCodeNavigation, '') }}: + enableRichCodeNavigation: true + richCodeNavigationLanguage: 'csharp,typescript,java' + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}: + enableMicrobuild: true + enablePublishBuildAssets: true + enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} + ${{ if and(eq(parameters.isAzDOTestingJob, true), ne(parameters.enablePublishTestResults, false)) }}: + enablePublishTestResults: true + testResultsFormat: xUnit # Have no vsTest results in any job. + mergeTestResults: true + testRunTitle: ${{ parameters.testRunTitle }} + enableSbom: ${{ parameters.enableSbom }} + enableTelemetry: true + helixRepo: dotnet/aspnetcore + helixType: build.product/ + workspace: + clean: all + # Map friendly OS names to the right queue + # See https://github.com/dotnet/arcade/blob/master/Documentation/ChoosingAMachinePool.md + pool: + ${{ if eq(parameters.agentOs, 'macOS') }}: + name: Azure Pipelines + image: macOS-12 + os: macOS + ${{ if eq(parameters.agentOs, 'Linux') }}: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2004 + os: linux + ${{ if eq(parameters.agentOs, 'Windows') }}: name: $(DncEngInternalBuildPool) # Visual Studio Enterprise - contains some stuff, like SQL Server and IIS Express, that we use for testing - demands: ImageOverride -equals 1es-windows-2022 - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - ${{ if ne(parameters.disableComponentGovernance, '') }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - variables: - - AgentOsName: ${{ parameters.agentOs }} - - ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping - - BuildScriptArgs: ${{ parameters.buildArgs }} - - _BuildConfig: ${{ parameters.configuration }} - - BuildConfiguration: ${{ parameters.configuration }} - - BuildDirectory: ${{ parameters.buildDirectory }} - - DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory) - - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - - TeamName: AspNetCore - - ${{ if eq(parameters.agentOs, 'Linux') }}: + image: windows.vs2022preview.amd64 + os: windows + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + ${{ if ne(parameters.disableComponentGovernance, '') }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + variables: + - AgentOsName: ${{ parameters.agentOs }} + - ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping + - BuildScriptArgs: ${{ parameters.buildArgs }} + - _BuildConfig: ${{ parameters.configuration }} + - BuildConfiguration: ${{ parameters.configuration }} + - BuildDirectory: ${{ parameters.buildDirectory }} + - DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory) + - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + - TeamName: AspNetCore + - ${{ if eq(parameters.agentOs, 'Linux') }}: + - LC_ALL: 'en_US.UTF-8' + - LANG: 'en_US.UTF-8' + - LANGUAGE: 'en_US.UTF-8' + - ${{ if eq(parameters.codeSign, true) }}: + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - _SignType: real + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - _SignType: test - LC_ALL: 'en_US.UTF-8' - LANG: 'en_US.UTF-8' - LANGUAGE: 'en_US.UTF-8' - - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: - - JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk\win-x64 - - ${{ if or(ne(parameters.codeSign, true), ne(variables['System.TeamProject'], 'internal')) }}: - - _SignType: '' - - ${{ if and(eq(parameters.codeSign, true), eq(variables['System.TeamProject'], 'internal')) }}: - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - - _SignType: real - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - _SignType: test - - LC_ALL: 'en_US.UTF-8' - - LANG: 'en_US.UTF-8' - - LANGUAGE: 'en_US.UTF-8' - # Log environment variables in binary logs to ease debugging - - MSBUILDLOGALLENVIRONMENTVARIABLES: true - # Rely on task Arcade injects, not auto-injected build step. - - skipComponentGovernanceDetection: true - - ${{ each variable in parameters.variables }}: - # handle a variable list using "name" and "value" properties - # example: - # - name: [key] - # value: [value] - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} + # Log environment variables in binary logs to ease debugging + - MSBUILDLOGALLENVIRONMENTVARIABLES: true + # Rely on task Arcade injects, not auto-injected build step. + - skipComponentGovernanceDetection: true + - ${{ each variable in parameters.variables }}: + # handle a variable list using "name" and "value" properties + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} - # handle variable groups - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} - # handle name/value pairs (converting them into variable list entries) - # example: - # - [name]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} - steps: - - ${{ if ne(parameters.agentOs, 'Windows') }}: - - script: df -h - displayName: Disk size - - ${{ if eq(parameters.agentOs, 'macOS') }}: - - script: sudo xcode-select -s /Applications/Xcode_12.5.1.app/Contents/Developer - displayName: Use XCode 12.5.1 - - checkout: self - clean: true - - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: - - powershell: ./eng/scripts/InstallProcDump.ps1 - displayName: Install ProcDump - - powershell: ./eng/scripts/StartDumpCollectionForHangingBuilds.ps1 $(ProcDumpPath)procdump.exe artifacts/dumps/ (Get-Date).AddMinutes(160) dotnet - displayName: Start background dump collection - - ${{ if eq(parameters.installNodeJs, 'true') }}: - - task: NodeTool@0 - displayName: Install Node 18.x - inputs: - versionSpec: 18.x - - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: - - powershell: ./eng/scripts/InstallJdk.ps1 - displayName: Install JDK 11 - - ${{ if eq(parameters.isAzDOTestingJob, true) }}: + # handle name/value pairs (converting them into variable list entries) + # example: + # - [name]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + steps: + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: df -h + displayName: Disk size + - ${{ if eq(parameters.agentOs, 'macOS') }}: + - script: sudo xcode-select -s /Applications/Xcode_14.2.0.app/Contents/Developer + displayName: Use XCode 14.2.0 + - checkout: self + clean: true + - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: + - powershell: ./eng/scripts/InstallProcDump.ps1 + displayName: Install ProcDump + - powershell: ./eng/scripts/StartDumpCollectionForHangingBuilds.ps1 $(ProcDumpPath)procdump.exe artifacts/dumps/ (Get-Date).AddMinutes(160) dotnet + displayName: Start background dump collection + - ${{ if eq(parameters.installNodeJs, 'true') }}: + - task: NodeTool@0 + displayName: Install Node 20.x + inputs: + versionSpec: 20.x + - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: - powershell: | Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\" ./eng/scripts/InstallGoogleChrome.ps1 displayName: Install Chrome - - ${{ if eq(parameters.agentOs, 'Windows') }}: - - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools" - displayName: Add dotnet tools to path - - ${{ if ne(parameters.agentOs, 'Windows') }}: - - script: echo "##vso[task.prependpath]$(DOTNET_CLI_HOME)/.dotnet/tools" - displayName: Add dotnet tools to path + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools" + displayName: Add dotnet tools to path + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: echo "##vso[task.prependpath]$(DOTNET_CLI_HOME)/.dotnet/tools" + displayName: Add dotnet tools to path - - ${{ parameters.beforeBuild }} + - ${{ parameters.beforeBuild }} - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - ${{ if eq(parameters.agentOs, 'Windows') }}: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) - ${{ if ne(parameters.agentOs, 'Windows') }}: - task: Bash@3 displayName: Setup Private Feeds Credentials @@ -238,100 +439,100 @@ jobs: env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - # Add COMPlus_* environment variables to build steps. - - ${{ if ne(parameters.steps, '')}}: - - ${{ each step in parameters.steps }}: - # Include all properties e.g. `task: CmdLine@2` or `displayName: Build x64` _except_ a provided `env:`. - # Aim here is to avoid having two `env:` properties in the expanded YAML. - - ${{ each pair in step }}: - ${{ if ne(pair.key, 'env') }}: - ${{ pair.key }}: ${{ pair.value }} + # Add COMPlus_* environment variables to build steps. + - ${{ if ne(parameters.steps, '')}}: + - ${{ each step in parameters.steps }}: + # Include all properties e.g. `task: CmdLine@2` or `displayName: Build x64` _except_ a provided `env:`. + # Aim here is to avoid having two `env:` properties in the expanded YAML. + - ${{ each pair in step }}: + ${{ if ne(pair.key, 'env') }}: + ${{ pair.key }}: ${{ pair.value }} + env: + # Include the variables we always want. + COMPlus_DbgEnableMiniDump: 1 + COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" + DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + # Expand provided `env:` properties, if any. + ${{ if step.env }}: + ${{ step.env }} + - ${{ if eq(parameters.steps, '')}}: + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - script: $(BuildDirectory)\build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) + /p:DotNetSignType=$(_SignType) + displayName: Run build.cmd + env: + COMPlus_DbgEnableMiniDump: 1 + COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" + DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: $(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs) + displayName: Run build.sh env: - # Include the variables we always want. COMPlus_DbgEnableMiniDump: 1 COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) - # Expand provided `env:` properties, if any. - ${{ if step.env }}: - ${{ step.env }} - - ${{ if eq(parameters.steps, '')}}: - - ${{ if eq(parameters.agentOs, 'Windows') }}: - - script: $(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) - /p:DotNetSignType=$(_SignType) - displayName: Run build.cmd - env: - COMPlus_DbgEnableMiniDump: 1 - COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" - DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) - - ${{ if ne(parameters.agentOs, 'Windows') }}: - - script: $(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs) - displayName: Run build.sh - env: - COMPlus_DbgEnableMiniDump: 1 - COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" - DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) - - ${{ parameters.afterBuild }} + - ${{ parameters.afterBuild }} - - ${{ if eq(parameters.agentOs, 'Linux') }}: - - script: df -h && du -h --threshold=50MB .. - displayName: Disk utilization - - ${{ if eq(parameters.agentOs, 'macOS') }}: - - script: df -h && du -h -d 3 .. - displayName: Disk utilization + - ${{ if eq(parameters.agentOs, 'Linux') }}: + - script: df -h && du -h --threshold=50MB .. + displayName: Disk utilization + - ${{ if eq(parameters.agentOs, 'macOS') }}: + - script: df -h && du -h -d 3 .. + displayName: Disk utilization - - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: - - powershell: ./eng/scripts/FinishDumpCollectionForHangingBuilds.ps1 artifacts/dumps/ - displayName: Finish background dump collection - continueOnError: true - condition: always() + - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: + - powershell: ./eng/scripts/FinishDumpCollectionForHangingBuilds.ps1 artifacts/dumps/ + displayName: Finish background dump collection + continueOnError: true + condition: always() - - ${{ if eq(parameters.agentOs, 'Windows') }}: - - powershell: eng\scripts\KillProcesses.ps1 - displayName: Kill processes - continueOnError: true - condition: always() - - ${{ if ne(parameters.agentOs, 'Windows') }}: - - script: eng/scripts/KillProcesses.sh - displayName: Kill processes - continueOnError: true - condition: always() + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - powershell: eng\scripts\KillProcesses.ps1 + displayName: Kill processes + continueOnError: true + condition: always() + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: eng/scripts/KillProcesses.sh + displayName: Kill processes + continueOnError: true + condition: always() - - ${{ each artifact in parameters.artifacts }}: - - task: PublishBuildArtifacts@1 - displayName: Upload artifacts from ${{ artifact.path }} - condition: and(or(succeeded(), eq('${{ artifact.publishOnError }}', 'true')), or(eq(variables['system.pullrequest.isfork'], false), eq('${{ artifact.includeForks }}', 'true'))) - continueOnError: true - inputs: - # Assume runtime variable values are absolute paths already. - ${{ if startsWith(artifact.path, '$(') }}: - pathToPublish: ${{ artifact.path }} - ${{ if not(startsWith(artifact.path, '$(')) }}: - pathToPublish: $(Build.SourcesDirectory)/${{ artifact.path }} - ${{ if eq(artifact.name, '') }}: - artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration) - ${{ if ne(artifact.name, '') }}: - artifactName: ${{ artifact.name }} - artifactType: Container - parallel: true + - ${{ each artifact in parameters.artifacts }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: Upload artifacts from ${{ artifact.path }} + condition: and(or(succeeded(), eq('${{ artifact.publishOnError }}', 'true')), or(eq(variables['system.pullrequest.isfork'], false), eq('${{ artifact.includeForks }}', 'true'))) + continueOnError: true + inputs: + # Assume runtime variable values are absolute paths already. + ${{ if startsWith(artifact.path, '$(') }}: + path: ${{ artifact.path }} + ${{ if not(startsWith(artifact.path, '$(')) }}: + path: $(Build.SourcesDirectory)/${{ artifact.path }} + ${{ if eq(artifact.name, '') }}: + artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration) + ${{ if ne(artifact.name, '') }}: + artifactName: ${{ artifact.name }} + artifactType: Container + parallel: true - - ${{ if eq(parameters.agentOs, 'Windows') }}: - - powershell: $(Build.SourcesDirectory)/eng/scripts/UploadCores.ps1 -ProcDumpOutputPath artifacts/dumps/ - condition: failed() - displayName: Upload cores - - ${{ if ne(parameters.agentOs, 'Windows') }}: - - script: $(Build.SourcesDirectory)/eng/scripts/upload-cores.sh - condition: failed() - displayName: Upload cores + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - powershell: $(Build.SourcesDirectory)/eng/scripts/UploadCores.ps1 -ProcDumpOutputPath artifacts/dumps/ + condition: failed() + displayName: Upload cores + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: $(Build.SourcesDirectory)/eng/scripts/upload-cores.sh + condition: failed() + displayName: Upload cores - - ${{ if and(eq(parameters.isAzDOTestingJob, true), ne(parameters.enablePublishTestResults, false)) }}: - - task: PublishTestResults@2 - displayName: Publish js test results - condition: always() - inputs: - testResultsFormat: JUnit - testResultsFiles: '**/artifacts/log/**/*.junit.xml' - testRunTitle: $(AgentOsName)-$(BuildConfiguration)-js - mergeTestResults: true - buildConfiguration: $(BuildConfiguration) - buildPlatform: $(AgentOsName) + - ${{ if and(eq(parameters.isAzDOTestingJob, true), ne(parameters.enablePublishTestResults, false)) }}: + - task: PublishTestResults@2 + displayName: Publish js test results + condition: always() + inputs: + testResultsFormat: JUnit + testResultsFiles: '**/artifacts/log/**/*.junit.xml' + testRunTitle: $(AgentOsName)-$(BuildConfiguration)-js + mergeTestResults: true + buildConfiguration: $(BuildConfiguration) + buildPlatform: $(AgentOsName) From 6d2a1c75f5fd8493bd2ecdde403dc0c87eab01f3 Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Thu, 21 Mar 2024 09:06:55 -0700 Subject: [PATCH 03/16] Install JDK --- .azure/pipelines/jobs/default-build.yml | 38 +++++++++++++------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index f37363f73344..1c04cda752b5 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -131,10 +131,8 @@ jobs: - DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory) - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - TeamName: AspNetCore - - ${{ if eq(parameters.agentOs, 'Linux') }}: - - LC_ALL: 'en_US.UTF-8' - - LANG: 'en_US.UTF-8' - - LANGUAGE: 'en_US.UTF-8' + - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: + - JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk\win-x64 - ${{ if ne(parameters.codeSign, true) }}: - _SignType: '' - LC_ALL: 'en_US.UTF-8' @@ -183,11 +181,14 @@ jobs: displayName: Install Node 20.x inputs: versionSpec: 20.x - - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: - - powershell: | - Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\" - ./eng/scripts/InstallGoogleChrome.ps1 - displayName: Install Chrome + - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: + - powershell: ./eng/scripts/InstallJdk.ps1 + displayName: Install JDK 11 + - ${{ if eq(parameters.isAzDOTestingJob, true) }}: + - powershell: | + Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\" + ./eng/scripts/InstallGoogleChrome.ps1 + displayName: Install Chrome - ${{ if eq(parameters.agentOs, 'Windows') }}: - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools" displayName: Add dotnet tools to path @@ -353,10 +354,8 @@ jobs: - DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory) - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - TeamName: AspNetCore - - ${{ if eq(parameters.agentOs, 'Linux') }}: - - LC_ALL: 'en_US.UTF-8' - - LANG: 'en_US.UTF-8' - - LANGUAGE: 'en_US.UTF-8' + - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: + - JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk\win-x64 - ${{ if eq(parameters.codeSign, true) }}: - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - _SignType: real @@ -408,11 +407,14 @@ jobs: displayName: Install Node 20.x inputs: versionSpec: 20.x - - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: - - powershell: | - Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\" - ./eng/scripts/InstallGoogleChrome.ps1 - displayName: Install Chrome + - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: + - powershell: ./eng/scripts/InstallJdk.ps1 + displayName: Install JDK 11 + - ${{ if eq(parameters.isAzDOTestingJob, true) }}: + - powershell: | + Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\" + ./eng/scripts/InstallGoogleChrome.ps1 + displayName: Install Chrome - ${{ if eq(parameters.agentOs, 'Windows') }}: - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools" displayName: Add dotnet tools to path From df9f4e8447ccc5797f8d5daf858c739f3631c2a7 Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Thu, 21 Mar 2024 09:14:58 -0700 Subject: [PATCH 04/16] Node 18 --- .azure/pipelines/jobs/default-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index 1c04cda752b5..616d2406334e 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -178,9 +178,9 @@ jobs: displayName: Start background dump collection - ${{ if eq(parameters.installNodeJs, 'true') }}: - task: NodeTool@0 - displayName: Install Node 20.x + displayName: Install Node 18.x inputs: - versionSpec: 20.x + versionSpec: 18.x - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: - powershell: ./eng/scripts/InstallJdk.ps1 displayName: Install JDK 11 @@ -404,9 +404,9 @@ jobs: displayName: Start background dump collection - ${{ if eq(parameters.installNodeJs, 'true') }}: - task: NodeTool@0 - displayName: Install Node 20.x + displayName: Install Node 18.x inputs: - versionSpec: 20.x + versionSpec: 18.x - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: - powershell: ./eng/scripts/InstallJdk.ps1 displayName: Install JDK 11 From 3fd75728b914f9b3ff4e9e511f3ede7d49c81a84 Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Thu, 21 Mar 2024 10:30:11 -0700 Subject: [PATCH 05/16] Fix spacing --- .azure/pipelines/ci-public.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml index 482d68cb1a60..25e7817735b3 100644 --- a/.azure/pipelines/ci-public.yml +++ b/.azure/pipelines/ci-public.yml @@ -241,7 +241,7 @@ stages: $(_BuildArgs) $(_PublishArgs) $(_InternalRuntimeDownloadArgs) - installNodeJs: false + installNodeJs: false artifacts: - name: MacOS_arm64_Logs path: artifacts/log/ @@ -272,7 +272,7 @@ stages: $(_BuildArgs) $(_PublishArgs) $(_InternalRuntimeDownloadArgs) - installNodeJs: false + installNodeJs: false artifacts: - name: MacOS_x64_Logs path: artifacts/log/ @@ -350,7 +350,7 @@ stages: $(_BuildArgs) $(_PublishArgs) $(_InternalRuntimeDownloadArgs) - installNodeJs: false + installNodeJs: false artifacts: - name: Linux_arm_Logs path: artifacts/log/ @@ -458,7 +458,7 @@ stages: $(_BuildArgs) $(_PublishArgs) $(_InternalRuntimeDownloadArgs) - installNodeJs: false + installNodeJs: false artifacts: - name: Linux_musl_arm_Logs path: artifacts/log/ @@ -492,7 +492,7 @@ stages: $(_BuildArgs) $(_PublishArgs) $(_InternalRuntimeDownloadArgs) - installNodeJs: false + installNodeJs: false artifacts: - name: Linux_musl_arm64_Logs path: artifacts/log/ From 47248eed721dfb4219bd27c1552208a889fb33c9 Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Thu, 21 Mar 2024 10:32:51 -0700 Subject: [PATCH 06/16] More spacing --- .azure/pipelines/ci-public.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml index 25e7817735b3..49f7b91a7a27 100644 --- a/.azure/pipelines/ci-public.yml +++ b/.azure/pipelines/ci-public.yml @@ -305,19 +305,19 @@ stages: $(_BuildArgs) $(_InternalRuntimeDownloadArgs) displayName: Run build.sh - - script: git clean -xfd src/**/obj/; - ./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs - -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=deb - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=deb + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) displayName: Build Debian installers - - script: git clean -xfd src/**/obj/; - ./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs - -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm - -p:AssetManifestFileName=aspnetcore-Linux_x64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm + -p:AssetManifestFileName=aspnetcore-Linux_x64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) displayName: Build RPM installers installNodeJs: false artifacts: From 079525fdc8c95bec32cd320498965dbac4ac9b4e Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Thu, 21 Mar 2024 10:35:25 -0700 Subject: [PATCH 07/16] More spacing again --- .azure/pipelines/ci-public.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml index 49f7b91a7a27..e9a41468b5ca 100644 --- a/.azure/pipelines/ci-public.yml +++ b/.azure/pipelines/ci-public.yml @@ -382,13 +382,13 @@ stages: $(_BuildArgs) $(_InternalRuntimeDownloadArgs) displayName: Run build.sh - - script: git clean -xfd src/**/obj/; - ./dockerbuild.sh rhel --ci --nobl --arch arm64 --build-installers --no-build-deps --no-build-nodejs - -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm - -p:AssetManifestFileName=aspnetcore-Linux_arm64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh rhel --ci --nobl --arch arm64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm + -p:AssetManifestFileName=aspnetcore-Linux_arm64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) displayName: Build RPM installers installNodeJs: false artifacts: @@ -585,13 +585,13 @@ stages: timeoutInMinutes: 240 steps: # Build the shared framework - - script: ./eng/build.cmd -ci -nobl -all -pack -arch x64 - /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + - script: ./eng/build.cmd -ci -nobl -all -pack -arch x64 + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) displayName: Build shared fx # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. - - script: ./eng/build.cmd -ci -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test - -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true - /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + - script: ./eng/build.cmd -ci -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test + -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) displayName: Run build.cmd helix target env: HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues From 7aff276d707ee3538215653cae231b632228e451 Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Thu, 21 Mar 2024 10:36:05 -0700 Subject: [PATCH 08/16] One last ride --- .azure/pipelines/ci-public.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml index e9a41468b5ca..06130afb1c54 100644 --- a/.azure/pipelines/ci-public.yml +++ b/.azure/pipelines/ci-public.yml @@ -609,7 +609,7 @@ stages: platform: name: 'Managed' container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' - buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)' + buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)' skipPublishValidation: true jobProperties: timeoutInMinutes: 120 From ab43f27afe5e395fb61a401b16691166573cafe6 Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Thu, 21 Mar 2024 10:42:07 -0700 Subject: [PATCH 09/16] More spacing fixes --- .azure/pipelines/ci-public.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml index 06130afb1c54..6d6692f75fd8 100644 --- a/.azure/pipelines/ci-public.yml +++ b/.azure/pipelines/ci-public.yml @@ -319,7 +319,7 @@ stages: $(_PublishArgs) $(_InternalRuntimeDownloadArgs) displayName: Build RPM installers - installNodeJs: false + installNodeJs: false artifacts: - name: Linux_x64_Logs path: artifacts/log/ @@ -390,7 +390,7 @@ stages: $(_PublishArgs) $(_InternalRuntimeDownloadArgs) displayName: Build RPM installers - installNodeJs: false + installNodeJs: false artifacts: - name: Linux_arm64_Logs path: artifacts/log/ From 1adfb081e6fe962b867ca88958c46b856499afdc Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Thu, 21 Mar 2024 11:02:16 -0700 Subject: [PATCH 10/16] Fix pools --- .azure/pipelines/jobs/default-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index 616d2406334e..331b0038921e 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -107,16 +107,16 @@ jobs: # See https://github.com/dotnet/arcade/blob/master/Documentation/ChoosingAMachinePool.md pool: ${{ if eq(parameters.agentOs, 'macOS') }}: - vmImage: macOS-12 + vmImage: macOS-11 ${{ if eq(parameters.agentOs, 'Linux') }}: - ${{ if and(eq(parameters.useHostedUbuntu, true), in(variables['Build.Reason'], 'Manual', 'PullRequest', 'Schedule')) }}: + ${{ if eq(parameters.useHostedUbuntu, true) }}: vmImage: ubuntu-20.04 - ${{ if or(eq(parameters.useHostedUbuntu, false), notin(variables['Build.Reason'], 'Manual', 'PullRequest', 'Schedule')) }}: + ${{ if eq(parameters.useHostedUbuntu, false) }}: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals Build.Ubuntu.2004.Amd64.Open ${{ if eq(parameters.agentOs, 'Windows') }}: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2022preview.amd64.open + demands: ImageOverride -equals 1es-windows-2022-open ${{ if ne(parameters.container, '') }}: container: ${{ parameters.container }} ${{ if ne(parameters.disableComponentGovernance, '') }}: From 0351f6a8152b33ac5e1df49ab2e6abf876fc1909 Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Thu, 21 Mar 2024 11:04:30 -0700 Subject: [PATCH 11/16] Fix xcode --- .azure/pipelines/jobs/default-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index 331b0038921e..4e6d0812b09f 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -167,8 +167,8 @@ jobs: - script: df -h displayName: Disk size - ${{ if eq(parameters.agentOs, 'macOS') }}: - - script: sudo xcode-select -s /Applications/Xcode_14.2.0.app/Contents/Developer - displayName: Use XCode 14.2.0 + - script: sudo xcode-select -s /Applications/Xcode_12.5.1.app/Contents/Developer + displayName: Use XCode 12.5.1 - checkout: self clean: true - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: @@ -393,8 +393,8 @@ jobs: - script: df -h displayName: Disk size - ${{ if eq(parameters.agentOs, 'macOS') }}: - - script: sudo xcode-select -s /Applications/Xcode_14.2.0.app/Contents/Developer - displayName: Use XCode 14.2.0 + - script: sudo xcode-select -s /Applications/Xcode_12.5.1.app/Contents/Developer + displayName: Use XCode 12.5.1 - checkout: self clean: true - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: From ed269f2bd9dd1b93867b445add862fad2cad0d53 Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Thu, 21 Mar 2024 11:14:53 -0700 Subject: [PATCH 12/16] Don't prepare the machine --- .azure/pipelines/jobs/default-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index 4e6d0812b09f..7a7324997c26 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -216,7 +216,7 @@ jobs: ${{ step.env }} - ${{ if eq(parameters.steps, '')}}: - ${{ if eq(parameters.agentOs, 'Windows') }}: - - script: $(BuildDirectory)\build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) + - script: $(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) /p:DotNetSignType=$(_SignType) displayName: Run build.cmd env: @@ -459,7 +459,7 @@ jobs: ${{ step.env }} - ${{ if eq(parameters.steps, '')}}: - ${{ if eq(parameters.agentOs, 'Windows') }}: - - script: $(BuildDirectory)\build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) + - script: $(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) /p:DotNetSignType=$(_SignType) displayName: Run build.cmd env: From 33588096ce4cf1a080730f1cb2ccf6369afb6c58 Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Thu, 21 Mar 2024 12:20:06 -0700 Subject: [PATCH 13/16] Enable internal builds --- .azure/pipelines/ci.yml | 1257 +++++++++++------------ .azure/pipelines/jobs/default-build.yml | 4 +- 2 files changed, 603 insertions(+), 658 deletions(-) diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index b184f7cef685..197e1a4984f2 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -21,17 +21,6 @@ pr: include: - '*' -schedules: -- cron: 0 9 * * 1 - displayName: "Run CodeQL3000 weekly, Monday at 2:00 AM PDT" - branches: - include: - - release/2.1 - - release/6.0 - - release/7.0 - - main - always: true - parameters: # Choose whether to skip tests when running pipeline manually. - name: skipTests @@ -40,12 +29,6 @@ parameters: type: boolean # Parameters below are ignored in public builds. # -# Choose whether to run the CodeQL3000 tasks. -# Manual builds align w/ official builds unless this parameter is true. -- name: runCodeQL3000 - default: false - displayName: Run CodeQL3000 tasks - type: boolean # Choose whether to enable binlogs when running pipeline manually. # Binary logs are enabled by default in public builds and aren't designed to be disabled there. - name: produceBinlogs @@ -68,10 +51,10 @@ variables: value: true - name: _UseHelixOpenQueues value: ${{ ne(variables['System.TeamProject'], 'internal') }} -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: +- ${{ if and(notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - name: enableSourceIndex value: true -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - name: _BuildArgs value: /p:TeamName=$(_TeamName) /p:OfficialBuildId=$(Build.BuildNumber) @@ -99,12 +82,12 @@ variables: value: -ExcludeCIBinaryLog - name: WindowsArm64InstallersLogArgs value: -ExcludeCIBinaryLog -- ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - name: _BuildArgs value: '/p:SkipTestBuild=true /p:PostBuildSign=$(PostBuildSign)' - name: _PublishArgs value: '' -- ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'), eq(parameters.produceBinlogs, 'true')) }}: +- ${{ if or(in(variables['Build.Reason'], 'PullRequest'), eq(parameters.produceBinlogs, 'true')) }}: # Write binary logs for all main Windows build steps except the x86 one in public and PR builds. - name: WindowsArm64LogArgs value: /bl:artifacts/log/Release/Build.arm64.binlog @@ -118,689 +101,651 @@ variables: value: /bl:artifacts/log/Release/Build.Installers.binlog - name: WindowsArm64InstallersLogArgs value: /bl:artifacts/log/Release/Build.Installers.Arm64.binlog -- ${{ if ne(variables['System.TeamProject'], 'internal') }}: +- group: DotNetBuilds storage account read tokens +- name: _InternalRuntimeDownloadArgs + value: -RuntimeSourceFeed https://dotnetbuilds.blob.core.windows.net/internal + -RuntimeSourceFeedKey $(dotnetbuilds-internal-container-read-token-base64) + /p:DotNetAssetRootAccessTokenSuffix='$(dotnetbuilds-internal-container-read-token-base64)' +# The code signing doesn't use the aspnet build scripts, so the msbuild parameters have to be passed directly. This +# is awkward but necessary because the eng/common/ build scripts don't add the msbuild properties automatically. +- name: _InternalRuntimeDownloadCodeSignArgs + value: $(_InternalRuntimeDownloadArgs) + /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal + /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) +- group: DotNet-HelixApi-Access +- ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - name: _SignType - value: '' - - name: _InternalRuntimeDownloadArgs - value: '' - - name: _InternalRuntimeDownloadCodeSignArgs - value: '' -- ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBuilds storage account read tokens - - name: _InternalRuntimeDownloadArgs - value: -RuntimeSourceFeed https://dotnetbuilds.blob.core.windows.net/internal - -RuntimeSourceFeedKey $(dotnetbuilds-internal-container-read-token-base64) - /p:DotNetAssetRootAccessTokenSuffix='$(dotnetbuilds-internal-container-read-token-base64)' - # The code signing doesn't use the aspnet build scripts, so the msbuild parameters have to be passed directly. This - # is awkward but necessary because the eng/common/ build scripts don't add the msbuild properties automatically. - - name: _InternalRuntimeDownloadCodeSignArgs - value: $(_InternalRuntimeDownloadArgs) - /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal - /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - - group: DotNet-HelixApi-Access - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - - name: _SignType - value: real - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - name: _SignType - value: test -- name: runCodeQL3000 - value: ${{ or(eq(variables['Build.Reason'], 'Schedule'), and(eq(variables['Build.Reason'], 'Manual'), eq(parameters.runCodeQL3000, 'true'))) }} -- template: /eng/common/templates/variables/pool-providers.yml - -stages: -- stage: build - displayName: Build - jobs: - - ${{ if and(ne(variables['System.TeamProject'], 'public'), eq(variables.runCodeQL3000, 'true')) }}: - - template: jobs/default-build.yml - parameters: - jobName: build - jobDisplayName: Build and run CodeQL3000 - agentOs: Windows - codeSign: false - # Component governance and SBOM creation are not needed here. Disable what Arcade would inject. - disableComponentGovernance: true - enableSbom: false - variables: - # Security analysis is included in normal runs. Disable its auto-injection. - - skipNugetSecurityAnalysis: true - # Do not let CodeQL3000 Extension gate scan frequency. - - Codeql.Cadence: 0 - # Enable CodeQL3000 unconditionally so it may be run on any branch. - - Codeql.Enabled: true - # Ignore the small amount of infrastructure Python code in this repo. - - Codeql.Language: cpp,csharp,java,javascript - - Codeql.ExcludePathPatterns: submodules - # Ignore test and infrastructure code. - - Codeql.SourceRoot: src - # CodeQL3000 needs this plumbed along as a variable to enable TSA. - - Codeql.TSAEnabled: ${{ eq(variables['Build.Reason'], 'Schedule') }} - # Default expects tsaoptions.json under SourceRoot. - - Codeql.TSAOptionsPath: '$(Build.SourcesDirectory)/.config/tsaoptions.json' - beforeBuild: - - task: CodeQL3000Init@0 - displayName: CodeQL Initialize - - script: "echo ##vso[build.addbuildtag]CodeQL3000" - displayName: 'Set CI CodeQL3000 tag' - condition: ne(variables.CODEQL_DIST,'') - steps: - - script: ./eng/build.cmd - -ci - -arch x64 - -all - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - /p:UseSharedCompilation=false - displayName: Build x64 - afterBuild: - - task: CodeQL3000Finalize@0 - displayName: CodeQL Finalize - artifacts: - - name: Build_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - - ${{ else }}: # regular build - # Code check - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual')) }}: - - template: jobs/default-build.yml + value: real +- ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - name: _SignType + value: test +- template: /eng/common/templates-official/variables/pool-providers.yml@self + +resources: + repositories: + # Repo: 1ESPipelineTemplates/1ESPipelineTemplates + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + parameters: + sdl: + sourceAnalysisPool: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022-pt + os: windows + containers: + alpine314WithNode: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-WithNode + ubuntu1804CrossArmAlpine: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm-alpine + ubuntu1804CrossArm64Alpine: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-alpine + stages: + - stage: build + displayName: Build + jobs: + # Code check + - ${{ if in(variables['Build.Reason'], 'PullRequest', 'Manual') }}: + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Code_check + jobDisplayName: Code check + agentOs: Windows + steps: + - powershell: ./eng/scripts/CodeCheck.ps1 -ci $(_InternalRuntimeDownloadArgs) + displayName: Run eng/scripts/CodeCheck.ps1 + artifacts: + - name: Code_Check_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + + # Build Windows (x64/x86/arm64) + - template: .azure/pipelines/jobs/default-build.yml@self parameters: - jobName: Code_check - jobDisplayName: Code check + codeSign: true + jobName: Windows_build + jobDisplayName: "Build: Windows x64/x86/arm64" agentOs: Windows steps: - - powershell: ./eng/scripts/CodeCheck.ps1 -ci $(_InternalRuntimeDownloadArgs) - displayName: Run eng/scripts/CodeCheck.ps1 - artifacts: - - name: Code_Check_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true + - ${{ if notIn(variables['Build.Reason'], 'PullRequest') }}: + - script: "echo ##vso[build.addbuildtag]daily-build" + displayName: 'Set CI daily-build tag' + + # !!! NOTE !!! Some of these steps have disabled code signing. + # This is intentional to workaround https://github.com/dotnet/arcade/issues/1957 which always re-submits for code-signing, even + # if they have already been signed. This results in slower builds due to re-submitting the same .nupkg many times for signing. + # The sign settings have been configured to + - script: ./eng/build.cmd + -ci + -arch x64 + -pack + -all + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(Windows64LogArgs) + displayName: Build x64 - # Build Windows (x64/x86/arm64) - - template: jobs/default-build.yml - parameters: - codeSign: true - jobName: Windows_build - jobDisplayName: "Build: Windows x64/x86/arm64" - agentOs: Windows - steps: - - ${{ if notIn(variables['Build.Reason'], 'PullRequest') }}: - - script: "echo ##vso[build.addbuildtag]daily-build" - displayName: 'Set CI daily-build tag' - - # !!! NOTE !!! Some of these steps have disabled code signing. - # This is intentional to workaround https://github.com/dotnet/arcade/issues/1957 which always re-submits for code-signing, even - # if they have already been signed. This results in slower builds due to re-submitting the same .nupkg many times for signing. - # The sign settings have been configured to - - script: ./eng/build.cmd - -ci - -arch x64 - -pack - -all - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - $(Windows64LogArgs) - displayName: Build x64 - - # Build the x86 shared framework - # This is going to actually build x86 native assets. - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -arch x86 - -pack - -all - -noBuildJava - -noBuildNative - /p:OnlyPackPlatformSpecificPackages=true - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - $(Windows86LogArgs) - displayName: Build x86 - - # Build the arm64 shared framework - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -arch arm64 - -sign - -pack - -noBuildJava - -noBuildNative - /p:DotNetSignType=$(_SignType) - /p:OnlyPackPlatformSpecificPackages=true - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - $(WindowsArm64LogArgs) - displayName: Build ARM64 - - # Submit a manual build (in public or internal project) to validate changes to site extensions. - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - script: .\src\SiteExtensions\build.cmd + # Build the x86 shared framework + # This is going to actually build x86 native assets. + - script: ./eng/build.cmd -ci -noBuildRepoTasks + -arch x86 -pack - -noBuildDeps + -all + -noBuildJava -noBuildNative + /p:OnlyPackPlatformSpecificPackages=true $(_BuildArgs) $(_InternalRuntimeDownloadArgs) - displayName: Build SiteExtension - - # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If - # https://github.com/dotnet/arcade/issues/1957 is resolved, consider running code-signing inline with the other - # previous steps. Sign check is disabled because it is run in a separate step below, after installers are built. - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -noBuildNative - -noBuild - -sign - /p:DotNetSignType=$(_SignType) - $(_BuildArgs) - $(WindowsSignLogArgs) - displayName: Code sign packages - - # Windows installers bundle x86/x64/arm64 assets - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -sign - -buildInstallers - -noBuildNative - /p:DotNetSignType=$(_SignType) - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - $(WindowsInstallersLogArgs) - displayName: Build Installers - - # Windows installers bundle and sharedfx msi for arm64 - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -arch arm64 - -sign - -buildInstallers - -noBuildNative - /p:DotNetSignType=$(_SignType) - /p:AssetManifestFileName=aspnetcore-win.xml - $(_BuildArgs) - $(_PublishArgs) - /p:PublishInstallerBaseVersion=true - $(_InternalRuntimeDownloadArgs) - $(WindowsArm64InstallersLogArgs) - displayName: Build ARM64 Installers - - artifacts: - - name: Windows_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Windows_Packages - path: artifacts/packages/ - - name: Windows_HostingBundle - path: artifacts/bin/WindowsHostingBundle - - name: Windows_ANCM_Msi - path: artifacts/bin/ANCMv2 - - name: Windows_ANCMIISExpress_Msi - path: artifacts/bin/AncmIISExpressV2 - - # Build MacOS arm64 - - template: jobs/default-build.yml - parameters: - jobName: MacOs_arm64_build - jobDisplayName: "Build: macOS arm64" - agentOs: macOs - buildArgs: - --arch arm64 - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-MacOS_arm64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false - artifacts: - - name: MacOS_arm64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: MacOS_arm64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: MacOS_arm64 - - # Build MacOS x64 - - template: jobs/default-build.yml - parameters: - jobName: MacOs_x64_build - jobDisplayName: "Build: macOS x64" - agentOs: macOs - buildArgs: - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-MacOS_x64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false - artifacts: - - name: MacOS_x64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: MacOS_x64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml + $(Windows86LogArgs) + displayName: Build x86 + + # Build the arm64 shared framework + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -arch arm64 + -sign + -pack + -noBuildJava + -noBuildNative + /p:DotNetSignType=$(_SignType) + /p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(WindowsArm64LogArgs) + displayName: Build ARM64 + + # Submit a manual build (in public or internal project) to validate changes to site extensions. + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - script: .\src\SiteExtensions\build.cmd + -ci + -noBuildRepoTasks + -pack + -noBuildDeps + -noBuildNative + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Build SiteExtension + + # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If + # https://github.com/dotnet/arcade/issues/1957 is resolved, consider running code-signing inline with the other + # previous steps. Sign check is disabled because it is run in a separate step below, after installers are built. + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -noBuildNative + -noBuild + -sign + /p:DotNetSignType=$(_SignType) + $(_BuildArgs) + $(WindowsSignLogArgs) + displayName: Code sign packages + + # Windows installers bundle x86/x64/arm64 assets + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -sign + -buildInstallers + -noBuildNative + /p:DotNetSignType=$(_SignType) + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(WindowsInstallersLogArgs) + displayName: Build Installers + + # Windows installers bundle and sharedfx msi for arm64 + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -arch arm64 + -sign + -buildInstallers + -noBuildNative + /p:DotNetSignType=$(_SignType) + /p:AssetManifestFileName=aspnetcore-win.xml + $(_BuildArgs) + $(_PublishArgs) + /p:PublishInstallerBaseVersion=true + $(_InternalRuntimeDownloadArgs) + $(WindowsArm64InstallersLogArgs) + displayName: Build ARM64 Installers + + artifacts: + - name: Windows_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Windows_Packages + path: artifacts/packages/ + - name: Windows_HostingBundle + path: artifacts/bin/WindowsHostingBundle + - name: Windows_ANCM_Msi + path: artifacts/bin/ANCMv2 + - name: Windows_ANCMIISExpress_Msi + path: artifacts/bin/AncmIISExpressV2 + + # Build MacOS arm64 + - template: .azure/pipelines/jobs/default-build.yml@self parameters: - inputName: MacOS_x64 - - # Build Linux x64 - - template: jobs/default-build.yml - parameters: - jobName: Linux_x64_build - jobDisplayName: "Build: Linux x64" - agentOs: Linux - useHostedUbuntu: false - steps: - - script: ./eng/build.sh - --ci - --arch x64 - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - displayName: Run build.sh - - script: git clean -xfd src/**/obj/; - ./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs - -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=deb - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - displayName: Build Debian installers - - script: git clean -xfd src/**/obj/; - ./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs - -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm - -p:AssetManifestFileName=aspnetcore-Linux_x64.xml + jobName: MacOs_arm64_build + jobDisplayName: "Build: macOS arm64" + agentOs: macOs + buildArgs: + --arch arm64 + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-MacOS_arm64.xml $(_BuildArgs) $(_PublishArgs) $(_InternalRuntimeDownloadArgs) - displayName: Build RPM installers - installNodeJs: false - artifacts: - - name: Linux_x64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_x64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_x64 - - # Build Linux ARM - - template: jobs/default-build.yml - parameters: - jobName: Linux_arm_build - jobDisplayName: "Build: Linux ARM" - agentOs: Linux - buildArgs: - --arch arm - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-Linux_arm.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false - artifacts: - - name: Linux_arm_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_arm_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml + installNodeJs: false + artifacts: + - name: MacOS_arm64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: MacOS_arm64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: MacOS_arm64 + + # Build MacOS x64 + - template: .azure/pipelines/jobs/default-build.yml@self parameters: - inputName: Linux_arm - - # Build Linux ARM64 - - template: jobs/default-build.yml - parameters: - jobName: Linux_arm64_build - jobDisplayName: "Build: Linux ARM64" - agentOs: Linux - steps: - - script: ./eng/build.sh - --ci - --arch arm64 - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - displayName: Run build.sh - - script: git clean -xfd src/**/obj/; - ./dockerbuild.sh rhel --ci --nobl --arch arm64 --build-installers --no-build-deps --no-build-nodejs - -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm - -p:AssetManifestFileName=aspnetcore-Linux_arm64.xml + jobName: MacOs_x64_build + jobDisplayName: "Build: macOS x64" + agentOs: macOs + buildArgs: + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-MacOS_x64.xml $(_BuildArgs) $(_PublishArgs) $(_InternalRuntimeDownloadArgs) - displayName: Build RPM installers - installNodeJs: false - artifacts: - - name: Linux_arm64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_arm64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_arm64 - - # Build Linux Musl x64 - - template: jobs/default-build.yml - parameters: - jobName: Linux_musl_x64_build - jobDisplayName: "Build: Linux Musl x64" - agentOs: Linux - container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-WithNode - buildArgs: - --arch x64 - --os-name linux-musl - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-Linux_musl_x64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false - disableComponentGovernance: true - artifacts: - - name: Linux_musl_x64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_musl_x64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_musl_x64 - - # Build Linux Musl ARM - - template: jobs/default-build.yml - parameters: - jobName: Linux_musl_arm_build - jobDisplayName: "Build: Linux Musl ARM" - agentOs: Linux - useHostedUbuntu: false - container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm-alpine - buildArgs: - --arch arm - --os-name linux-musl - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-Linux_musl_arm.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false - artifacts: - - name: Linux_musl_arm_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_musl_arm_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_musl_arm - - # Build Linux Musl ARM64 - - template: jobs/default-build.yml - parameters: - jobName: Linux_musl_arm64_build - jobDisplayName: "Build: Linux Musl ARM64" - agentOs: Linux - useHostedUbuntu: false - container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-alpine - buildArgs: - --arch arm64 - --os-name linux-musl - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-Linux_musl_arm64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false - artifacts: - - name: Linux_musl_arm64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_musl_arm64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_musl_arm64 + installNodeJs: false + artifacts: + - name: MacOS_x64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: MacOS_x64_Packages + path: artifacts/packages/ - - ${{ if and(ne(parameters.skipTests, 'true'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual'))) }}: - # Test jobs - - template: jobs/default-build.yml + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: MacOS_x64 + + # Build Linux x64 + - template: .azure/pipelines/jobs/default-build.yml@self parameters: - jobName: Windows_Test - jobDisplayName: "Test: Windows Server x64" - agentOs: Windows - isAzDOTestingJob: true - # Just uploading artifacts/logs/ files can take 15 minutes. Doubling the cancel timeout for this job. - cancelTimeoutInMinutes: 30 - buildArgs: -all -pack -test -binaryLog /p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true - /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false /p:RunBlazorPlaywrightTemplateTests=true - $(_InternalRuntimeDownloadArgs) - beforeBuild: - - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1" - displayName: Setup IISExpress test certificates and schema + jobName: Linux_x64_build + jobDisplayName: "Build: Linux x64" + agentOs: Linux + useHostedUbuntu: false + steps: + - script: ./eng/build.sh + --ci + --arch x64 + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Run build.sh + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=deb + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Build Debian installers + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm + -p:AssetManifestFileName=aspnetcore-Linux_x64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Build RPM installers + installNodeJs: false artifacts: - - name: Windows_Test_Logs + - name: Linux_x64_Logs path: artifacts/log/ publishOnError: true includeForks: true - - name: Windows_Test_Results - path: artifacts/TestResults/ + - name: Linux_x64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: Linux_x64 + + # Build Linux ARM + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Linux_arm_build + jobDisplayName: "Build: Linux ARM" + agentOs: Linux + buildArgs: + --arch arm + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_arm.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false + artifacts: + - name: Linux_arm_Logs + path: artifacts/log/ publishOnError: true includeForks: true + - name: Linux_arm_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: Linux_arm - - template: jobs/default-build.yml + # Build Linux ARM64 + - template: .azure/pipelines/jobs/default-build.yml@self parameters: - jobName: MacOS_Test - jobDisplayName: "Test: macOS" - agentOs: macOS - timeoutInMinutes: 240 - isAzDOTestingJob: true - buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) - beforeBuild: - - bash: "./eng/scripts/install-nginx-mac.sh" - displayName: Installing Nginx + jobName: Linux_arm64_build + jobDisplayName: "Build: Linux ARM64" + agentOs: Linux + steps: + - script: ./eng/build.sh + --ci + --arch arm64 + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Run build.sh + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh rhel --ci --nobl --arch arm64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm + -p:AssetManifestFileName=aspnetcore-Linux_arm64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Build RPM installers + installNodeJs: false artifacts: - - name: MacOS_Test_Logs + - name: Linux_arm64_Logs path: artifacts/log/ publishOnError: true includeForks: true - - name: MacOS_Test_Results - path: artifacts/TestResults/ + - name: Linux_arm64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: Linux_arm64 + + # Build Linux Musl x64 + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Linux_musl_x64_build + jobDisplayName: "Build: Linux Musl x64" + agentOs: Linux + container: alpine314WithNode + buildArgs: + --arch x64 + --os-name linux-musl + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_musl_x64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false + disableComponentGovernance: true + artifacts: + - name: Linux_musl_x64_Logs + path: artifacts/log/ publishOnError: true includeForks: true + - name: Linux_musl_x64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: Linux_musl_x64 - - template: jobs/default-build.yml + # Build Linux Musl ARM + - template: .azure/pipelines/jobs/default-build.yml@self parameters: - jobName: Linux_Test - jobDisplayName: "Test: Ubuntu x64" + jobName: Linux_musl_arm_build + jobDisplayName: "Build: Linux Musl ARM" agentOs: Linux - isAzDOTestingJob: true useHostedUbuntu: false - buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) - beforeBuild: - - bash: "./eng/scripts/install-nginx-linux.sh" - displayName: Installing Nginx - - bash: "echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p" - displayName: Increase inotify limit + container: ubuntu1804CrossArmAlpine + buildArgs: + --arch arm + --os-name linux-musl + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_musl_arm.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false artifacts: - - name: Linux_Test_Logs + - name: Linux_musl_arm_Logs path: artifacts/log/ publishOnError: true includeForks: true - - name: Linux_Test_Results - path: artifacts/TestResults/ - publishOnError: true - includeForks: true + - name: Linux_musl_arm_Packages + path: artifacts/packages/ - # Helix x64 - - template: jobs/default-build.yml - parameters: - jobName: Helix_x64 - jobDisplayName: 'Tests: Helix x64' - agentOs: Windows - timeoutInMinutes: 240 - steps: - # Build the shared framework - - script: ./eng/build.cmd -ci -nobl -all -pack -arch x64 - /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) - displayName: Build shared fx - # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. - - script: ./eng/build.cmd -ci -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test - -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true - /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) - displayName: Run build.cmd helix target - env: - HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues - SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: Linux_musl_arm + # Build Linux Musl ARM64 + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Linux_musl_arm64_build + jobDisplayName: "Build: Linux Musl ARM64" + agentOs: Linux + useHostedUbuntu: false + container: ubuntu1804CrossArm64Alpine + buildArgs: + --arch arm64 + --os-name linux-musl + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_musl_arm64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false artifacts: - - name: Helix_logs + - name: Linux_musl_arm64_Logs path: artifacts/log/ publishOnError: true includeForks: true - - # Source build - - template: /eng/common/templates/job/source-build.yml - parameters: - platform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' - buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)' - skipPublishValidation: true - jobProperties: - timeoutInMinutes: 120 - variables: - # Log environment variables in binary logs to ease debugging - MSBUILDLOGALLENVIRONMENTVARIABLES: true - - - ${{ if eq(variables.enableSourceIndex, 'true') }}: - - template: /eng/common/templates/job/source-index-stage1.yml + - name: Linux_musl_arm64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: Linux_musl_arm64 + + - ${{ if and(ne(parameters.skipTests, 'true'), in(variables['Build.Reason'], 'PullRequest', 'Manual')) }}: + # Test jobs + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Windows_Test + jobDisplayName: "Test: Windows Server x64" + agentOs: Windows + isAzDOTestingJob: true + # Just uploading artifacts/logs/ files can take 15 minutes. Doubling the cancel timeout for this job. + cancelTimeoutInMinutes: 30 + buildArgs: -all -pack -test -binaryLog /p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true + /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false /p:RunBlazorPlaywrightTemplateTests=true + $(_InternalRuntimeDownloadArgs) + beforeBuild: + - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1" + displayName: Setup IISExpress test certificates and schema + artifacts: + - name: Windows_Test_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Windows_Test_Results + path: artifacts/TestResults/ + publishOnError: true + includeForks: true + + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: MacOS_Test + jobDisplayName: "Test: macOS" + agentOs: macOS + timeoutInMinutes: 240 + isAzDOTestingJob: true + buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) + beforeBuild: + - bash: "./eng/scripts/install-nginx-mac.sh" + displayName: Installing Nginx + artifacts: + - name: MacOS_Test_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: MacOS_Test_Results + path: artifacts/TestResults/ + publishOnError: true + includeForks: true + + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Linux_Test + jobDisplayName: "Test: Ubuntu x64" + agentOs: Linux + isAzDOTestingJob: true + useHostedUbuntu: false + buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) + beforeBuild: + - bash: "./eng/scripts/install-nginx-linux.sh" + displayName: Installing Nginx + - bash: "echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p" + displayName: Increase inotify limit + artifacts: + - name: Linux_Test_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_Test_Results + path: artifacts/TestResults/ + publishOnError: true + includeForks: true + + # Helix x64 + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Helix_x64 + jobDisplayName: 'Tests: Helix x64' + agentOs: Windows + timeoutInMinutes: 240 + steps: + # Build the shared framework + - script: ./eng/build.cmd -ci -nobl -all -pack -arch x64 + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + displayName: Build shared fx + # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. + - script: ./eng/build.cmd -ci -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test + -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + displayName: Run build.cmd helix target + env: + HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues + SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops + + artifacts: + - name: Helix_logs + path: artifacts/log/ + publishOnError: true + includeForks: true + + # Source build + - template: /eng/common/templates-official/job/source-build.yml@self parameters: - sourceIndexBuildCommand: ./eng/build.cmd -Configuration Release -ci -noBuildJava -binaryLog /p:OnlyPackPlatformSpecificPackages=true - binlogPath: artifacts/log/Release/Build.binlog - presteps: - - task: NodeTool@0 - displayName: Install Node 18.x - inputs: - versionSpec: 18.x - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2022 - - # Publish to the BAR and perform source indexing. Wait until everything else is done. - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/job/publish-build-assets.yml + platform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)' + skipPublishValidation: true + jobProperties: + timeoutInMinutes: 120 + variables: + # Log environment variables in binary logs to ease debugging + MSBUILDLOGALLENVIRONMENTVARIABLES: true + + - ${{ if eq(variables.enableSourceIndex, 'true') }}: + - template: /eng/common/templates-official/job/source-index-stage1.yml@self + parameters: + sourceIndexBuildCommand: ./eng/build.cmd -Configuration Release -ci -noBuildJava -binaryLog /p:OnlyPackPlatformSpecificPackages=true + binlogPath: artifacts/log/Release/Build.binlog + presteps: + - task: NodeTool@0 + displayName: Install Node 18.x + inputs: + versionSpec: 18.x + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-windows-2022 + + # Publish to the BAR and perform source indexing. Wait until everything else is done. + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/templates-official/job/publish-build-assets.yml@self + parameters: + dependsOn: + - Windows_build + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - CodeSign_Xplat_MacOS_arm64 + - CodeSign_Xplat_MacOS_x64 + - CodeSign_Xplat_Linux_x64 + - CodeSign_Xplat_Linux_arm + - CodeSign_Xplat_Linux_arm64 + - CodeSign_Xplat_Linux_musl_x64 + - CodeSign_Xplat_Linux_musl_arm + - CodeSign_Xplat_Linux_musl_arm64 + - ${{ if eq(variables.PostBuildSign, 'true') }}: + - MacOs_arm64_build + - MacOs_x64_build + - Linux_x64_build + - Linux_arm_build + - Linux_arm64_build + - Linux_musl_x64_build + - Linux_musl_arm_build + - Linux_musl_arm64_build + # In addition to the dependencies above that provide assets, ensure the build was successful overall. + - ${{ if in(variables['Build.Reason'], 'Manual') }}: + - Code_check + - ${{ if ne(parameters.skipTests, 'true') }}: + - Windows_Test + - MacOS_Test + - Linux_Test + - Helix_x64 + - ${{ if eq(variables.enableSourceIndex, 'true') }}: + - SourceIndexStage1 + - Source_Build_Managed + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-windows-2019 + publishUsingPipelines: ${{ variables._PublishUsingPipelines }} + enablePublishBuildArtifacts: true # publish artifacts/log files + publishAssetsImmediately: true # Don't use a separate stage for darc publishing. + + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/templates-official/post-build/post-build.yml@self parameters: - dependsOn: - - Windows_build - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - CodeSign_Xplat_MacOS_arm64 - - CodeSign_Xplat_MacOS_x64 - - CodeSign_Xplat_Linux_x64 - - CodeSign_Xplat_Linux_arm - - CodeSign_Xplat_Linux_arm64 - - CodeSign_Xplat_Linux_musl_x64 - - CodeSign_Xplat_Linux_musl_arm - - CodeSign_Xplat_Linux_musl_arm64 - - ${{ if eq(variables.PostBuildSign, 'true') }}: - - MacOs_arm64_build - - MacOs_x64_build - - Linux_x64_build - - Linux_arm_build - - Linux_arm64_build - - Linux_musl_x64_build - - Linux_musl_arm_build - - Linux_musl_arm64_build - # In addition to the dependencies above that provide assets, ensure the build was successful overall. - - ${{ if in(variables['Build.Reason'], 'Manual') }}: - - Code_check - - ${{ if ne(parameters.skipTests, 'true') }}: - - Windows_Test - - MacOS_Test - - Linux_Test - - Helix_x64 - - ${{ if eq(variables.enableSourceIndex, 'true') }}: - - SourceIndexStage1 - - Source_Build_Managed - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2019 - publishUsingPipelines: ${{ variables._PublishUsingPipelines }} - enablePublishBuildArtifacts: true # publish artifacts/log files - publishAssetsImmediately: true # Don't use a separate stage for darc publishing. - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.runCodeQL3000, 'true')) }}: - - template: /eng/common/templates/post-build/post-build.yml - parameters: - publishingInfraVersion: 3 - enableSymbolValidation: false - enableSigningValidation: false - enableNugetValidation: false - publishInstallersAndChecksums: true - publishAssetsImmediately: true + publishingInfraVersion: 3 + enableSymbolValidation: false + enableSigningValidation: false + enableNugetValidation: false + publishInstallersAndChecksums: true + publishAssetsImmediately: true diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index 7a7324997c26..8a8755e69335 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -329,7 +329,7 @@ jobs: pool: ${{ if eq(parameters.agentOs, 'macOS') }}: name: Azure Pipelines - image: macOS-12 + image: macOS-11 os: macOS ${{ if eq(parameters.agentOs, 'Linux') }}: name: $(DncEngInternalBuildPool) @@ -338,7 +338,7 @@ jobs: ${{ if eq(parameters.agentOs, 'Windows') }}: name: $(DncEngInternalBuildPool) # Visual Studio Enterprise - contains some stuff, like SQL Server and IIS Express, that we use for testing - image: windows.vs2022preview.amd64 + image: 1es-windows-2022 os: windows ${{ if ne(parameters.container, '') }}: container: ${{ parameters.container }} From f91bd437c1a9d9b7afb7706df862f27918c6c703 Mon Sep 17 00:00:00 2001 From: William Godbe Date: Mon, 25 Mar 2024 13:49:29 -0700 Subject: [PATCH 14/16] Update ci-public.yml --- .azure/pipelines/ci-public.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml index 6d6692f75fd8..5db588cb7d74 100644 --- a/.azure/pipelines/ci-public.yml +++ b/.azure/pipelines/ci-public.yml @@ -50,11 +50,10 @@ variables: value: true - name: _UseHelixOpenQueues value: 'true' -- ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - name: _BuildArgs - value: '/p:SkipTestBuild=true /p:PostBuildSign=$(PostBuildSign)' - - name: _PublishArgs - value: '' +- name: _BuildArgs + value: '/p:SkipTestBuild=true /p:PostBuildSign=$(PostBuildSign)' +- name: _PublishArgs + value: '' # Write binary logs for all main Windows build steps except the x86 one in public and PR builds. - name: WindowsArm64LogArgs value: /bl:artifacts/log/Release/Build.arm64.binlog @@ -615,4 +614,4 @@ stages: timeoutInMinutes: 120 variables: # Log environment variables in binary logs to ease debugging - MSBUILDLOGALLENVIRONMENTVARIABLES: true \ No newline at end of file + MSBUILDLOGALLENVIRONMENTVARIABLES: true From 5cd059e2ce451b7462cc6067126a494bf9f118c6 Mon Sep 17 00:00:00 2001 From: Rich Lander Date: Tue, 26 Mar 2024 19:56:48 -0700 Subject: [PATCH 15/16] Update EOL versions (#54791) --- .azure/pipelines/ci-public.yml | 6 +++--- .azure/pipelines/ci.yml | 20 ++++++++++---------- eng/targets/Helix.Common.props | 16 ++++++++-------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml index 5db588cb7d74..e9e48ff35a22 100644 --- a/.azure/pipelines/ci-public.yml +++ b/.azure/pipelines/ci-public.yml @@ -444,7 +444,7 @@ stages: jobDisplayName: "Build: Linux Musl ARM" agentOs: Linux useHostedUbuntu: false - container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + container: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine buildArgs: --arch arm --os-name linux-musl @@ -478,7 +478,7 @@ stages: jobDisplayName: "Build: Linux Musl ARM64" agentOs: Linux useHostedUbuntu: false - container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm64-alpine + container: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine buildArgs: --arch arm64 --os-name linux-musl @@ -607,7 +607,7 @@ stages: parameters: platform: name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)' skipPublishValidation: true jobProperties: diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index 197e1a4984f2..daba25973421 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -138,12 +138,12 @@ extends: image: 1es-windows-2022-pt os: windows containers: - alpine314WithNode: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-WithNode - ubuntu1804CrossArmAlpine: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm-alpine - ubuntu1804CrossArm64Alpine: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-alpine + alpine319WithNode: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + mariner20CrossArmAlpine: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine + mariner20CrossArm64Alpine: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine stages: - stage: build displayName: Build @@ -477,7 +477,7 @@ extends: jobName: Linux_musl_x64_build jobDisplayName: "Build: Linux Musl x64" agentOs: Linux - container: alpine314WithNode + container: alpine319WithNode buildArgs: --arch x64 --os-name linux-musl @@ -512,7 +512,7 @@ extends: jobDisplayName: "Build: Linux Musl ARM" agentOs: Linux useHostedUbuntu: false - container: ubuntu1804CrossArmAlpine + container: mariner20CrossArmAlpine buildArgs: --arch arm --os-name linux-musl @@ -546,7 +546,7 @@ extends: jobDisplayName: "Build: Linux Musl ARM64" agentOs: Linux useHostedUbuntu: false - container: ubuntu1804CrossArm64Alpine + container: mariner20CrossArm64Alpine buildArgs: --arch arm64 --os-name linux-musl @@ -675,7 +675,7 @@ extends: parameters: platform: name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)' skipPublishValidation: true jobProperties: diff --git a/eng/targets/Helix.Common.props b/eng/targets/Helix.Common.props index e586fa5efc16..426386e98ebf 100644 --- a/eng/targets/Helix.Common.props +++ b/eng/targets/Helix.Common.props @@ -1,12 +1,12 @@ - (AlmaLinux.8.Amd64.Open)Ubuntu.2004.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-helix-amd64 - (Alpine.316.Amd64.Open)Ubuntu.2004.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.16-helix-amd64 - (Debian.11.Amd64.Open)Ubuntu.2004.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 - (Fedora.34.Amd64.Open)Ubuntu.2004.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix - (Mariner)Ubuntu.2004.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-1.0-helix - (Debian.12.Arm64.Open)ubuntu.2004.armarch.open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-helix-arm64v8 + (AlmaLinux.8.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-helix-amd64 + (Alpine.317.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-amd64 + (Debian.11.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 + (Fedora.38.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-38-helix + (Mariner)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-1.0-helix + (Debian.12.Arm64.Open)ubuntu.2204.armarch.open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-helix-arm64v8 false @@ -42,9 +42,9 @@ - + - + From 24771f048aed6d2042486982c12fde881dc82dda Mon Sep 17 00:00:00 2001 From: William Godbe Date: Mon, 1 Apr 2024 12:52:02 -0700 Subject: [PATCH 16/16] Update ci-public.yml --- .azure/pipelines/ci-public.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml index e9e48ff35a22..eb9ffeaa3cd5 100644 --- a/.azure/pipelines/ci-public.yml +++ b/.azure/pipelines/ci-public.yml @@ -73,6 +73,10 @@ variables: value: '' - name: _InternalRuntimeDownloadCodeSignArgs value: '' +- name: Codeql.Enabled + value: false +- name: Codeql.SkipTaskAutoInjection + value: true - template: /eng/common/templates/variables/pool-providers.yml stages: