From 719c1ce5f0310139edf302454e4febaf1a406a7d Mon Sep 17 00:00:00 2001 From: Coen van den Munckhof Date: Mon, 2 Sep 2024 21:05:36 +0200 Subject: [PATCH] Update azure devops pipeline (#140) * Update pipeline * Rename * Fix pipeline: * Use yaml instead of yml * Fix indent in yaml file * cleanup and improve yaml * cleanup * fix release dependencies * Added icons| * Fix * minor update| * update icons --- .../Pipelines/Templates/create-installer.yaml | 34 +++++ .../Templates/dotnet-sdk-install.yaml | 16 +++ .../Templates/prepare-code-analysis.yaml | 12 ++ .../Templates/publish-code-analysis.yaml | 19 +++ .../publish-release-dependencies.yaml | 13 ++ .azuredevops/Pipelines/build.yaml | 87 ++---------- .azuredevops/Pipelines/publish.yaml | 131 ++++++++++-------- .azuredevops/Pipelines/pull-request.yaml | 42 ++---- .azuredevops/Variables/DotNetSdkVersion.ps1 | 2 + .azuredevops/Variables/_all.ps1 | 20 +++ .azuredevops/Variables/_pipelines.ps1 | 31 +++++ dotnet-releaser.toml | 5 + mdsnippets.json | 2 +- 13 files changed, 254 insertions(+), 160 deletions(-) create mode 100644 .azuredevops/Pipelines/Templates/create-installer.yaml create mode 100644 .azuredevops/Pipelines/Templates/dotnet-sdk-install.yaml create mode 100644 .azuredevops/Pipelines/Templates/prepare-code-analysis.yaml create mode 100644 .azuredevops/Pipelines/Templates/publish-code-analysis.yaml create mode 100644 .azuredevops/Pipelines/Templates/publish-release-dependencies.yaml create mode 100644 .azuredevops/Variables/DotNetSdkVersion.ps1 create mode 100644 .azuredevops/Variables/_all.ps1 create mode 100644 .azuredevops/Variables/_pipelines.ps1 create mode 100644 dotnet-releaser.toml diff --git a/.azuredevops/Pipelines/Templates/create-installer.yaml b/.azuredevops/Pipelines/Templates/create-installer.yaml new file mode 100644 index 00000000..05b67428 --- /dev/null +++ b/.azuredevops/Pipelines/Templates/create-installer.yaml @@ -0,0 +1,34 @@ +steps: + +- task: DeleteFiles@1 + displayName: Remove unneeded files + inputs: + contents: | + _output/**/*.xlm + _output/**/*.config + _output/**/*.pdb + _output/Win/Assemblies/runtimes/!(win*)/**/*.* + _output/Win/Assemblies/!(runtimes)/*.resources.dll + +- pwsh: | + Get-ChildItem -Path "_output\Win\Assemblies" -Recurse | Where-Object { $_.PSIsContainer -and @(Get-ChildItem -LiteralPath $_.FullName -Recurse -Force).Count -eq 0 } | Remove-Item -Recurse + Get-ChildItem -Path "_output\Win\Assemblies\runtimes" -Recurse | Where-Object { $_.PSIsContainer -and @(Get-ChildItem -LiteralPath $_.FullName -Recurse -Force).Count -eq 0 } | Remove-Item -Recurse + displayName: Remove empty directories + +- pwsh: | + nuget install NSIS -OutputDirectory ./packages + makensis.exe /DPRODUCT_VERSION=$Env:NBGV_SEMVER2 _setup/RepoM.nsi + displayName: โš™๏ธ Create installer + +- task: DeleteFiles@1 + displayName: Remove binaries + inputs: + contents: | + _output/win + +- task: PublishBuildArtifacts@1 + displayName: ๐Ÿ“ข Publish artifact + inputs: + # PathtoPublish: "$(Build.ArtifactStagingDirectory)" + PathtoPublish: "_output" + ArtifactName: installer \ No newline at end of file diff --git a/.azuredevops/Pipelines/Templates/dotnet-sdk-install.yaml b/.azuredevops/Pipelines/Templates/dotnet-sdk-install.yaml new file mode 100644 index 00000000..94ff579c --- /dev/null +++ b/.azuredevops/Pipelines/Templates/dotnet-sdk-install.yaml @@ -0,0 +1,16 @@ +steps: + + - powershell: .azuredevops/Variables/_pipelines.ps1 + failOnStderr: true + displayName: โš™๏ธ Set pipeline variables based on source + name: SetPipelineVariables + + - task: UseDotNet@2 + displayName: โš™๏ธ Use specified dotnet sdk version + inputs: + version: $(DOTNETSDKVERSION) + includePreviewVersions: false + condition: and(succeeded(), ne(variables['DOTNETSDKVERSION'], '')) + + - script: dotnet --info + displayName: Show dotnet SDK info \ No newline at end of file diff --git a/.azuredevops/Pipelines/Templates/prepare-code-analysis.yaml b/.azuredevops/Pipelines/Templates/prepare-code-analysis.yaml new file mode 100644 index 00000000..d2428587 --- /dev/null +++ b/.azuredevops/Pipelines/Templates/prepare-code-analysis.yaml @@ -0,0 +1,12 @@ +steps: + +- task: SonarCloudPrepare@2 + displayName: Prepare analysis on SonarCloud + inputs: + SonarCloud: SonarCloudGithub + organization: coenm-github + projectKey: RepoM + projectName: RepoM + extraProperties: | + sonar.cs.vstest.reportsPaths=$(Agent.TempDirectory)/**/*.trx + sonar.cs.opencover.reportsPaths=$(Agent.TempDirectory)/**/*.opencover.xml \ No newline at end of file diff --git a/.azuredevops/Pipelines/Templates/publish-code-analysis.yaml b/.azuredevops/Pipelines/Templates/publish-code-analysis.yaml new file mode 100644 index 00000000..9e5a9e65 --- /dev/null +++ b/.azuredevops/Pipelines/Templates/publish-code-analysis.yaml @@ -0,0 +1,19 @@ +steps: + +- task: SonarCloudAnalyze@2 + displayName: โš™๏ธ Run Code Analysis + +- task: SonarCloudPublish@2 + displayName: ๐Ÿ“ข Publish Quality Gate Result + +- pwsh: | + dotnet tool install --tool-path . dotnet-reportgenerator-globaltool + ./reportgenerator "-reports:$(Agent.TempDirectory)/**/coverage.cobertura.xml" "-targetdir:$(Agent.TempDirectory)" "-reporttypes:Cobertura" + displayName: Merge coverage reports + name: MergeCoverageReports + +- task: PublishCodeCoverageResults@2 + displayName: ๐Ÿ“ข Publish code coverage to devops + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(Agent.TempDirectory)/Cobertura.xml' \ No newline at end of file diff --git a/.azuredevops/Pipelines/Templates/publish-release-dependencies.yaml b/.azuredevops/Pipelines/Templates/publish-release-dependencies.yaml new file mode 100644 index 00000000..5d61a883 --- /dev/null +++ b/.azuredevops/Pipelines/Templates/publish-release-dependencies.yaml @@ -0,0 +1,13 @@ +steps: + +- task: CopyFiles@2 + displayName: ๐Ÿ—ƒ๏ธ Copy release dependencies + inputs: + contents: 'dotnet-releaser.toml' + targetFolder: _output_release_dependencies + +- task: PublishBuildArtifacts@1 + displayName: ๐Ÿ“ข Publish release dependencies + inputs: + PathtoPublish: "_output_release_dependencies" + ArtifactName: release-dependencies \ No newline at end of file diff --git a/.azuredevops/Pipelines/build.yaml b/.azuredevops/Pipelines/build.yaml index 5a1463ae..82632b95 100644 --- a/.azuredevops/Pipelines/build.yaml +++ b/.azuredevops/Pipelines/build.yaml @@ -42,20 +42,17 @@ stages: steps: - - script: git submodule update --init --recursive - displayName: Update git submodules + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true - - task: UseDotNet@2 - displayName: "Use dotnet sdk 8.0.401" - inputs: - version: 8.0.401 - includePreviewVersions: false + - script: git submodule update --init --recursive + displayName: ๐Ÿ”„ Update git submodules - - script: dotnet --info - displayName: Show dotnet SDK info + - template: Templates/dotnet-sdk-install.yaml - task: NuGetToolInstaller@1 - displayName: Install Nuget 5x + displayName: โฌ‡๏ธ Install Nuget 5x inputs: versionSpec: '5.*' checkLatest: true @@ -63,7 +60,7 @@ stages: - pwsh: | dotnet tool install --tool-path . nbgv ./nbgv cloud -a - displayName: Set build number + displayName: 4๏ธโƒฃ2๏ธโƒฃ Set build number name: Nbgv - task: PowerShell@2 @@ -102,24 +99,15 @@ stages: # name: mdsnippets - task: DotNetCoreCLI@2 - displayName: DotNet Restore + displayName: ๐Ÿ› ๏ธ DotNet Restore inputs: command: restore projects: '**/*.csproj' - - task: SonarCloudPrepare@2 - displayName: Prepare analysis on SonarCloud - inputs: - SonarCloud: SonarCloudGithub - organization: coenm-github - projectKey: RepoM - projectName: RepoM - extraProperties: | - sonar.cs.vstest.reportsPaths=$(Agent.TempDirectory)/**/*.trx - sonar.cs.opencover.reportsPaths=$(Agent.TempDirectory)/**/*.opencover.xml + - template: Templates/prepare-code-analysis.yaml - task: DotNetCoreCLI@2 - displayName: DotNet Build + displayName: ๐Ÿ› ๏ธ DotNet Build inputs: command: build projects: '**/*.sln' @@ -127,7 +115,7 @@ stages: # https://docs.microsoft.com/en-us/azure/devops/pipelines/ecosystems/dotnet-core?view=azure-devops&tabs=dotnetfive - task: DotNetCoreCLI@2 - displayName: Dotnet Test + displayName: ๐Ÿงช Dotnet Test inputs: command: test projects: 'tests/**/*.csproj' @@ -138,23 +126,7 @@ stages: /p:ExcludeByAttribute=CompilerGenerated publishTestResults: true - - task: SonarCloudAnalyze@2 - displayName: Run Code Analysis - - - task: SonarCloudPublish@2 - displayName: Publish Quality Gate Result - - - pwsh: | - dotnet tool install --tool-path . dotnet-reportgenerator-globaltool - ./reportgenerator "-reports:$(Agent.TempDirectory)/**/coverage.cobertura.xml" "-targetdir:$(Agent.TempDirectory)" "-reporttypes:Cobertura" - displayName: Merge coverage reports - name: MergeCoverageReports - - - task: PublishCodeCoverageResults@2 - displayName: Publish code coverage to devops - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(Agent.TempDirectory)/Cobertura.xml' + - template: Templates/publish-code-analysis.yaml - task: DotNetCoreCLI@2 displayName: Dotnet Publish RepoM @@ -166,36 +138,7 @@ stages: modifyOutputPath: false zipAfterPublish: false - - task: DeleteFiles@1 - displayName: Remove unneeded files - inputs: - contents: | - _output/**/*.xlm - _output/**/*.config - _output/**/*.pdb - _output/Win/Assemblies/runtimes/!(win*)/**/*.* - _output/Win/Assemblies/!(runtimes)/*.resources.dll - - - pwsh: | - Get-ChildItem -Path "_output\Win\Assemblies" -Recurse | Where-Object { $_.PSIsContainer -and @(Get-ChildItem -LiteralPath $_.FullName -Recurse -Force).Count -eq 0 } | Remove-Item -Recurse - Get-ChildItem -Path "_output\Win\Assemblies\runtimes" -Recurse | Where-Object { $_.PSIsContainer -and @(Get-ChildItem -LiteralPath $_.FullName -Recurse -Force).Count -eq 0 } | Remove-Item -Recurse - displayName: Remove empty directories - - - pwsh: | - nuget install NSIS -OutputDirectory ./packages - makensis.exe /DPRODUCT_VERSION=$Env:NBGV_SEMVER2 _setup/RepoM.nsi - displayName: Create installer - - - task: DeleteFiles@1 - displayName: Remove binaries - inputs: - contents: | - _output/win + - template: Templates/create-installer.yaml - - task: PublishBuildArtifacts@1 - displayName: Publish artifact - inputs: - # PathtoPublish: "$(Build.ArtifactStagingDirectory)" - PathtoPublish: "_output" - ArtifactName: installer + - template: Templates/publish-release-dependencies.yaml \ No newline at end of file diff --git a/.azuredevops/Pipelines/publish.yaml b/.azuredevops/Pipelines/publish.yaml index 199ea855..a2222561 100644 --- a/.azuredevops/Pipelines/publish.yaml +++ b/.azuredevops/Pipelines/publish.yaml @@ -5,68 +5,87 @@ resources: pipelines: - pipeline: CI source: RepoM.Build - trigger: - tags: - - auto-release + trigger: true + +variables: +- group: Publishing secrets stages: -- stage: GitHubRelease - displayName: GitHub Release +- stage: UpdateBuildNumber + displayName: Set build number jobs: - - deployment: create + - job: SetBuildNumber + displayName: Set build number pool: vmImage: ubuntu-latest - environment: No-Approval - strategy: - runOnce: - deploy: - steps: - - download: none - - powershell: | - Write-Host "##vso[build.updatebuildnumber]$(resources.pipeline.CI.runName)" - displayName: Set pipeline name - - task: GitHubRelease@1 - displayName: GitHub release (create) - inputs: - gitHubConnection: github_coenm_pan - repositoryName: $(Build.Repository.Name) - target: $(resources.pipeline.CI.sourceCommit) - tagSource: userSpecifiedTag - tag: v$(resources.pipeline.CI.runName) - title: v$(resources.pipeline.CI.runName) - isDraft: true - changeLogCompareToRelease: lastNonDraftRelease - changeLogType: issueBased - changeLogLabels: | - [ - { "label" : "bug", "displayName" : "Fixes", "state" : "closed" }, - { "label" : "enhancement", "displayName": "Enhancements", "state" : "closed" } - ] + steps: + - checkout: none + - powershell: | + Write-Host "##vso[build.updatebuildnumber]$(resources.pipeline.CI.runName)" + displayName: โš™๏ธ Set up pipeline + +- stage: ManualValidation + displayName: "Manual Validation" + jobs: + - job: WaitForValidation + pool: server + timeoutInMinutes: 1441 + displayName: Wait for validation + steps: + - task: ManualValidation@0 + displayName: ๐Ÿง Manual validation + timeoutInMinutes: 1440 + inputs: + notifyUsers: + instructions: 'Please approve or reject the deployment.' + onTimeout: 'reject' -- stage: nuget_org - displayName: nuget.org - dependsOn: GitHubRelease +- stage: GithubRelease + displayName: Github Release jobs: - - deployment: push + - job: release + displayName: Github Release pool: vmImage: ubuntu-latest - environment: No-Approval - strategy: - runOnce: - deploy: - steps: - - download: CI - artifact: packages - displayName: Download nuget packages artifact - patterns: 'packages/*' - - task: NuGetToolInstaller@1 - displayName: Use NuGet 5.x - inputs: - versionSpec: 5.x - - task: NuGetCommand@2 - displayName: NuGet push nupkg - inputs: - command: push - packagesToPush: '$(Pipeline.Workspace)/CI/packages/*.nupkg;!$(Pipeline.Workspace)/**/*.symbols.nupkg' - nuGetFeedType: external - publishFeedCredentials: nuget.org \ No newline at end of file + steps: + - checkout: none + - powershell: | + if ('$(resources.pipeline.CI.runName)'.Contains('-')) { + Write-Host "##vso[task.setvariable variable=IsPrerelease]true" + } else { + Write-Host "##vso[task.setvariable variable=IsPrerelease]false" + } + displayName: โš™๏ธ Set up pipeline + - download: CI + artifact: installer + displayName: ๐Ÿ”ป Download installer artifact + patterns: 'installer/*' + - download: CI + artifact: release-dependencies + displayName: ๐Ÿ”ป Download publisher artifact + patterns: 'release-dependencies/*' + - task: GitHubRelease@1 + displayName: ๐Ÿ“ข Create GitHub release + inputs: + gitHubConnection: github_coenm_pan + repositoryName: $(Build.Repository.Name) + target: $(resources.pipeline.CI.sourceCommit) + tagSource: userSpecifiedTag + tag: v$(resources.pipeline.CI.runName) + title: v$(resources.pipeline.CI.runName) + isDraft: false + isPreRelease: $(IsPrerelease) + assets: $(Pipeline.Workspace)/CI/installer/*Setup.exe + changeLogCompareToRelease: lastNonDraftRelease + changeLogType: issueBased + changeLogLabels: | + [ + { "label" : "breaking change", "displayName" : "Breaking changes", "state" : "closed" }, + { "label" : "bug", "displayName" : "Fixes", "state" : "closed" }, + { "label" : "enhancement", "displayName": "Enhancements", "state" : "closed" } + ] + - powershell: | + dotnet tool install --global dotnet-releaser + dotnet-releaser changelog --update --github-token $(GITHUB_TOKEN) $(Pipeline.Workspace)/CI/release-dependencies/dotnet-releaser.toml $(resources.pipeline.CI.runName) + displayName: ๐Ÿ“ข Update GitHub release using Releaser + diff --git a/.azuredevops/Pipelines/pull-request.yaml b/.azuredevops/Pipelines/pull-request.yaml index 6ecec068..b9cec0ed 100644 --- a/.azuredevops/Pipelines/pull-request.yaml +++ b/.azuredevops/Pipelines/pull-request.yaml @@ -33,19 +33,12 @@ stages: clean: true - script: git submodule update --init --recursive - displayName: Update git submodules + displayName: ๐Ÿ”„ Update git submodules - - task: UseDotNet@2 - displayName: "Use dotnet sdk 8.0.401" - inputs: - version: 8.0.401 - includePreviewVersions: false - - - script: dotnet --info - displayName: Show dotnet SDK info + - template: Templates/dotnet-sdk-install.yaml - task: PowerShell@2 - displayName: Export variables + displayName: ๐Ÿค– Export variables inputs: targetType: 'inline' script: | @@ -60,7 +53,7 @@ stages: ignoreLASTEXITCODE: true - task: NuGetToolInstaller@1 - displayName: Install Nuget 5x + displayName: โฌ‡๏ธ Install Nuget 5x inputs: versionSpec: '5.*' checkLatest: true @@ -82,42 +75,33 @@ stages: Write-Error "There are local changes in the git repository meaning not all markdown snippets are up to date." exit 1 } - displayName: Verify markdown snippets are up to date + displayName: ๐Ÿ“„ Verify markdown snippets name: mdsnippets failOnStderr: true - pwsh: | dotnet tool install --tool-path . nbgv ./nbgv cloud -a - displayName: Set build number + displayName: 4๏ธโƒฃ2๏ธโƒฃ Set build number name: Nbgv - task: DotNetCoreCLI@2 - displayName: DotNet Restore + displayName: ๐Ÿ› ๏ธ DotNet Restore inputs: command: restore projects: '**/*.csproj' - - task: SonarCloudPrepare@2 - displayName: Prepare analysis on SonarCloud - inputs: - SonarCloud: SonarCloudGithub - organization: coenm-github - projectKey: RepoM - projectName: RepoM - extraProperties: | - sonar.cs.vstest.reportsPaths=$(Agent.TempDirectory)/**/*.trx - sonar.cs.opencover.reportsPaths=$(Agent.TempDirectory)/**/*.opencover.xml + - template: Templates/prepare-code-analysis.yaml - task: DotNetCoreCLI@2 - displayName: DotNet Build + displayName: ๐Ÿ› ๏ธ DotNet Build inputs: command: build projects: '**/*.sln' arguments: '--configuration $(BuildConfiguration)' - task: DotNetCoreCLI@2 - displayName: Dotnet Test + displayName: ๐Ÿงช Dotnet Test inputs: command: test projects: 'tests/**/*.csproj' @@ -128,8 +112,4 @@ stages: /p:ExcludeByAttribute=CompilerGenerated publishTestResults: true - - task: SonarCloudAnalyze@2 - displayName: Run Code Analysis - - - task: SonarCloudPublish@2 - displayName: Publish Quality Gate Result \ No newline at end of file + - template: Templates/publish-code-analysis.yaml \ No newline at end of file diff --git a/.azuredevops/Variables/DotNetSdkVersion.ps1 b/.azuredevops/Variables/DotNetSdkVersion.ps1 new file mode 100644 index 00000000..b213fbc2 --- /dev/null +++ b/.azuredevops/Variables/DotNetSdkVersion.ps1 @@ -0,0 +1,2 @@ +$globalJson = Get-Content -Path "$PSScriptRoot\..\..\global.json" | ConvertFrom-Json +$globalJson.sdk.version diff --git a/.azuredevops/Variables/_all.ps1 b/.azuredevops/Variables/_all.ps1 new file mode 100644 index 00000000..afecd705 --- /dev/null +++ b/.azuredevops/Variables/_all.ps1 @@ -0,0 +1,20 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS + This script returns a hashtable of build variables that should be set + at the start of a build or release definition's execution. +#> + +[CmdletBinding(SupportsShouldProcess = $true)] +param ( +) + +$vars = @{} + +Get-ChildItem "$PSScriptRoot\*.ps1" -Exclude "_*" |ForEach-Object { + Write-Host "Computing $($_.BaseName) variable" + $vars[$_.BaseName] = & $_ +} + +$vars diff --git a/.azuredevops/Variables/_pipelines.ps1 b/.azuredevops/Variables/_pipelines.ps1 new file mode 100644 index 00000000..d575019e --- /dev/null +++ b/.azuredevops/Variables/_pipelines.ps1 @@ -0,0 +1,31 @@ +<# +.SYNOPSIS + This script translates the variables returned by the _all.ps1 script + into commands that instruct Azure Pipelines to actually set those variables for other pipeline tasks to consume. + + The build or release definition may have set these variables to override + what the build would do. So only set them if they have not already been set. +#> + +[CmdletBinding()] +param ( +) + +(& "$PSScriptRoot\_all.ps1").GetEnumerator() |ForEach-Object { + # Always use ALL CAPS for env var names since Azure Pipelines converts variable names to all caps and on non-Windows OS, env vars are case sensitive. + $keyCaps = $_.Key.ToUpper() + if ((Test-Path "env:$keyCaps") -and (Get-Content "env:$keyCaps")) { + Write-Host "Skipping setting $keyCaps because variable is already set to '$(Get-Content env:$keyCaps)'." -ForegroundColor Cyan + } else { + Write-Host "$keyCaps=$($_.Value)" -ForegroundColor Yellow + if ($env:TF_BUILD) { + # Create two variables: the first that can be used by its simple name and accessible only within this job. + Write-Host "##vso[task.setvariable variable=$keyCaps]$($_.Value)" + # and the second that works across jobs and stages but must be fully qualified when referenced. + Write-Host "##vso[task.setvariable variable=$keyCaps;isOutput=true]$($_.Value)" + } elseif ($env:GITHUB_ACTIONS) { + Add-Content -Path $env:GITHUB_ENV -Value "$keyCaps=$($_.Value)" + } + Set-Item -Path "env:$keyCaps" -Value $_.Value + } +} diff --git a/dotnet-releaser.toml b/dotnet-releaser.toml new file mode 100644 index 00000000..39e8f8f1 --- /dev/null +++ b/dotnet-releaser.toml @@ -0,0 +1,5 @@ +# configuration file for dotnet-releaser +[github] +user = "coenm" +repo = "RepoM" +version_prefix = "v" \ No newline at end of file diff --git a/mdsnippets.json b/mdsnippets.json index fb0ced03..3cf614a3 100644 --- a/mdsnippets.json +++ b/mdsnippets.json @@ -2,7 +2,7 @@ "$schema": "https://raw.githubusercontent.com/SimonCropp/MarkdownSnippets/master/schema.json", "LinkFormat": "GitHub", "TocLevel": 3, - "ExcludeDirectories": [ "_ReSharper.Caches", "packages", "build"], + "ExcludeDirectories": [ "_ReSharper.Caches", "packages", "build", ".azuredevops" ], "MaxWidth": 160, "TreatMissingAsWarning": true, "WriteHeader": false,