Merge branch 'main' into allow-upstream-half-close

.azure-pipelines/ci.yml

@@ -328,7 +328,6 @@ steps:
     else
         sudo chown -R azure-pipelines:azure-pipelines ${{ parameters.pathCacheTemp }}/all
     fi
-
   displayName: "Cache/save (${{ parameters.cacheName}})"
   condition: and(succeeded(), ne('${{ parameters.cacheName }}', ''), ne(variables.CACHE_RESTORED, 'true'))
 

.azure-pipelines/stage/checks.yml

@@ -83,6 +83,7 @@ jobs:
       cacheName: $(CI_TARGET)
       envoyBuildFilterExample: $(ENVOY_FILTER_EXAMPLE)
       cacheTestResults: ${{ parameters.cacheTestResults }}
+      diskspaceHack: true
       managedAgent: false
       repoFetchDepth: $(REPO_FETCH_DEPTH)
       repoFetchTags: $(REPO_FETCH_TAGS)

.azure-pipelines/stage/prechecks.yml

@@ -25,17 +25,26 @@ parameters:
   type: string
   default: ""
 
+- name: runBuild
+  displayName: "Run build"
+  type: string
+  default: true
+- name: runPrechecks
+  displayName: "Run prechecks"
+  type: string
+  default: true
+
 # Timeout/s
 - name: timeoutPrechecks
   type: number
   # Building the rst from protos can take a while even with RBE if there is
   # a lot of change - eg protobuf changed, or a primitve proto changed.
   default: 40
 
-- name: runPrechecks
-  displayName: "Run prechecks"
+- name: bazelConfigRBE
   type: string
-  default: true
+  default: --config=remote-ci --config=rbe-google --jobs=$(RbeJobs)
+
 
 jobs:
 - job: prechecks
@@ -53,17 +62,21 @@ jobs:
     matrix:
       format:
         CI_TARGET: "format"
+        CI_CACHE: format
       protobuf:
         CI_TARGET: "check_and_fix_proto_format"
+        CI_CACHE: check_and_fix_proto_format
       ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
         publishing:
           CI_TARGET: docs
+          CI_CACHE: docs
+
   steps:
   - template: ../ci.yml
     parameters:
       bazelBuildExtraOptions: --config=docs-ci
       ciTarget: $(CI_TARGET)
-      cacheName: $(CI_TARGET)
+      cacheName: $(CI_CACHE)
       cacheTestResults: ${{ parameters.cacheTestResults }}
       cacheVersion: $(cacheKeyBazel)
       publishEnvoy: false
@@ -157,9 +170,47 @@ jobs:
           GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }}
         condition: eq(variables['CI_TARGET'], 'docs')
 
+- job: precheck_release_x64
+  displayName: Precheck release (x64)
+  condition: |
+    and(not(canceled()),
+        eq(${{ parameters.runBuild }}, 'true'))
+  timeoutInMinutes: 180
+  pool: envoy-x64-large
+  steps:
+  - template: ../ci.yml
+    parameters:
+      artifactName: release
+      ciTarget: release.test_only
+      cacheName: release-test-only
+      bazelConfigRBE: ${{ parameters.bazelConfigRBE }}
+      cacheTestResults: ${{ parameters.cacheTestResults }}
+      cacheVersion: $(cacheKeyBazel)
+      rbe: true
+
+- job: precheck_release_arm64
+  displayName: Precheck release (arm64)
+  condition: |
+    and(not(canceled()),
+        eq(${{ parameters.runBuild }}, 'true'))
+  timeoutInMinutes: 180
+  pool: envoy-arm-large
+  steps:
+  - template: ../ci.yml
+    parameters:
+      artifactName: release
+      ciTarget: release.test_only
+      cacheName: release-test-only
+      bazelConfigRBE: ${{ parameters.bazelConfigRBE }}
+      bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base"
+      cacheTestResults: ${{ parameters.cacheTestResults }}
+      cacheVersion: $(cacheKeyBazel)
+      artifactSuffix: .arm64
+      rbe: false
+
 - job: prechecked
   displayName: Prechecked
-  dependsOn: ["prechecks"]
+  dependsOn: ["prechecks", "precheck_release_arm64", "precheck_release_x64"]
   pool:
     vmImage: $(agentUbuntu)
   # This condition ensures that this (required) job passes if all of
@@ -169,7 +220,8 @@ jobs:
   condition: |
     and(
       eq(variables['Build.Reason'], 'PullRequest'),
-      in(dependencies.prechecks.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
+      in(dependencies.prechecks.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
+      in(dependencies.precheck_release_arm64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
   steps:
   - checkout: none
   - bash: |

.azure-pipelines/stage/publish.yml

@@ -45,6 +45,14 @@ parameters:
   type: string
   default: ""
 
+- name: bazelConfigRBE
+  type: string
+  default: --config=remote-ci --config=rbe-google --jobs=$(RbeJobs)
+
+- name: runBuild
+  displayName: "Run Build"
+  type: string
+  default: true
 - name: runDocker
   displayName: "Run Docker"
   type: string
@@ -64,8 +72,58 @@ parameters:
   default: false
 
 jobs:
+
+- job: release_x64
+  displayName: Binary release
+  condition: |
+    and(not(canceled()),
+        eq(${{ parameters.runBuild }}, 'true'))
+  timeoutInMinutes: 120
+  pool:
+    vmImage: $(agentUbuntu)
+  steps:
+  - template: ../ci.yml
+    parameters:
+      artifactName: release
+      ciTarget: release.server_only
+      cacheName: release-server-only
+      bazelConfigRBE: ${{ parameters.bazelConfigRBE }}
+      cacheVersion: $(cacheKeyBazel)
+      cacheTestResults: false
+      publishTestResults: false
+      rbe: true
+
+- job: release_arm64
+  displayName: Binary release (arm64)
+  condition: |
+    and(not(canceled()),
+        eq(${{ parameters.runBuild }}, 'true'))
+  timeoutInMinutes: 180
+  pool: envoy-arm-large
+  steps:
+  - bash: |
+      CI_TARGET="release.server_only"
+      echo "${CI_TARGET}"
+      echo "##vso[task.setvariable variable=value;isoutput=true]${CI_TARGET}"
+    name: target
+  - template: ../ci.yml
+    parameters:
+      artifactName: release
+      ciTarget: release.server_only
+      cacheName: release-server-only
+      bazelConfigRBE: ${{ parameters.bazelConfigRBE }}
+      bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base"
+      cacheVersion: $(cacheKeyBazel)
+      cacheTestResults: false
+      artifactSuffix: .arm64
+      publishTestResults: false
+      rbe: false
+
 - job: docker
   displayName: "Docker (Linux multi arch)"
+  dependsOn:
+  - release_x64
+  - release_arm64
   condition: |
     and(not(canceled()),
         eq(${{ parameters.runDocker }}, 'true'))
@@ -128,7 +186,7 @@ jobs:
 
 - job: package_x64
   displayName: Linux debs (x64)
-  dependsOn: []
+  dependsOn: ["release_x64"]
   condition: |
     and(not(canceled()),
         eq(${{ parameters.runPackaging }}, 'true'))
@@ -162,7 +220,7 @@ jobs:
 
 - job: package_arm64
   displayName: Linux debs (arm64)
-  dependsOn: []
+  dependsOn: ["release_arm64"]
   condition: |
     and(not(canceled()),
         eq(${{ parameters.runPackaging }}, 'true'))

.azure-pipelines/stages.yml

@@ -12,7 +12,7 @@ parameters:
   type: object
   default:
   - env
-  - linux_x64
+  - prechecks
 - name: concurrencyChecks
   displayName: "Check concurrency"
   type: number
@@ -59,40 +59,6 @@ stages:
       bucketGCP: $(GcsArtifactBucket)
       runPrechecks: stageDependencies.env.repo.outputs['run.releaseTests']
 
-- stage: linux_x64
-  displayName: Linux x64
-  dependsOn: ${{ parameters.buildStageDeps }}
-  variables:
-    RUN_BUILD: $[stageDependencies.env.repo.outputs['run.build']]
-    RUN_TESTS: $[stageDependencies.env.repo.outputs['run.releaseTests']]
-  jobs:
-  - template: stage/linux.yml
-    parameters:
-      cacheTestResults: ${{ parameters.cacheTestResults }}
-      bazelConfigRBE: --config=remote-ci --config=rbe-google --jobs=200
-      pool: envoy-x64-large
-      # these are parsed differently and _must_ be expressed in this way
-      runBuild: variables['RUN_BUILD']
-      runTests: $(RUN_TESTS)
-
-- stage: linux_arm64
-  displayName: Linux arm64
-  dependsOn: ${{ parameters.buildStageDeps }}
-  variables:
-    RUN_BUILD: $[stageDependencies.env.repo.outputs['run.build']]
-    RUN_TESTS: $[stageDependencies.env.repo.outputs['run.releaseTests']]
-  jobs:
-  - template: stage/linux.yml
-    parameters:
-      cacheTestResults: ${{ parameters.cacheTestResults }}
-      rbe: false
-      artifactSuffix: .arm64
-      timeoutBuild: 180
-      pool: envoy-arm-large
-      runBuild: variables['RUN_BUILD']
-      runTests: $(RUN_TESTS)
-      bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base"
-
 - stage: check
   displayName: Checks (Linux x64)
   dependsOn: ${{ parameters.checkStageDeps }}
@@ -110,8 +76,9 @@ stages:
 
 - stage: publish
   displayName: Publish
-  dependsOn: ["env", "linux_x64", "linux_arm64"]
+  dependsOn: ["env", "prechecks"]
   variables:
+    RUN_BUILD: $[stageDependencies.env.repo.outputs['run.build']]
     RUN_DOCKER: $[stageDependencies.env.repo.outputs['run.docker']]
     RUN_PACKAGING: $[stageDependencies.env.repo.outputs['run.packaging']]
     PUBLISH_GITHUB_RELEASE: $[stageDependencies.env.repo.outputs['publish.githubRelease']]
@@ -131,6 +98,7 @@ stages:
       bucketGCP: $(GcsArtifactBucket)
       timeoutDockerBuild: ${{ parameters.timeoutDockerBuild }}
       timeoutDockerPublish: ${{ parameters.timeoutDockerPublish }}
+      runBuild: variables['RUN_BUILD']
       runDocker: variables['RUN_DOCKER']
       runPackaging: variables['RUN_PACKAGING']
       publishDockerhub: variables['PUBLISH_DOCKERHUB']