Merge branch 'main' into announce

Signed-off-by: Alyssa Wilk <alyssar@chromium.org>

.azure-pipelines/env.yml

@@ -174,32 +174,18 @@ jobs:
 
       PUBLISH_GITHUB_RELEASE=$(run.packaging)
       PUBLISH_DOCKERHUB=false
-      PUBLISH_DOCS=false
-      PUBLISH_DOCS_LATEST=false
-      PUBLISH_DOCS_RELEASE=false
 
       if [[ "$ISSTABLEBRANCH" == True && -n "$POSTSUBMIT" && "$NOSYNC" != true ]]; then
-          # Build docs for publishing either latest or a release build
-          PUBLISH_DOCS=true
           # main
           if [[ "$ISMAIN" == True ]]; then
               # Update the Dockerhub README
               PUBLISH_DOCKERHUB=true
-              if [[ "$(state.isDev)" == true ]]; then
-                  # Postsubmit on `main` trigger rebuild of latest docs
-                  PUBLISH_DOCS_LATEST=true
-              fi
           # Not main, and not -dev
           elif [[ "$(state.isDev)" == false ]]; then
               if [[ "$(state.versionPatch)" -eq 0 ]]; then
                   # A just-forked branch
                   PUBLISH_GITHUB_RELEASE=false
               fi
-              # A stable release, publish docs to the release
-              PUBLISH_DOCS_RELEASE=true
-          else
-              # Postsubmit for non-main/release, skip publishing docs in this case
-              PUBLISH_DOCS=false
           fi
       fi
 
@@ -210,9 +196,6 @@ jobs:
 
       echo "##vso[task.setvariable variable=githubRelease;isoutput=true]${PUBLISH_GITHUB_RELEASE}"
       echo "##vso[task.setvariable variable=dockerhub;isoutput=true]${PUBLISH_DOCKERHUB}"
-      echo "##vso[task.setvariable variable=docs;isoutput=true]${PUBLISH_DOCS}"
-      echo "##vso[task.setvariable variable=docsLatest;isoutput=true]${PUBLISH_DOCS_LATEST}"
-      echo "##vso[task.setvariable variable=docsRelease;isoutput=true]${PUBLISH_DOCS_RELEASE}"
 
     displayName: "Decide what to publish"
     workingDirectory: $(Build.SourcesDirectory)
@@ -234,9 +217,6 @@ jobs:
       echo
       echo "env.outputs['publish.githubRelease']: $(publish.githubRelease)"
       echo "env.outputs['publish.dockerhub]: $(publish.dockerhub)"
-      echo "env.outputs['publish.docs]: $(publish.docs)"
-      echo "env.outputs['publish.docsLatest]: $(publish.docsLatest)"
-      echo "env.outputs['publish.docsRelease]: $(publish.docsRelease)"
 
     displayName: "Print build environment"
 

.azure-pipelines/stage/checks.yml

@@ -77,7 +77,7 @@ jobs:
   timeoutInMinutes: 180
   pool: envoy-x64-small
   steps:
-  - template: ../bazel.yml
+  - template: ../ci.yml
     parameters:
       ciTarget: $(CI_TARGET)
       cacheName: $(CI_TARGET)

.azure-pipelines/stage/linux.yml

@@ -45,7 +45,7 @@ jobs:
   timeoutInMinutes: ${{ parameters.timeoutBuild }}
   pool: ${{ parameters.pool }}
   steps:
-  - template: ../bazel.yml
+  - template: ../ci.yml
     parameters:
       managedAgent: ${{ parameters.managedAgent }}
       ciTarget: release

.azure-pipelines/stage/prechecks.yml

@@ -48,11 +48,13 @@ jobs:
         CI_TARGET: "format"
       protobuf:
         CI_TARGET: "check_and_fix_proto_format"
-      publishing:
-        CI_TARGET: docs
+      ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+        publishing:
+          CI_TARGET: docs
   steps:
-  - template: ../bazel.yml
+  - template: ../ci.yml
     parameters:
+      bazelBuildExtraOptions: --config=docs-ci
       ciTarget: $(CI_TARGET)
       cacheName: $(CI_TARGET)
       cacheTestResults: ${{ parameters.cacheTestResults }}

.azure-pipelines/stage/publish.yml

@@ -39,24 +39,12 @@ parameters:
 - name: authGPGKey
   type: string
   default: ""
-- name: authNetlifyURL
-  type: string
-  default: ""
 - name: authDockerUser
   type: string
   default: ""
 - name: authDockerPassword
   type: string
   default: ""
-- name: authSSHDocsKey
-  type: string
-  default: ""
-- name: authSSHDocsKeyPublic
-  type: string
-  default: ""
-- name: authSSHKeyPassphrase
-  type: string
-  default: ""
 
 - name: runDocker
   displayName: "Run Docker"
@@ -71,18 +59,6 @@ parameters:
   displayName: "Publish Dockerhub"
   type: string
   default: false
-- name: publishDocs
-  displayName: "Publish Docs"
-  type: string
-  default: false
-- name: publishDocsLatest
-  displayName: "Publish latest docs"
-  type: string
-  default: false
-- name: publishDocsRelease
-  displayName: "Publish release docs"
-  type: string
-  default: false
 - name: publishGithubRelease
   displayName: "Publish Github release"
   type: string
@@ -104,7 +80,7 @@ jobs:
       artifactName: "release"
       itemPattern: "release/**/bin/*"
       targetPath: $(Build.StagingDirectory)
-  - template: ../bazel.yml
+  - template: ../ci.yml
     parameters:
       ciTarget: docker-upload
       cacheName: docker-upload
@@ -120,32 +96,40 @@ jobs:
           echo "disk space at beginning of Docker build:"
           df -h
         displayName: "Check disk space before Docker build"
+      # TODO(phlax): switch docker <> docker-upload as main task
       - bash: |
           set -e
-
-          mkdir -p linux/amd64 linux/arm64
-
-          # x64
-          cp -a $(Build.StagingDirectory)/release/x64/bin/release.tar.zst linux/amd64/release.tar.zst
-          cp -a $(Build.StagingDirectory)/release/x64/bin/schema_validator_tool linux/amd64/schema_validator_tool
-
-          # arm64
-          cp -a $(Build.StagingDirectory)/release/arm64/bin/release.tar.zst linux/arm64/release.tar.zst
-          cp -a $(Build.StagingDirectory)/release/arm64/bin/schema_validator_tool linux/arm64/schema_validator_tool
-
-          # Debug what files appear to have been downloaded
-          find linux -type f -name "*" | xargs ls -l
-
-          ci/docker_ci.sh
+          mkdir -p $(Build.StagingDirectory)/envoy
+          rm -rf $(Build.StagingDirectory)/envoy/*
+          mv $(Build.StagingDirectory)/release/* $(Build.StagingDirectory)/envoy
+          ./ci/run_envoy_docker.sh 'ci/do_ci.sh docker'
         displayName: Build Docker images
         timeoutInMinutes: ${{ parameters.timeoutDockerPublish }}
         workingDirectory: $(Build.SourcesDirectory)
         env:
-          AZP_BRANCH: $(Build.SourceBranch)
-          AZP_SHA1: $(Build.SourceVersion)
+          CI_BRANCH: $(Build.SourceBranch)
+          CI_SHA1: $(Build.SourceVersion)
           DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }}
           DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }}
           DOCKER_BUILD_TIMEOUT: ${{ parameters.timeoutDockerBuild }}
+          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+          ENVOY_DOCKER_IN_DOCKER: 1
+
+      stepsPost:
+      - script: |
+          ci/run_envoy_docker.sh 'ci/do_ci.sh dockerhub-publish'
+        condition: |
+          and(not(canceled()), succeeded(),
+              eq(${{ parameters.publishDockerhub }}, 'true'))
+        displayName: "Publish Dockerhub description and README"
+        env:
+          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+          ENVOY_RBE: "1"
+          BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --jobs=$(RbeJobs)"
+          GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
+          GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }}
+          DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }}
+          DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }}
 
 - job: package_x64
   displayName: Linux debs (x64)
@@ -163,7 +147,7 @@ jobs:
       artifactName: "release"
       itemPattern: "release/x64/bin/*"
       targetPath: $(Build.StagingDirectory)
-  - template: ../bazel.yml
+  - template: ../ci.yml
     parameters:
       ciTarget: distribution
       cacheName: distribution
@@ -197,7 +181,7 @@ jobs:
       itemPattern: "release/arm64/bin/*"
       targetPath: $(Build.StagingDirectory)
 
-  - template: ../bazel.yml
+  - template: ../ci.yml
     parameters:
       managedAgent: false
       ciTarget: distribution
@@ -220,80 +204,6 @@ jobs:
           set -e
           rm -rf $(Build.StagingDirectory)/.gnupg
 
-- job: docs
-  displayName: Publish docs
-  dependsOn: []
-  condition: |
-    and(not(canceled()),
-        eq(${{ parameters.publishDocs }}, 'true'))
-  pool:
-    vmImage: $(agentUbuntu)
-  steps:
-  - template: ../bazel.yml
-    parameters:
-      ciTarget: docs
-      cacheName: docs
-      cacheVersion: $(cacheKeyBazel)
-      publishEnvoy: false
-      publishTestResults: false
-      env:
-        AZP_BRANCH: $(Build.SourceBranch)
-      stepsPost:
-
-      - script: |
-          ci/run_envoy_docker.sh 'ci/do_ci.sh dockerhub-publish'
-        condition: |
-          and(not(canceled()),
-              eq(${{ parameters.publishDockerhub }}, 'true'))
-        displayName: "Publish Dockerhub description and README"
-        env:
-          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-          ENVOY_RBE: "1"
-          BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --jobs=$(RbeJobs)"
-          GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
-          GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }}
-          DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }}
-          DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }}
-
-      # Trigger Netlify rebuild of latest docs
-      - script: |
-          ci/run_envoy_docker.sh 'ci/do_ci.sh docs-upload'
-        displayName: "Upload Docs to GCS"
-        condition: |
-          and(not(canceled()),
-              eq(${{ parameters.publishDocsLatest }}, 'true'))
-        env:
-          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-          ENVOY_RBE: "1"
-          BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --jobs=$(RbeJobs)"
-          GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
-          GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }}
-      - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs-publish-latest'
-        condition: |
-          and(not(canceled()),
-              eq(${{ parameters.publishDocsLatest }}, 'true'))
-        displayName: "Publish latest docs"
-        workingDirectory: $(Build.SourcesDirectory)
-        env:
-          NETLIFY_TRIGGER_URL: ${{ parameters.authNetlifyURL }}
-
-      # Publish docs to the website
-      - task: InstallSSHKey@0
-        condition: |
-          and(not(canceled()),
-              eq(${{ parameters.publishDocsRelease }}, 'true'))
-        inputs:
-          hostName: $(authGithubSSHKeyPublic)
-          sshPublicKey: "${{ parameters.authSSHDocsKeyPublic }}"
-          sshPassphrase: "${{ parameters.authSSHKeyPassphrase }}"
-          sshKeySecureFile: "${{ parameters.authSSHDocsKey }}"
-      - script: docs/publish.sh
-        condition: |
-          and(not(canceled()),
-              eq(${{ parameters.publishDocsRelease }}, 'true'))
-        displayName: "Publish release docs"
-        workingDirectory: $(Build.SourcesDirectory)
-
 - job: signed_release
   displayName: Signed binaries
   dependsOn:
@@ -318,7 +228,7 @@ jobs:
       artifactName: "distribution"
       itemPattern: "distribution/**/packages.*.tar.gz"
       targetPath: $(Build.StagingDirectory)
-  - template: ../bazel.yml
+  - template: ../ci.yml
     parameters:
       ciTarget: release.signed
       cacheName: release-signed
@@ -334,7 +244,7 @@ jobs:
           pathGPGConfiguredHome: /build/.gnupg
           pathGPGHome: $(Build.StagingDirectory)/.gnupg
 - job: success
-  dependsOn: ["docker", "docs", "signed_release"]
+  dependsOn: ["docker", "signed_release"]
   displayName: Success (linux artefacts)
   pool:
     vmImage: $(agentUbuntu)
@@ -345,7 +255,6 @@ jobs:
   condition: |
     and(
       in(dependencies.docker.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
-      in(dependencies.docs.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
       in(dependencies.signed_release.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
   steps:
   - checkout: none
@@ -362,7 +271,7 @@ jobs:
   pool:
     vmImage: $(agentUbuntu)
   steps:
-  - template: ../bazel.yml
+  - template: ../ci.yml
     parameters:
       ciTarget: verify.trigger
       cacheName: verify-trigger

.azure-pipelines/stage/verify.yml

@@ -22,7 +22,7 @@ jobs:
       itemPattern: "distribution/x64/packages.x64.tar.gz"
       downloadType: single
       targetPath: $(Build.StagingDirectory)
-  - template: ../bazel.yml
+  - template: ../ci.yml
     parameters:
       ciTarget: verify_distro
       cacheName: verify_distro
@@ -43,7 +43,7 @@ jobs:
       itemPattern: "distribution/arm64/packages.arm64.tar.gz"
       downloadType: single
       targetPath: $(Build.StagingDirectory)
-  - template: ../bazel.yml
+  - template: ../ci.yml
     parameters:
       managedAgent: false
       ciTarget: verify_distro

.azure-pipelines/stage/windows.yml

@@ -84,8 +84,8 @@ jobs:
       ci/docker_ci.sh
     workingDirectory: $(Build.SourcesDirectory)
     env:
-      AZP_BRANCH: $(Build.SourceBranch)
-      AZP_SHA1: $(Build.SourceVersion)
+      CI_BRANCH: $(Build.SourceBranch)
+      CI_SHA1: $(Build.SourceVersion)
       DOCKERHUB_USERNAME: $(DockerUsername)
       DOCKERHUB_PASSWORD: $(DockerPassword)
       WINDOWS_BUILD_TYPE: $(windowsBuildType)

.azure-pipelines/stages.yml

@@ -57,7 +57,6 @@ stages:
   jobs:
   - template: env.yml
 
-
 - stage: prechecks
   displayName: Prechecks
   dependsOn: ["env"]
@@ -123,9 +122,6 @@ stages:
     RUN_PACKAGING: $[stageDependencies.env.repo.outputs['run.packaging']]
     PUBLISH_GITHUB_RELEASE: $[stageDependencies.env.repo.outputs['publish.githubRelease']]
     PUBLISH_DOCKERHUB: $[stageDependencies.env.repo.outputs['publish.dockerhub']]
-    PUBLISH_DOCS: $[stageDependencies.env.repo.outputs['publish.docs']]
-    PUBLISH_DOCS_LATEST: $[stageDependencies.env.repo.outputs['publish.docsLatest']]
-    PUBLISH_DOCS_RELEASE: $[stageDependencies.env.repo.outputs['publish.docsRelease']]
   jobs:
   - template: stage/publish.yml
     parameters:
@@ -138,18 +134,11 @@ stages:
       authGPGPassphrase: $(MaintainerGPGKeyPassphrase)
       authGPGKey: $(MaintainerGPGKeySecureFileDownloadPath)
       authGPGPath: $(MaintainerGPGKey.secureFilePath)
-      authNetlifyURL: $(NetlifyTriggerURL)
-      authSSHDocsKeyPublic: $(DocsPublicKey)
-      authSSHDocsKey: $(DocsPrivateKey)
-      authSSHKeyPassphrase: $(SshDeployKeyPassphrase)
       bucketGCP: $(GcsArtifactBucket)
       timeoutDockerBuild: ${{ parameters.timeoutDockerBuild }}
       timeoutDockerPublish: ${{ parameters.timeoutDockerPublish }}
       runDocker: variables['RUN_DOCKER']
       runPackaging: variables['RUN_PACKAGING']
-      publishDocs: variables['PUBLISH_DOCS']
-      publishDocsLatest: variables['PUBLISH_DOCS_LATEST']
-      publishDocsRelease: variables['PUBLISH_DOCS_RELEASE']
       publishDockerhub: variables['PUBLISH_DOCKERHUB']
       publishGithubRelease: variables['PUBLISH_GITHUB_RELEASE']