Release 3.6.0 (2024-05-01)

### Features * [Debug Reporting] Enable debug reporting by default * [DebugReporting] Add a log to get the libcurl queue time * add ability to tag AMIs * Add markdown-link-check to pre-commit * add support for AWS CodeBuild * add support for GCP Cloud Build * API Updates for Bid Currency Support for Top-Level Seller in Server-Orchestrated Multi-Seller Auctions * Enable TCMalloc for all B&A servers * support sending Nitro Enclave logs to AWS CloudWatch for debugging ### Bug Fixes * [Temp] Adds buyer bid to top level auctions for Chrome < v125 * Adjust ps_vlog pre-commit hook to avoid recursing symlinks * clean up log verbosity 2 * **deps:** Upgrade clang-format to v18 * **deps:** Upgrade pre-commit hooks * Ensure that the debug url length is capped Bug: 338216564 GitOrigin-RevId: 80d679ed731179f7041dd54ab336c2ec119923e4 Change-Id: I778be44f30e098d8ddc75399b0da797e5933f508
privacysandbox · May 2, 2024 · 2903295 · 2903295
1 parent 872865a
commit 2903295
Show file tree

Hide file tree

Showing 119 changed files with 2,961 additions and 836 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -27,6 +27,11 @@ build:clang --client_env=BAZEL_CXXOPTS=-std=c++17
 build:clang --copt=-Werror=thread-safety
 #disable the "int conversion" warning for external/nitrokmscli_aws_c_http/source/websocket_decoder.c
 build:clang --per_file_copt=external/nitrokmscli_.*\.c@-Wno-int-conversion
+# https://github.com/google/tcmalloc/blob/master/docs/tuning.md#build-time-optimizations
+build:clang --cxxopt=-fsized-deallocation
+build:clang --host_cxxopt=-fsized-deallocation
+build:clang --cxxopt=-fnew-alignment=8
+build:clang --host_cxxopt=-fnew-alignment=8
 
 # Flag compiler warnings as errors.
 build:cpp_no_warn --copt=-Werror
@@ -101,6 +106,7 @@ build:asan --copt=-fno-omit-frame-pointer
 build:asan --linkopt=-fsanitize=address
 build:asan --linkopt=-fuse-ld=lld
 build:asan --action_env=ASAN_OPTIONS=detect_leaks=1:color=always
+build:asan --custom_malloc=@bazel_tools//tools/cpp:malloc
 
 # Memory sanitizer, set action_env to segregate cache entries
 build:msan --action_env=PRIVACY_SANDBOX_SERVERS_MSAN=1

diff --git a/.markdown-link-check.json b/.markdown-link-check.json
@@ -0,0 +1,21 @@
+{
+  "ignorePatterns": [
+    {
+      "pattern": "^tg/"
+    },
+    {
+      "pattern": "^http://go/"
+    }
+  ],
+  "replacementPatterns": [
+    {
+      "pattern": "^/",
+      "replacement": "{{BASEURL}}/"
+    }
+  ],
+  "timeout": "20s",
+  "retryOn429": true,
+  "retryCount": 5,
+  "fallbackRetryDelay": "30s",
+  "aliveStatusCodes": [200, 206]
+}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -29,7 +29,7 @@ exclude: (?x)^(
 fail_fast: false
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.6.0
   hooks:
   - id: end-of-file-fixer
   - id: fix-byte-order-marker
@@ -63,7 +63,7 @@ repos:
     files: ^$
 
 - repo: https://github.com/bufbuild/buf
-  rev: v1.26.1
+  rev: v1.31.0
   hooks:
   - id: buf-format
     args:
@@ -72,7 +72,7 @@ repos:
     - api/
 
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: v16.0.6
+  rev: v18.1.4
   hooks:
   - id: clang-format
     types_or:
@@ -121,18 +121,17 @@ repos:
 
   - id: ps_vlog_123
     name: Don't use PS_VLOG(0/1/2/3)
-    entry: sh -c '! grep -RE "PS_VLOG[(][01][,)]" services/'
+    entry: sh -c '! grep --files-with-matches --recursive -E "PS_VLOG[(][012][,)]" services/'
     language: system
     types_or:
     - c++
     - c
     pass_filenames: false
     always_run: false
     fail_fast: false
-    verbose: true
 
 - repo: https://github.com/pre-commit/mirrors-prettier
-  rev: v3.0.3
+  rev: v3.1.0
   hooks:
   - id: prettier
     types_or:
@@ -146,11 +145,19 @@ repos:
              )$
 
 - repo: https://github.com/DavidAnson/markdownlint-cli2
-  rev: v0.10.0
+  rev: v0.13.0
   hooks:
   - id: markdownlint-cli2
     name: lint markdown
 
+- repo: https://github.com/tcort/markdown-link-check
+  rev: v3.11.2
+  hooks:
+    - id: markdown-link-check
+      args:
+      - -c
+      - .markdown-link-check.json
+
 - repo: local
   hooks:
   - id: buildifier
@@ -181,7 +188,7 @@ repos:
     - --quiet
 
 - repo: https://github.com/psf/black
-  rev: 23.9.1
+  rev: 24.4.1
   hooks:
   - id: black
     name: black python formatter

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,32 @@
 
 All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines.
 
+## 3.6.0 (2024-05-01)
+
+
+### Features
+
+* [Debug Reporting] Enable debug reporting by default
+* [DebugReporting] Add a log to get the libcurl queue time
+* add ability to tag AMIs
+* Add kv e2e tests environment
+* Add markdown-link-check to pre-commit
+* add support for AWS CodeBuild
+* add support for GCP Cloud Build
+* API Updates for Bid Currency Support for Top-Level Seller in Server-Orchestrated Multi-Seller Auctions
+* Enable TCMalloc for all B&A servers
+* support sending Nitro Enclave logs to AWS CloudWatch for debugging
+
+
+### Bug Fixes
+
+* [Temp] Adds buyer bid to top level auctions for Chrome < v125
+* Adjust ps_vlog pre-commit hook to avoid recursing symlinks
+* clean up log verbosity 2
+* **deps:** Upgrade clang-format to v18
+* **deps:** Upgrade pre-commit hooks
+* Ensure that the debug url length is capped
+
 ## 3.5.0 (2024-04-24)
 
 

diff --git a/WORKSPACE b/WORKSPACE
@@ -126,3 +126,15 @@ local_repository(
     name = "tensorflow_v2_14_0",
     path = "services/inference_sidecar/modules/tensorflow_v2_14_0",
 )
+
+http_archive(
+    name = "libevent",
+    build_file = "//third_party:libevent.BUILD",
+    patch_args = ["-p1"],
+    patches = [
+        "//third_party:libevent.patch",
+    ],
+    sha256 = "8836ad722ab211de41cb82fe098911986604f6286f67d10dfb2b6787bf418f49",
+    strip_prefix = "libevent-release-2.1.12-stable",
+    urls = ["https://github.com/libevent/libevent/archive/refs/tags/release-2.1.12-stable.zip"],
+)
diff --git a/api/bidding_auction_servers.proto b/api/bidding_auction_servers.proto
@@ -542,6 +542,20 @@ message SelectAdRequest {
     // object in SelectAdResponse will be encrypted with a public key
     // of the specified cloud platform instead of the shared encryption context.
     EncryptionCloudPlatform top_level_cloud_platform = 11;
+
+    // Optional. For use by the top-level seller in a server-orchestrated
+    // multiseller auction only. Specifies configuration information about the
+    // component sellers.
+    message PerComponentSellerConfig {
+      // The currency in which the top-level seller expects this component seller
+      // to submit bids. Expressed as three capital letters, ISO 4217 suggested.
+      string expected_currency = 1;
+    }
+
+    // Optional. For use by the top-level seller in a server-orchestrated
+    // multiseller auction only. The key in the map corresponds to the seller
+    // identifier in AuctionResult.auction_params.component_seller.
+    map<string, PerComponentSellerConfig> per_component_seller_config = 12;
   }
   message ComponentAuctionResult {
     //  AuctionResult from a server component auction.

diff --git a/builders/.github/workflows/scorecard.yaml b/builders/.github/workflows/scorecard.yaml
@@ -26,14 +26,14 @@ on:
     - cron: '35 10 * * 4'
   push:
     branches:
-    - main
+      - main
 
 # Declare default permissions as read only.
 permissions: read-all
 
 jobs:
   analysis:
-    name: Scorecard analysis
+    name: OpenSSF Scorecard analysis
     runs-on: ubuntu-latest
     permissions:
       # Needed to upload the results to code-scanning dashboard.
@@ -46,12 +46,12 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
           persist-credentials: false
 
       - name: Run analysis
-        uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 # v2.1.2
+        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
         with:
           results_file: results.sarif
           results_format: sarif
@@ -73,14 +73,14 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: Upload artifact
-        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: SARIF file
           path: results.sarif
           retention-days: 5
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: Upload to code-scanning
-        uses: github/codeql-action/upload-sarif@17573ee1cc1b9d061760f3a006fc4aac4f944fd5 # v2.2.4
+        uses: github/codeql-action/upload-sarif@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9
         with:
           sarif_file: results.sarif
diff --git a/builders/CHANGELOG.md b/builders/CHANGELOG.md
@@ -2,6 +2,20 @@
 
 All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines.
 
+## 0.58.0 (2024-04-26)
+
+
+### Features
+
+* add missing AWS env variable for CodeBuild
+
+## 0.57.1 (2024-03-28)
+
+
+### Bug Fixes
+
+* Upgrade OpenSSF scorecard GitHub Action
+
 ## 0.57.0 (2024-03-10)
 
 

diff --git a/builders/images/build-debian/Dockerfile b/builders/images/build-debian/Dockerfile
@@ -18,7 +18,7 @@ ARG BASE_IMAGE=ubuntu:20.04
 # hadolint ignore=DL3006
 FROM ${BASE_IMAGE} as libprofiler-builder
 ENV CC=clang \
-    CXX=clang
+    CXX=clang++
 ADD https://github.com/gperftools/gperftools/releases/download/gperftools-2.13/gperftools-2.13.tar.gz /build/gperftools.tar.gz
 ADD https://apt.llvm.org/llvm.sh /build/llvm.sh
 COPY compile_libprofiler /scripts/

diff --git a/builders/images/build-debian/compile_libprofiler b/builders/images/build-debian/compile_libprofiler
@@ -24,6 +24,7 @@ function install_clang() {
   /build/llvm.sh ${CLANG_VER}
   apt-get --quiet install -y --no-install-recommends libc++-${CLANG_VER}-dev
   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${CLANG_VER} 100
+  update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${CLANG_VER} 100
   rm -f llvm.sh
 
   clang --version

diff --git a/builders/tests/data/hashes/build-debian b/builders/tests/data/hashes/build-debian
@@ -1 +1 @@
-38cc8a23a6a56eb6567bef3685100cd3be1c0491dcc8b953993c42182da3fa40
+35e001149b0e33cba53e9a393c157ef920b3ab2adabcebd07eee0e3d4d9fccf3
diff --git a/builders/tools/builder.sh b/builders/tools/builder.sh
@@ -112,6 +112,7 @@ function builder::add_aws_env_vars() {
     "AWS_REGION"
     "AWS_DEFAULT_REGION"
     "AWS_PROFILE"
+    "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"
   )
 }
 

diff --git a/builders/version.txt b/builders/version.txt
@@ -1 +1 @@
-0.57.0
+0.58.0
diff --git a/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf b/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf
@@ -85,7 +85,7 @@ module "buyer" {
     #    "protectedAppSignalsBiddingWasmHelperUrl": "",
     #    "urlFetchPeriodMs": 13000000,
     #    "urlFetchTimeoutMs": 30000,
-    #    "enableBuyerDebugUrlGeneration": false,
+    #    "enableBuyerDebugUrlGeneration": true,
     #    "enableAdtechCodeLogging": false,
     #    "prepareDataForAdsRetrievalJsUrl": "",
     #    "prepareDataForAdsRetrievalWasmHelperUrl": "",
@@ -115,5 +115,12 @@ module "buyer" {
     INFERENCE_SIDECAR_BINARY_PATH = "" # Example: "/server/bin/inference_sidecar"
     INFERENCE_MODEL_BUCKET_NAME   = "" # Example: "<bucket_name>"
     INFERENCE_MODEL_BUCKET_PATHS  = "" # Example: "<model_path1>,<model_path2>"
+
+    # TCMalloc related config parameters.
+    # See: https://github.com/google/tcmalloc/blob/master/docs/tuning.md
+    BIDDING_TCMALLOC_BACKGROUND_RELEASE_RATE_BYTES_PER_SECOND = "4096"
+    BIDDING_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES             = "10737418240"
+    BFE_TCMALLOC_BACKGROUND_RELEASE_RATE_BYTES_PER_SECOND     = "4096"
+    BFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES                 = "10737418240"
   }
 }
diff --git a/production/deploy/aws/terraform/environment/demo/seller/seller.tf b/production/deploy/aws/terraform/environment/demo/seller/seller.tf
@@ -82,7 +82,7 @@ module "seller" {
     #     "auctionJsUrl": "https://example.com/scoreAd.js",
     #     "urlFetchPeriodMs": 13000000,
     #     "urlFetchTimeoutMs": 30000,
-    #     "enableSellerDebugUrlGeneration": false,
+    #     "enableSellerDebugUrlGeneration": true,
     # This flag should only be set if console.logs from the AdTech code(Ex:scoreAd(), reportResult(), reportWin())
     # execution need to be exported as VLOG.
     # Note: turning on this flag will lead to higher memory consumption for AdTech code execution
@@ -119,5 +119,12 @@ module "seller" {
     KEY_REFRESH_FLOW_RUN_FREQUENCY_SECONDS     = "" # Example: "10800"
     MAX_ALLOWED_SIZE_DEBUG_URL_BYTES           = "" # Example: "65536"
     MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB         = "" # Example: "3000"
+
+    # TCMalloc related config parameters.
+    # See: https://github.com/google/tcmalloc/blob/master/docs/tuning.md
+    AUCTION_TCMALLOC_BACKGROUND_RELEASE_RATE_BYTES_PER_SECOND = "4096"
+    AUCTION_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES             = "10737418240"
+    SFE_TCMALLOC_BACKGROUND_RELEASE_RATE_BYTES_PER_SECOND     = "4096"
+    SFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES                 = "10737418240"
   }
 }