From c98a51c7dc11de92e9c8fb719242a033e620a1b4 Mon Sep 17 00:00:00 2001 From: Daniel Kocoj Date: Tue, 28 May 2024 13:28:32 -0500 Subject: [PATCH] Release 3.8.0 (2024-05-24) ### Features * Add tee-container-log-redirect option in terraform * Create logging library for inference consented logs * Forward per request consented debugging config to roma callback * Support CPU isolation in the inference sidecar ### Bug Fixes * [IsolateBuyerAndSellerCodeExecution] Add a new code wrapper with only scoreAd and reportResult * clean up log verbosity 3 Bug: N/A GitOrigin-RevId: 52b1ca1a4c43e0769bcc74cb09ff6730e8e4c62e Change-Id: Ib016484ad09e88f320e726136adfe2b7a29383a2 --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 16 + WORKSPACE | 8 +- .../terraform/environment/demo/buyer/buyer.tf | 1 + .../environment/demo/seller/seller.tf | 1 + .../gcp/terraform/modules/buyer/service.tf | 1 + .../terraform/modules/buyer/service_vars.tf | 6 + .../gcp/terraform/modules/seller/service.tf | 1 + .../terraform/modules/seller/service_vars.tf | 6 + .../terraform/services/autoscaling/main.tf | 4 +- .../services/autoscaling/variables.tf | 6 + .../dashboards/buyer_dashboard/main.tf | 48 +- .../dashboards/seller_dashboard/main.tf | 74 +-- .../code_wrapper/seller_code_wrapper.cc | 10 + .../code_wrapper/seller_code_wrapper.h | 160 +++--- .../code_wrapper/seller_code_wrapper_test.cc | 10 + .../seller_code_wrapper_test_constants.h | 484 ++++++++++-------- services/auction_service/score_ads_reactor.cc | 8 +- .../base_generate_bids_reactor.h | 6 + services/bidding_service/bidding_main.cc | 3 +- .../bidding_service/generate_bids_reactor.cc | 5 +- services/bidding_service/inference/BUILD | 1 + .../inference/inference_flags.cc | 1 + .../inference/inference_utils.cc | 5 +- .../inference/inference_utils.h | 6 +- .../inference/inference_utils_test.cc | 11 +- ...ected_app_signals_generate_bids_reactor.cc | 6 +- ...tected_app_signals_generate_bids_reactor.h | 7 +- .../get_bids_unary_reactor.cc | 12 +- services/common/clients/code_dispatcher/BUILD | 14 + .../code_dispatcher/request_context.cc | 57 +++ .../clients/code_dispatcher/request_context.h | 88 ++++ .../clients/code_dispatcher/v8_dispatcher.h | 13 +- .../buyer_key_value_async_http_client.cc | 2 +- .../code_dispatch/code_dispatch_reactor.h | 2 +- services/common/encryption/BUILD | 1 + .../common/encryption/key_fetcher_factory.cc | 4 +- services/common/test/mocks.h | 2 +- services/common/util/BUILD | 1 + services/common/util/async_task_tracker.cc | 8 +- services/common/util/binary_http_utils.h | 4 +- services/common/util/reporting_util.cc | 2 +- .../common/util/request_response_constants.h | 13 +- services/inference_sidecar/common/BUILD | 1 + .../inference_sidecar/common/grpc_sidecar.cc | 10 + .../inference_sidecar/common/grpc_sidecar.h | 2 + .../common/inference_sidecar_main.cc | 4 + .../common/proto/inference_sidecar.proto | 15 + .../inference_sidecar/common/test_constants.h | 8 +- services/inference_sidecar/common/utils/BUILD | 21 + services/inference_sidecar/common/utils/log.h | 83 +++ .../common/utils/log_test.cc | 64 +++ .../select_ad_reactor.cc | 13 +- .../select_ad_reactor_app.cc | 16 +- .../util/encryption_util.cc | 5 +- .../seller_frontend_service/util/web_utils.cc | 4 +- version.txt | 2 +- 57 files changed, 975 insertions(+), 393 deletions(-) create mode 100644 services/common/clients/code_dispatcher/request_context.cc create mode 100644 services/common/clients/code_dispatcher/request_context.h create mode 100644 services/inference_sidecar/common/utils/log.h create mode 100644 services/inference_sidecar/common/utils/log_test.cc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e42bbf6..f0e755d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -121,7 +121,7 @@ repos: - id: ps_vlog_123 name: Don't use PS_VLOG(0/1/2/3) - entry: sh -c '! grep --files-with-matches --recursive -E "PS_VLOG[(][012][,)]" services/' + entry: sh -c '! grep --files-with-matches --recursive -E "PS_VLOG[(][0123][,)]" services/' language: system types_or: - c++ diff --git a/CHANGELOG.md b/CHANGELOG.md index a11407b9..61040347 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines. +## 3.8.0 (2024-05-24) + + +### Features + +* Add tee-container-log-redirect option in terraform +* Create logging library for inference consented logs +* Forward per request consented debugging config to roma callback +* Support CPU isolation in the inference sidecar + + +### Bug Fixes + +* [IsolateBuyerAndSellerCodeExecution] Add a new code wrapper with only scoreAd and reportResult +* clean up log verbosity 3 + ## 3.7.0 (2024-05-13) diff --git a/WORKSPACE b/WORKSPACE index ea223048..ca1131c8 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -23,11 +23,11 @@ http_archive( http_archive( name = "google_privacysandbox_servers_common", - # 2024-05-06 - sha256 = "1b0d52c6218fa4db7f4760a7e197c2e656a1408549fee40cdea2cb7496a3f836", - strip_prefix = "data-plane-shared-libraries-10be592dd582044a79172d4c0530ce8b7ac39ae4", + # 2024-05-24 + sha256 = "444f76d69afba08dfe7424dae0ff8afb26bd3f4e3396d76b7bf55668d3afd7c6", + strip_prefix = "data-plane-shared-libraries-3e09f7993cf6e4ef0fd33360b778f993d3cfd9ef", urls = [ - "https://github.com/privacysandbox/data-plane-shared-libraries/archive/10be592dd582044a79172d4c0530ce8b7ac39ae4.zip", + "https://github.com/privacysandbox/data-plane-shared-libraries/archive/3e09f7993cf6e4ef0fd33360b778f993d3cfd9ef.zip", ], ) diff --git a/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf b/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf index e5dc94f8..10cb10dc 100644 --- a/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf +++ b/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf @@ -173,4 +173,5 @@ module "buyer" { } } } + enable_tee_container_log_redirect = false } diff --git a/production/deploy/gcp/terraform/environment/demo/seller/seller.tf b/production/deploy/gcp/terraform/environment/demo/seller/seller.tf index 8f8cd2c7..67bad96d 100644 --- a/production/deploy/gcp/terraform/environment/demo/seller/seller.tf +++ b/production/deploy/gcp/terraform/environment/demo/seller/seller.tf @@ -174,4 +174,5 @@ module "seller" { } } } + enable_tee_container_log_redirect = false } diff --git a/production/deploy/gcp/terraform/modules/buyer/service.tf b/production/deploy/gcp/terraform/modules/buyer/service.tf index 58ae4f4c..a1054e10 100644 --- a/production/deploy/gcp/terraform/modules/buyer/service.tf +++ b/production/deploy/gcp/terraform/modules/buyer/service.tf @@ -62,6 +62,7 @@ module "autoscaling" { instance_template_waits_for_instances = var.instance_template_waits_for_instances depends_on = [module.security, module.networking, resource.google_secret_manager_secret.runtime_flag_secrets, resource.google_secret_manager_secret_version.runtime_flag_secret_values] collector_startup_script = var.collector_startup_script + enable_tee_container_log_redirect = var.enable_tee_container_log_redirect } module "load_balancing" { diff --git a/production/deploy/gcp/terraform/modules/buyer/service_vars.tf b/production/deploy/gcp/terraform/modules/buyer/service_vars.tf index 0e9b2bb2..e538b483 100644 --- a/production/deploy/gcp/terraform/modules/buyer/service_vars.tf +++ b/production/deploy/gcp/terraform/modules/buyer/service_vars.tf @@ -139,3 +139,9 @@ variable "fast_nat" { type = bool default = false } + +variable "enable_tee_container_log_redirect" { + description = "If true, redirect the TEE container logs to the VM's serial port." + type = bool + default = true +} diff --git a/production/deploy/gcp/terraform/modules/seller/service.tf b/production/deploy/gcp/terraform/modules/seller/service.tf index 936ca68b..1c61431e 100644 --- a/production/deploy/gcp/terraform/modules/seller/service.tf +++ b/production/deploy/gcp/terraform/modules/seller/service.tf @@ -60,6 +60,7 @@ module "autoscaling" { instance_template_waits_for_instances = var.instance_template_waits_for_instances depends_on = [module.security, module.networking, resource.google_secret_manager_secret.runtime_flag_secrets, resource.google_secret_manager_secret_version.runtime_flag_secret_values] collector_startup_script = var.collector_startup_script + enable_tee_container_log_redirect = var.enable_tee_container_log_redirect } module "load_balancing" { diff --git a/production/deploy/gcp/terraform/modules/seller/service_vars.tf b/production/deploy/gcp/terraform/modules/seller/service_vars.tf index d59332b1..bf4b96a1 100644 --- a/production/deploy/gcp/terraform/modules/seller/service_vars.tf +++ b/production/deploy/gcp/terraform/modules/seller/service_vars.tf @@ -144,3 +144,9 @@ variable "fast_nat" { type = bool default = false } + +variable "enable_tee_container_log_redirect" { + description = "If true, redirect the TEE container logs to the VM's serial port." + type = bool + default = true +} diff --git a/production/deploy/gcp/terraform/services/autoscaling/main.tf b/production/deploy/gcp/terraform/services/autoscaling/main.tf index de732415..ff59877d 100644 --- a/production/deploy/gcp/terraform/services/autoscaling/main.tf +++ b/production/deploy/gcp/terraform/services/autoscaling/main.tf @@ -91,7 +91,7 @@ resource "google_compute_instance_template" "frontends" { } metadata = { tee-image-reference = var.frontend_tee_image - tee-container-log-redirect = true + tee-container-log-redirect = var.enable_tee_container_log_redirect tee-impersonate-service-accounts = var.tee_impersonate_service_accounts mesh-name = var.mesh_name environment = var.environment @@ -248,7 +248,7 @@ resource "google_compute_instance_template" "backends" { metadata = { mesh-name = var.mesh_name tee-image-reference = var.backend_tee_image - tee-container-log-redirect = true + tee-container-log-redirect = var.enable_tee_container_log_redirect tee-impersonate-service-accounts = var.tee_impersonate_service_accounts operator = var.operator } diff --git a/production/deploy/gcp/terraform/services/autoscaling/variables.tf b/production/deploy/gcp/terraform/services/autoscaling/variables.tf index 9b22317a..3429848a 100644 --- a/production/deploy/gcp/terraform/services/autoscaling/variables.tf +++ b/production/deploy/gcp/terraform/services/autoscaling/variables.tf @@ -168,3 +168,9 @@ variable "collector_startup_script" { description = "Script to configure and start the otel collector." type = string } + +variable "enable_tee_container_log_redirect" { + description = "If true, redirect the TEE container logs to the VM's serial port." + type = bool + default = true +} diff --git a/production/deploy/gcp/terraform/services/dashboards/buyer_dashboard/main.tf b/production/deploy/gcp/terraform/services/dashboards/buyer_dashboard/main.tf index 23ff31c3..31bce711 100644 --- a/production/deploy/gcp/terraform/services/dashboards/buyer_dashboard/main.tf +++ b/production/deploy/gcp/terraform/services/dashboards/buyer_dashboard/main.tf @@ -314,7 +314,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -353,12 +353,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/bfe.initiated_request.to_bidding.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"status_code\"", @@ -398,12 +398,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/js_execution.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"service_name\"", @@ -442,12 +442,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/bidding.business_logic.bids_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"service_name\"", @@ -485,12 +485,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/bidding.business_logic.zero_bid_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"service_name\"", @@ -529,7 +529,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -569,12 +569,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/initiated_request.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"server_name\"", @@ -613,7 +613,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -652,7 +652,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -691,7 +691,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -730,7 +730,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -770,7 +770,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -810,12 +810,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/bfe.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"error_code\"", @@ -854,12 +854,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/bidding.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"error_code\"", @@ -899,12 +899,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"status_code\"", @@ -987,7 +987,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", diff --git a/production/deploy/gcp/terraform/services/dashboards/seller_dashboard/main.tf b/production/deploy/gcp/terraform/services/dashboards/seller_dashboard/main.tf index d2b60f3c..5277a446 100644 --- a/production/deploy/gcp/terraform/services/dashboards/seller_dashboard/main.tf +++ b/production/deploy/gcp/terraform/services/dashboards/seller_dashboard/main.tf @@ -314,7 +314,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -353,12 +353,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_bfe.errors_count_by_status\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"status_code\"", @@ -398,12 +398,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/js_execution.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"service_name\"", @@ -442,12 +442,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/auction.business_logic.bids_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"service_name\"", @@ -485,12 +485,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/auction.business_logic.bid_rejected_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"seller_rejection_reason\"", @@ -530,7 +530,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -569,12 +569,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/initiated_request.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"server_name\"", @@ -613,7 +613,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -652,7 +652,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -691,7 +691,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -730,7 +730,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -769,7 +769,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -808,12 +808,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/sfe.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"error_code\"", @@ -852,12 +852,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/auction.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"error_code\"", @@ -897,12 +897,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/initiated_request.to_kv.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"status_code\"", @@ -942,12 +942,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_auction.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"status_code\"", @@ -1031,7 +1031,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", @@ -1112,12 +1112,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_bfe.errors_count_by_buyer\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"buyer\"", @@ -1157,12 +1157,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_bfe.count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"buyer\"", @@ -1225,12 +1225,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_request.to_bfe.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"buyer\"", @@ -1269,12 +1269,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/sfe.initiated_response.to_bfe.size_bytes\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"buyer\"", @@ -1475,12 +1475,12 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "perSeriesAligner": "ALIGN_RATE" }, "filter": "metric.type=\"workload.googleapis.com/sfe.business_logic.request_with_winner_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", "secondaryAggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_MEAN", "groupByFields": [ "metric.label.\"service_name\"", @@ -1519,7 +1519,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "timeSeriesQuery": { "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "300s", "crossSeriesReducer": "REDUCE_PERCENTILE_95", "groupByFields": [ "metric.label.\"service_name\"", diff --git a/services/auction_service/code_wrapper/seller_code_wrapper.cc b/services/auction_service/code_wrapper/seller_code_wrapper.cc index e0a6ac00..2bd65540 100644 --- a/services/auction_service/code_wrapper/seller_code_wrapper.cc +++ b/services/auction_service/code_wrapper/seller_code_wrapper.cc @@ -142,4 +142,14 @@ std::string GetSellerWrappedCode( return wrap_code; } +std::string GetSellerWrappedCode(absl::string_view seller_js_code, + bool enable_report_result_url_generation) { + std::string wrap_code{kEntryFunction}; + if (enable_report_result_url_generation) { + wrap_code.append(kReportResultWrapperFunction); + } + wrap_code.append(seller_js_code); + return wrap_code; +} + } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/code_wrapper/seller_code_wrapper.h b/services/auction_service/code_wrapper/seller_code_wrapper.h index dc9ebf9e..70b88723 100644 --- a/services/auction_service/code_wrapper/seller_code_wrapper.h +++ b/services/auction_service/code_wrapper/seller_code_wrapper.h @@ -38,34 +38,29 @@ inline constexpr char kReportWinWrapperFunctionName[] = "reportWinWrapper"; // This wrapper supports the features below: //- Exporting logs to Auction Service using console.log constexpr absl::string_view kEntryFunction = R"JS_CODE( + var forDebuggingOnly_auction_loss_url = undefined; + var forDebuggingOnly_auction_win_url = undefined; + const forDebuggingOnly = {}; + forDebuggingOnly.reportAdAuctionLoss = function(url){ + forDebuggingOnly_auction_loss_url = url; + } + forDebuggingOnly.reportAdAuctionWin = function(url){ + forDebuggingOnly_auction_win_url = url; + } + globalThis.forDebuggingOnly = forDebuggingOnly; + function scoreAdEntryFunction(adMetadata, bid, auctionConfig, trustedScoringSignals, browserSignals, directFromSellerSignals, featureFlags){ - var ps_logs = []; - var ps_errors = []; - var ps_warns = []; - if(featureFlags.enable_logging){ - console.log = function(...args) { - ps_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_errors.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_warns.push(JSON.stringify(args)) - } - } - - var forDebuggingOnly_auction_loss_url = undefined; - var forDebuggingOnly_auction_win_url = undefined; - const forDebuggingOnly = {}; - forDebuggingOnly.reportAdAuctionLoss = function(url){ - forDebuggingOnly_auction_loss_url = url; - } - forDebuggingOnly.reportAdAuctionWin = function(url){ - forDebuggingOnly_auction_win_url = url; - } - globalThis.forDebuggingOnly = forDebuggingOnly; - + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (featureFlags.enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } var scoreAdResponse = {}; try { scoreAdResponse = scoreAd(adMetadata, bid, auctionConfig, @@ -91,6 +86,58 @@ constexpr absl::string_view kEntryFunction = R"JS_CODE( } )JS_CODE"; +inline constexpr absl::string_view kReportResultWrapperFunction = + R"JSCODE( + //Handler method to call adTech provided reportResult method and wrap the + // response with reportResult url and interaction reporting urls. + function reportingResultEntryFunction(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging) { + ps_signalsForWinner = "" + const ps_report_result_response = { + reportResultUrl : "", + interactionReportingUrls : {}, + sendReportToInvoked : false, + registerAdBeaconInvoked : false, + } + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } + globalThis.sendReportTo = function sendReportTo(url){ + if(ps_report_result_response.sendReportToInvoked) { + throw new Error("sendReportTo function invoked more than once"); + } + ps_report_result_response.reportResultUrl = url; + ps_report_result_response.sendReportToInvoked = true; + } + globalThis.registerAdBeacon = function registerAdBeacon(eventUrlMap){ + if(ps_report_result_response.registerAdBeaconInvoked) { + throw new Error("registerAdBeaconInvoked function invoked more than once"); + } + ps_report_result_response.interactionReportingUrls=eventUrlMap; + ps_report_result_response.registerAdBeaconInvoked = true; + } + try{ + ps_signalsForWinner = reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals); + } catch(ex){ + console.error(ex.message) + } + return { + signalsForWinner: ps_signalsForWinner, + interactionReportingUrls: ps_report_result_response.interactionReportingUrls, + reportResultUrl: ps_report_result_response.reportResultUrl + logs: ps_logs, + errors: ps_errors, + warnings: ps_warns + } + } +)JSCODE"; + // The function that will be called by Roma to generate reporting urls. // The dispatch function name will be reportingEntryFunction. // This wrapper supports the features below: @@ -105,26 +152,22 @@ inline constexpr absl::string_view kReportingEntryFunction = // response with reportResult url and interaction reporting urls. function reportingEntryFunction$suffix(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging, buyerReportingMetadata, $extraArgs) { ps_signalsForWinner = "" - var ps_report_result_response = { + const ps_report_result_response = { reportResultUrl : "", interactionReportingUrls : {}, sendReportToInvoked : false, registerAdBeaconInvoked : false, } - var ps_logs = []; - var ps_errors = []; - var ps_warns = []; - if(enable_logging){ - console.log = function(...args) { - ps_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_errors.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_warns.push(JSON.stringify(args)) - } - } + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } globalThis.sendReportTo = function sendReportTo(url){ if(ps_report_result_response.sendReportToInvoked) { throw new Error("sendReportTo function invoked more than once"); @@ -199,26 +242,22 @@ inline constexpr absl::string_view kReportingWinWrapperTemplate = // response with reportWin url and interaction reporting urls. function $reportWinWrapperName(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, directFromSellerSignals, enable_logging, $extraArgs) { - var ps_report_win_response = { + const ps_report_win_response = { reportWinUrl : "", interactionReportingUrls : {}, sendReportToInvoked : false, registerAdBeaconInvoked : false, } - var ps_buyer_logs = []; - var ps_buyer_error_logs = []; - var ps_buyer_warning_logs = []; - if(enable_logging){ - console.log = function(...args) { - ps_buyer_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_buyer_error_logs.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_buyer_warning_logs.push(JSON.stringify(args)) - } - } + const ps_buyer_logs = []; + const ps_buyer_error_logs = []; + const ps_buyer_warning_logs = []; + if (enable_logging) { + console.log = (...args) => ps_buyer_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_buyer_warning_logs.push(JSON.stringify(args)); + console.error = (...args) => ps_buyer_error_logs.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } globalThis.sendReportTo = function sendReportTo(url){ if(ps_report_win_response.sendReportToInvoked) { throw new Error("sendReportTo function invoked more than once"); @@ -267,7 +306,14 @@ std::string GetSellerWrappedCode( const absl::flat_hash_map& buyer_origin_code_map, const absl::flat_hash_map& protected_app_signals_buyer_origin_code_map); - +// Returns the complete wrapped code for Seller. +// The function adds wrappers to the Seller provided scoreAd and reportResult +// UDF. The wrapper supports: +// - Generation of event level reporting urls for Seller +// - Generation of event level debug reporting +// - Exporting console.logs from the AdTech execution. +std::string GetSellerWrappedCode(absl::string_view seller_js_code, + bool enable_report_result_url_generation); } // namespace privacy_sandbox::bidding_auction_servers #endif // FLEDGE_SERVICES_SELLER_CODE_WRAPPER_H_ diff --git a/services/auction_service/code_wrapper/seller_code_wrapper_test.cc b/services/auction_service/code_wrapper/seller_code_wrapper_test.cc index b3c124e8..28a0b167 100644 --- a/services/auction_service/code_wrapper/seller_code_wrapper_test.cc +++ b/services/auction_service/code_wrapper/seller_code_wrapper_test.cc @@ -38,6 +38,16 @@ TEST(GetSellerWrappedCode, GeneratesCompleteFinalCode) { << "Expected: " << kExpectedFinalCode; } +TEST(GetSellerWrappedCode, GeneratesWrappedCodeWithScoreAdAndReportResult) { + bool enable_report_result_url_generation = true; + std::string observed = GetSellerWrappedCode( + kSellerBaseCode, enable_report_result_url_generation); + EXPECT_EQ(observed, kExpectedSellerCodeWithScoreAdAndReportResult) + << "Observed:\n" + << observed << "\n\n" + << "Expected: " << kExpectedSellerCodeWithScoreAdAndReportResult; +} + TEST(GetSellerWrappedCode, GeneratesCompleteFinalCodeWithProtectedAppSignals) { bool enable_report_result_url_generation = true; bool enable_report_win_url_generation = true; diff --git a/services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h b/services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h index 55110ef3..247f020a 100644 --- a/services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h +++ b/services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h @@ -243,34 +243,29 @@ constexpr absl::string_view kTopLevelAuctionCode = R"JS_CODE( )JS_CODE"; constexpr absl::string_view kExpectedFinalCode = R"JS_CODE( + var forDebuggingOnly_auction_loss_url = undefined; + var forDebuggingOnly_auction_win_url = undefined; + const forDebuggingOnly = {}; + forDebuggingOnly.reportAdAuctionLoss = function(url){ + forDebuggingOnly_auction_loss_url = url; + } + forDebuggingOnly.reportAdAuctionWin = function(url){ + forDebuggingOnly_auction_win_url = url; + } + globalThis.forDebuggingOnly = forDebuggingOnly; + function scoreAdEntryFunction(adMetadata, bid, auctionConfig, trustedScoringSignals, browserSignals, directFromSellerSignals, featureFlags){ - var ps_logs = []; - var ps_errors = []; - var ps_warns = []; - if(featureFlags.enable_logging){ - console.log = function(...args) { - ps_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_errors.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_warns.push(JSON.stringify(args)) - } - } - - var forDebuggingOnly_auction_loss_url = undefined; - var forDebuggingOnly_auction_win_url = undefined; - const forDebuggingOnly = {}; - forDebuggingOnly.reportAdAuctionLoss = function(url){ - forDebuggingOnly_auction_loss_url = url; - } - forDebuggingOnly.reportAdAuctionWin = function(url){ - forDebuggingOnly_auction_win_url = url; - } - globalThis.forDebuggingOnly = forDebuggingOnly; - + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (featureFlags.enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } var scoreAdResponse = {}; try { scoreAdResponse = scoreAd(adMetadata, bid, auctionConfig, @@ -299,26 +294,22 @@ constexpr absl::string_view kExpectedFinalCode = R"JS_CODE( // response with reportResult url and interaction reporting urls. function reportingEntryFunction(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging, buyerReportingMetadata, ) { ps_signalsForWinner = "" - var ps_report_result_response = { + const ps_report_result_response = { reportResultUrl : "", interactionReportingUrls : {}, sendReportToInvoked : false, registerAdBeaconInvoked : false, } - var ps_logs = []; - var ps_errors = []; - var ps_warns = []; - if(enable_logging){ - console.log = function(...args) { - ps_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_errors.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_warns.push(JSON.stringify(args)) - } - } + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } globalThis.sendReportTo = function sendReportTo(url){ if(ps_report_result_response.sendReportToInvoked) { throw new Error("sendReportTo function invoked more than once"); @@ -390,26 +381,22 @@ constexpr absl::string_view kExpectedFinalCode = R"JS_CODE( // response with reportWin url and interaction reporting urls. function reportWinWrapperhttpbuyer1com(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, directFromSellerSignals, enable_logging, ) { - var ps_report_win_response = { + const ps_report_win_response = { reportWinUrl : "", interactionReportingUrls : {}, sendReportToInvoked : false, registerAdBeaconInvoked : false, } - var ps_buyer_logs = []; - var ps_buyer_error_logs = []; - var ps_buyer_warning_logs = []; - if(enable_logging){ - console.log = function(...args) { - ps_buyer_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_buyer_error_logs.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_buyer_warning_logs.push(JSON.stringify(args)) - } - } + const ps_buyer_logs = []; + const ps_buyer_error_logs = []; + const ps_buyer_warning_logs = []; + if (enable_logging) { + console.log = (...args) => ps_buyer_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_buyer_warning_logs.push(JSON.stringify(args)); + console.error = (...args) => ps_buyer_error_logs.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } globalThis.sendReportTo = function sendReportTo(url){ if(ps_report_win_response.sendReportToInvoked) { throw new Error("sendReportTo function invoked more than once"); @@ -499,35 +486,156 @@ constexpr absl::string_view kExpectedFinalCode = R"JS_CODE( } )JS_CODE"; -constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( +constexpr absl::string_view kExpectedSellerCodeWithScoreAdAndReportResult = + R"JS_CODE( + var forDebuggingOnly_auction_loss_url = undefined; + var forDebuggingOnly_auction_win_url = undefined; + const forDebuggingOnly = {}; + forDebuggingOnly.reportAdAuctionLoss = function(url){ + forDebuggingOnly_auction_loss_url = url; + } + forDebuggingOnly.reportAdAuctionWin = function(url){ + forDebuggingOnly_auction_win_url = url; + } + globalThis.forDebuggingOnly = forDebuggingOnly; + function scoreAdEntryFunction(adMetadata, bid, auctionConfig, trustedScoringSignals, browserSignals, directFromSellerSignals, featureFlags){ - var ps_logs = []; - var ps_errors = []; - var ps_warns = []; - if(featureFlags.enable_logging){ - console.log = function(...args) { - ps_logs.push(JSON.stringify(args)) + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (featureFlags.enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } + var scoreAdResponse = {}; + try { + scoreAdResponse = scoreAd(adMetadata, bid, auctionConfig, + trustedScoringSignals, browserSignals, directFromSellerSignals); + } catch({error, message}) { + console.error("[Error: " + error + "; Message: " + message + "]"); + } finally { + if( featureFlags.enable_debug_url_generation && + (forDebuggingOnly_auction_loss_url + || forDebuggingOnly_auction_win_url)) { + scoreAdResponse.debugReportUrls = { + auctionDebugLossUrl: forDebuggingOnly_auction_loss_url, + auctionDebugWinUrl: forDebuggingOnly_auction_win_url + } } - console.error = function(...args) { - ps_errors.push(JSON.stringify(args)) + } + return { + response: scoreAdResponse, + logs: ps_logs, + errors: ps_errors, + warnings: ps_warns + } + } + + //Handler method to call adTech provided reportResult method and wrap the + // response with reportResult url and interaction reporting urls. + function reportingResultEntryFunction(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging) { + ps_signalsForWinner = "" + const ps_report_result_response = { + reportResultUrl : "", + interactionReportingUrls : {}, + sendReportToInvoked : false, + registerAdBeaconInvoked : false, + } + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } + globalThis.sendReportTo = function sendReportTo(url){ + if(ps_report_result_response.sendReportToInvoked) { + throw new Error("sendReportTo function invoked more than once"); } - console.warn = function(...args) { - ps_warns.push(JSON.stringify(args)) + ps_report_result_response.reportResultUrl = url; + ps_report_result_response.sendReportToInvoked = true; + } + globalThis.registerAdBeacon = function registerAdBeacon(eventUrlMap){ + if(ps_report_result_response.registerAdBeaconInvoked) { + throw new Error("registerAdBeaconInvoked function invoked more than once"); } + ps_report_result_response.interactionReportingUrls=eventUrlMap; + ps_report_result_response.registerAdBeaconInvoked = true; } - - var forDebuggingOnly_auction_loss_url = undefined; - var forDebuggingOnly_auction_win_url = undefined; - const forDebuggingOnly = {}; - forDebuggingOnly.reportAdAuctionLoss = function(url){ - forDebuggingOnly_auction_loss_url = url; + try{ + ps_signalsForWinner = reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals); + } catch(ex){ + console.error(ex.message) } - forDebuggingOnly.reportAdAuctionWin = function(url){ - forDebuggingOnly_auction_win_url = url; + return { + signalsForWinner: ps_signalsForWinner, + interactionReportingUrls: ps_report_result_response.interactionReportingUrls, + reportResultUrl: ps_report_result_response.reportResultUrl + logs: ps_logs, + errors: ps_errors, + warnings: ps_warns } - globalThis.forDebuggingOnly = forDebuggingOnly; + } + function fibonacci(num) { + if (num <= 1) return 1; + return fibonacci(num - 1) + fibonacci(num - 2); + } + + function scoreAd(ad_metadata, bid, auction_config, scoring_signals, bid_metadata, directFromSellerSignals){ + // Do a random amount of work to generate the score: + const score = fibonacci(Math.floor(Math.random() * 10 + 1)); + console.log("Logging from ScoreAd") + console.error("Logging error from ScoreAd") + console.warn("Logging warn from ScoreAd") + return { + desirability: score, + allow_component_auction: false + } + } + function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ + console.log("Logging from ReportResult"); + if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ + sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) + } else { + sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller) + } + registerAdBeacon({"clickEvent":"http://click.com"}) + return "testSignalsForWinner" + } +)JS_CODE"; + +constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( + var forDebuggingOnly_auction_loss_url = undefined; + var forDebuggingOnly_auction_win_url = undefined; + const forDebuggingOnly = {}; + forDebuggingOnly.reportAdAuctionLoss = function(url){ + forDebuggingOnly_auction_loss_url = url; + } + forDebuggingOnly.reportAdAuctionWin = function(url){ + forDebuggingOnly_auction_win_url = url; + } + globalThis.forDebuggingOnly = forDebuggingOnly; + + function scoreAdEntryFunction(adMetadata, bid, auctionConfig, trustedScoringSignals, + browserSignals, directFromSellerSignals, featureFlags){ + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (featureFlags.enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } var scoreAdResponse = {}; try { scoreAdResponse = scoreAd(adMetadata, bid, auctionConfig, @@ -556,26 +664,22 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( // response with reportResult url and interaction reporting urls. function reportingEntryFunctionProtectedAppSignals(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging, buyerReportingMetadata, egressPayload, temporaryUnlimitedEgressPayload) { ps_signalsForWinner = "" - var ps_report_result_response = { + const ps_report_result_response = { reportResultUrl : "", interactionReportingUrls : {}, sendReportToInvoked : false, registerAdBeaconInvoked : false, } - var ps_logs = []; - var ps_errors = []; - var ps_warns = []; - if(enable_logging){ - console.log = function(...args) { - ps_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_errors.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_warns.push(JSON.stringify(args)) - } - } + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } globalThis.sendReportTo = function sendReportTo(url){ if(ps_report_result_response.sendReportToInvoked) { throw new Error("sendReportTo function invoked more than once"); @@ -647,26 +751,22 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( // response with reportWin url and interaction reporting urls. function reportWinWrapperhttpbuyer1comProtectedAppSignals(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, directFromSellerSignals, enable_logging, egressPayload, temporaryUnlimitedEgressPayload) { - var ps_report_win_response = { + const ps_report_win_response = { reportWinUrl : "", interactionReportingUrls : {}, sendReportToInvoked : false, registerAdBeaconInvoked : false, } - var ps_buyer_logs = []; - var ps_buyer_error_logs = []; - var ps_buyer_warning_logs = []; - if(enable_logging){ - console.log = function(...args) { - ps_buyer_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_buyer_error_logs.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_buyer_warning_logs.push(JSON.stringify(args)) - } - } + const ps_buyer_logs = []; + const ps_buyer_error_logs = []; + const ps_buyer_warning_logs = []; + if (enable_logging) { + console.log = (...args) => ps_buyer_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_buyer_warning_logs.push(JSON.stringify(args)); + console.error = (...args) => ps_buyer_error_logs.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } globalThis.sendReportTo = function sendReportTo(url){ if(ps_report_win_response.sendReportToInvoked) { throw new Error("sendReportTo function invoked more than once"); @@ -709,26 +809,22 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( // response with reportResult url and interaction reporting urls. function reportingEntryFunction(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging, buyerReportingMetadata, ) { ps_signalsForWinner = "" - var ps_report_result_response = { + const ps_report_result_response = { reportResultUrl : "", interactionReportingUrls : {}, sendReportToInvoked : false, registerAdBeaconInvoked : false, } - var ps_logs = []; - var ps_errors = []; - var ps_warns = []; - if(enable_logging){ - console.log = function(...args) { - ps_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_errors.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_warns.push(JSON.stringify(args)) - } - } + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } globalThis.sendReportTo = function sendReportTo(url){ if(ps_report_result_response.sendReportToInvoked) { throw new Error("sendReportTo function invoked more than once"); @@ -800,26 +896,22 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( // response with reportWin url and interaction reporting urls. function reportWinWrapperhttpbuyer1com(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, directFromSellerSignals, enable_logging, ) { - var ps_report_win_response = { + const ps_report_win_response = { reportWinUrl : "", interactionReportingUrls : {}, sendReportToInvoked : false, registerAdBeaconInvoked : false, } - var ps_buyer_logs = []; - var ps_buyer_error_logs = []; - var ps_buyer_warning_logs = []; - if(enable_logging){ - console.log = function(...args) { - ps_buyer_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_buyer_error_logs.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_buyer_warning_logs.push(JSON.stringify(args)) - } - } + const ps_buyer_logs = []; + const ps_buyer_error_logs = []; + const ps_buyer_warning_logs = []; + if (enable_logging) { + console.log = (...args) => ps_buyer_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_buyer_warning_logs.push(JSON.stringify(args)); + console.error = (...args) => ps_buyer_error_logs.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } globalThis.sendReportTo = function sendReportTo(url){ if(ps_report_win_response.sendReportToInvoked) { throw new Error("sendReportTo function invoked more than once"); @@ -882,34 +974,29 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( )JS_CODE"; constexpr absl::string_view kExpectedCodeWithReportWinDisabled = R"JS_CODE( + var forDebuggingOnly_auction_loss_url = undefined; + var forDebuggingOnly_auction_win_url = undefined; + const forDebuggingOnly = {}; + forDebuggingOnly.reportAdAuctionLoss = function(url){ + forDebuggingOnly_auction_loss_url = url; + } + forDebuggingOnly.reportAdAuctionWin = function(url){ + forDebuggingOnly_auction_win_url = url; + } + globalThis.forDebuggingOnly = forDebuggingOnly; + function scoreAdEntryFunction(adMetadata, bid, auctionConfig, trustedScoringSignals, browserSignals, directFromSellerSignals, featureFlags){ - var ps_logs = []; - var ps_errors = []; - var ps_warns = []; - if(featureFlags.enable_logging){ - console.log = function(...args) { - ps_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_errors.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_warns.push(JSON.stringify(args)) - } - } - - var forDebuggingOnly_auction_loss_url = undefined; - var forDebuggingOnly_auction_win_url = undefined; - const forDebuggingOnly = {}; - forDebuggingOnly.reportAdAuctionLoss = function(url){ - forDebuggingOnly_auction_loss_url = url; - } - forDebuggingOnly.reportAdAuctionWin = function(url){ - forDebuggingOnly_auction_win_url = url; - } - globalThis.forDebuggingOnly = forDebuggingOnly; - + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (featureFlags.enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } var scoreAdResponse = {}; try { scoreAdResponse = scoreAd(adMetadata, bid, auctionConfig, @@ -938,26 +1025,22 @@ constexpr absl::string_view kExpectedCodeWithReportWinDisabled = R"JS_CODE( // response with reportResult url and interaction reporting urls. function reportingEntryFunction(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging, buyerReportingMetadata, ) { ps_signalsForWinner = "" - var ps_report_result_response = { + const ps_report_result_response = { reportResultUrl : "", interactionReportingUrls : {}, sendReportToInvoked : false, registerAdBeaconInvoked : false, } - var ps_logs = []; - var ps_errors = []; - var ps_warns = []; - if(enable_logging){ - console.log = function(...args) { - ps_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_errors.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_warns.push(JSON.stringify(args)) - } - } + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } globalThis.sendReportTo = function sendReportTo(url){ if(ps_report_result_response.sendReportToInvoked) { throw new Error("sendReportTo function invoked more than once"); @@ -1054,34 +1137,29 @@ constexpr absl::string_view kExpectedCodeWithReportWinDisabled = R"JS_CODE( )JS_CODE"; constexpr absl::string_view kExpectedCodeWithReportingDisabled = R"JS_CODE( + var forDebuggingOnly_auction_loss_url = undefined; + var forDebuggingOnly_auction_win_url = undefined; + const forDebuggingOnly = {}; + forDebuggingOnly.reportAdAuctionLoss = function(url){ + forDebuggingOnly_auction_loss_url = url; + } + forDebuggingOnly.reportAdAuctionWin = function(url){ + forDebuggingOnly_auction_win_url = url; + } + globalThis.forDebuggingOnly = forDebuggingOnly; + function scoreAdEntryFunction(adMetadata, bid, auctionConfig, trustedScoringSignals, browserSignals, directFromSellerSignals, featureFlags){ - var ps_logs = []; - var ps_errors = []; - var ps_warns = []; - if(featureFlags.enable_logging){ - console.log = function(...args) { - ps_logs.push(JSON.stringify(args)) - } - console.error = function(...args) { - ps_errors.push(JSON.stringify(args)) - } - console.warn = function(...args) { - ps_warns.push(JSON.stringify(args)) - } - } - - var forDebuggingOnly_auction_loss_url = undefined; - var forDebuggingOnly_auction_win_url = undefined; - const forDebuggingOnly = {}; - forDebuggingOnly.reportAdAuctionLoss = function(url){ - forDebuggingOnly_auction_loss_url = url; - } - forDebuggingOnly.reportAdAuctionWin = function(url){ - forDebuggingOnly_auction_win_url = url; - } - globalThis.forDebuggingOnly = forDebuggingOnly; - + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (featureFlags.enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } var scoreAdResponse = {}; try { scoreAdResponse = scoreAd(adMetadata, bid, auctionConfig, diff --git a/services/auction_service/score_ads_reactor.cc b/services/auction_service/score_ads_reactor.cc index 8c155c73..8671fc45 100644 --- a/services/auction_service/score_ads_reactor.cc +++ b/services/auction_service/score_ads_reactor.cc @@ -911,7 +911,7 @@ void ScoreAdsReactor::ReportingCallback( } if (PS_VLOG_IS_ON(1) && enable_adtech_code_logging_) { for (std::string& log : reporting_response.value().seller_logs) { - PS_VLOG(kNoisyInfo, log_context_) + PS_VLOG(kUdfLog, log_context_) << "Log from Seller's execution script:" << log; } for (std::string& log : reporting_response.value().seller_error_logs) { @@ -920,11 +920,11 @@ void ScoreAdsReactor::ReportingCallback( } for (std::string& log : reporting_response.value().seller_warning_logs) { - PS_LOG(ERROR, log_context_) + PS_LOG(WARNING, log_context_) << "Warning Log from Seller's execution script:" << log; } for (std::string& log : reporting_response.value().buyer_logs) { - PS_VLOG(kNoisyInfo, log_context_) + PS_VLOG(kUdfLog, log_context_) << "Log from Buyer's execution script:" << log; } for (std::string& log : reporting_response.value().buyer_error_logs) { @@ -932,7 +932,7 @@ void ScoreAdsReactor::ReportingCallback( << "Error Log from Buyer's execution script:" << log; } for (std::string& log : reporting_response.value().buyer_warning_logs) { - PS_LOG(ERROR, log_context_) + PS_LOG(WARNING, log_context_) << "Warning Log from Buyer's execution script:" << log; } } diff --git a/services/bidding_service/base_generate_bids_reactor.h b/services/bidding_service/base_generate_bids_reactor.h index 529841e2..e9ab11cc 100644 --- a/services/bidding_service/base_generate_bids_reactor.h +++ b/services/bidding_service/base_generate_bids_reactor.h @@ -17,10 +17,12 @@ #ifndef SERVICES_BIDDING_SERVICE_BASE_GENERATE_BIDS_REACTOR_H_ #define SERVICES_BIDDING_SERVICE_BASE_GENERATE_BIDS_REACTOR_H_ +#include #include #include #include "services/bidding_service/data/runtime_config.h" +#include "services/common/clients/code_dispatcher/request_context.h" #include "services/common/code_dispatch/code_dispatch_reactor.h" #include "services/common/util/request_response_constants.h" #include "src/logger/request_context_impl.h" @@ -54,6 +56,9 @@ class BaseGenerateBidsReactor enable_buyer_debug_url_generation_( runtime_config.enable_buyer_debug_url_generation), roma_timeout_ms_(runtime_config.roma_timeout_ms), + roma_request_context_factory_( + GetLoggingContext(this->raw_request_), + this->raw_request_.consented_debug_config()), enable_adtech_code_logging_(runtime_config.enable_adtech_code_logging), log_context_( GetLoggingContext(this->raw_request_), @@ -84,6 +89,7 @@ class BaseGenerateBidsReactor bool enable_buyer_debug_url_generation_; std::string roma_timeout_ms_; + RomaRequestContextFactory roma_request_context_factory_; bool enable_adtech_code_logging_; server_common::log::ContextImpl log_context_; int max_allowed_size_debug_url_chars_; diff --git a/services/bidding_service/bidding_main.cc b/services/bidding_service/bidding_main.cc index f1ffd4d3..fd974e75 100644 --- a/services/bidding_service/bidding_main.cc +++ b/services/bidding_service/bidding_main.cc @@ -246,7 +246,8 @@ absl::Status RunServer() { if (enable_inference) { PS_LOG(INFO) << "Register runInference API."; auto run_inference_function_object = - std::make_unique>(); + std::make_unique>(); run_inference_function_object->function_name = std::string(inference::kInferenceFunctionName); run_inference_function_object->function = inference::RunInference; diff --git a/services/bidding_service/generate_bids_reactor.cc b/services/bidding_service/generate_bids_reactor.cc index 371bcd0b..fb0813f0 100644 --- a/services/bidding_service/generate_bids_reactor.cc +++ b/services/bidding_service/generate_bids_reactor.cc @@ -362,7 +362,7 @@ absl::StatusOr BuildGenerateBidRequest( } generate_bid_request.input[ArgIndex(GenerateBidArgs::kInterestGroup)] = std::make_shared(std::move(serialized_ig.value())); - PS_VLOG(3, log_context) + PS_VLOG(kStats, log_context) << "\nInterest Group Serialize Time: " << ToInt64Microseconds((absl::Now() - start_parse_time)) << " microseconds for " @@ -475,13 +475,14 @@ void GenerateBidsReactor::Execute() { log_context_, enable_adtech_code_logging_, protected_auction_generate_bid_version_); if (!generate_bid_request.ok()) { - PS_VLOG(3, log_context_) + PS_VLOG(kNoisyWarn, log_context_) << "Unable to build GenerateBidRequest: " << generate_bid_request.status().ToString( absl::StatusToStringMode::kWithEverything); } else { auto dispatch_request = generate_bid_request.value(); dispatch_request.tags[kTimeoutMs] = roma_timeout_ms_; + dispatch_request.metadata = roma_request_context_factory_.Create(); dispatch_requests_.push_back(dispatch_request); } } diff --git a/services/bidding_service/inference/BUILD b/services/bidding_service/inference/BUILD index 1a6c21c5..f746a709 100644 --- a/services/bidding_service/inference/BUILD +++ b/services/bidding_service/inference/BUILD @@ -26,6 +26,7 @@ cc_library( deps = [ ":inference_flags", "//services/common/blob_fetch:blob_fetcher", + "//services/common/clients/code_dispatcher:request_context", "//services/common/util:request_response_constants", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/base:core_headers", diff --git a/services/bidding_service/inference/inference_flags.cc b/services/bidding_service/inference/inference_flags.cc index b0f47366..ab65f6e5 100644 --- a/services/bidding_service/inference/inference_flags.cc +++ b/services/bidding_service/inference/inference_flags.cc @@ -38,6 +38,7 @@ ABSL_FLAG(std::optional, inference_model_bucket_paths, // "num_interop_threads": , // "num_intraop_threads": , // "module_name": , +// "cpuset": // ... // } // where each property corresponds to a field of the diff --git a/services/bidding_service/inference/inference_utils.cc b/services/bidding_service/inference/inference_utils.cc index 3f3affc4..54341fbe 100644 --- a/services/bidding_service/inference/inference_utils.cc +++ b/services/bidding_service/inference/inference_utils.cc @@ -34,6 +34,7 @@ #include "absl/synchronization/mutex.h" #include "proto/inference_sidecar.grpc.pb.h" #include "services/bidding_service/inference/inference_flags.h" +#include "services/common/clients/code_dispatcher/request_context.h" #include "services/common/util/request_response_constants.h" #include "src/logger/request_context_logger.h" #include "src/roma/interface/roma.h" @@ -130,7 +131,9 @@ absl::Status RegisterModelsFromBucket( return absl::OkStatus(); } -void RunInference(google::scp::roma::FunctionBindingPayload<>& wrapper) { +void RunInference( + google::scp::roma::FunctionBindingPayload& + wrapper) { const std::string& payload = wrapper.io_proto.input_string(); SandboxExecutor& executor = Executor(); diff --git a/services/bidding_service/inference/inference_utils.h b/services/bidding_service/inference/inference_utils.h index a6e69576..3d9a7050 100644 --- a/services/bidding_service/inference/inference_utils.h +++ b/services/bidding_service/inference/inference_utils.h @@ -15,6 +15,7 @@ #ifndef SERVICES_BIDDING_SERVICE_INFERENCE_INFERENCE_UTILS_H_ #define SERVICES_BIDDING_SERVICE_INFERENCE_INFERENCE_UTILS_H_ +#include #include #include @@ -22,6 +23,7 @@ #include "proto/inference_sidecar.pb.h" #include "sandbox/sandbox_executor.h" #include "services/common/blob_fetch/blob_fetcher.h" +#include "services/common/clients/code_dispatcher/request_context.h" #include "src/roma/interface/roma.h" namespace privacy_sandbox::bidding_auction_servers::inference { @@ -42,7 +44,9 @@ absl::Status RegisterModelsFromBucket( // single inference request to the inference sidecar. // // wrapper: Inference request backed by JS string. -void RunInference(google::scp::roma::FunctionBindingPayload<>& wrapper); +void RunInference( + google::scp::roma::FunctionBindingPayload& + wrapper); } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/bidding_service/inference/inference_utils_test.cc b/services/bidding_service/inference/inference_utils_test.cc index 1dc269bc..467c0945 100644 --- a/services/bidding_service/inference/inference_utils_test.cc +++ b/services/bidding_service/inference/inference_utils_test.cc @@ -31,8 +31,12 @@ constexpr absl::string_view kInit = "non-empty"; constexpr absl::string_view kTestModelPath = "external/inference_common/testdata/models/tensorflow_1_mib_saved_model.pb"; constexpr absl::string_view kBucketName = "test_bucket"; -constexpr absl::string_view kRuntimeConfig = - R"json({"num_interop_threads": 4, "num_intraop_threads": 5, "module_name": "test"})json"; +constexpr absl::string_view kRuntimeConfig = R"json({ + "num_interop_threads": 4, + "num_intraop_threads": 5, + "module_name": "test", + "cpuset": [0, 1] +})json"; class InferenceUtilsTest : public ::testing::Test { protected: @@ -56,7 +60,8 @@ TEST_F(InferenceUtilsTest, ReturnValueIsSet) { ASSERT_TRUE(RegisterModelsFromLocal({std::string(kTestModelPath)}).ok()); google::scp::roma::proto::FunctionBindingIoProto input_output_proto; - google::scp::roma::FunctionBindingPayload<> wrapper{input_output_proto, {}}; + google::scp::roma::FunctionBindingPayload wrapper{ + input_output_proto, {}}; wrapper.io_proto.set_input_string(absl::StrCat("1.0")); wrapper.io_proto.set_output_string(kInit); RunInference(wrapper); diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor.cc b/services/bidding_service/protected_app_signals_generate_bids_reactor.cc index a93eb8c3..edf01550 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor.cc +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor.cc @@ -221,6 +221,7 @@ ProtectedAppSignalsGenerateBidsReactor::CreateGenerateBidsRequest( .version_string = protected_app_signals_generate_bid_version_, .handler_name = kDispatchHandlerFunctionNameWithCodeWrapper, .input = std::move(input), + .metadata = roma_request_context_factory_.Create(), }; request.tags[kTimeoutMs] = roma_timeout_ms_; return request; @@ -267,7 +268,7 @@ void ProtectedAppSignalsGenerateBidsReactor::OnFetchAdsDataDone( << "Skipping protected app signals bid (" << GetBidDebugInfo(bid) << ")"; } else { - PS_VLOG(3, log_context_) + PS_VLOG(kNoisyInfo, log_context_) << "Successful non-zero protected app signals bid received"; auto* added_bid = raw_response_.add_bids(); *added_bid = bid; @@ -312,10 +313,11 @@ DispatchRequest ProtectedAppSignalsGenerateBidsReactor:: .version_string = ad_retrieval_version_, .handler_name = kPrepareDataForAdRetrievalEntryFunctionName, .input = std::move(input), + .metadata = roma_request_context_factory_.Create(), }; if (server_common::log::PS_VLOG_IS_ON(3)) { for (const auto& i : request.input) { - PS_VLOG(3, log_context_) + PS_VLOG(kDispatch, log_context_) << "Roma request input to prepared data for ads retrieval: " << *i; } } diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor.h b/services/bidding_service/protected_app_signals_generate_bids_reactor.h index 559afe56..bd373d9a 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor.h +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor.h @@ -120,8 +120,9 @@ class ProtectedAppSignalsGenerateBidsReactor return; } - PS_VLOG(3, log_context_) << "Response from " << roma_entry_function - << ": " << result[0]->resp; + PS_VLOG(kDispatch, log_context_) + << "Response from " << roma_entry_function << ": " + << result[0]->resp; auto parsed_response = std::move(parse_response)(result[0]->resp); if (!parsed_response.ok()) { PS_VLOG(kNoisyWarn, log_context_) @@ -133,7 +134,7 @@ class ProtectedAppSignalsGenerateBidsReactor return; } - PS_VLOG(3, log_context_) + PS_VLOG(kDispatch, log_context_) << "Successful V8 Response from: " << roma_entry_function; std::move(on_successful_response)(*std::move(parsed_response)); diff --git a/services/buyer_frontend_service/get_bids_unary_reactor.cc b/services/buyer_frontend_service/get_bids_unary_reactor.cc index d10deb62..f594abfa 100644 --- a/services/buyer_frontend_service/get_bids_unary_reactor.cc +++ b/services/buyer_frontend_service/get_bids_unary_reactor.cc @@ -168,7 +168,7 @@ void GetBidsUnaryReactor::OnAllBidsDone(bool any_successful_bids) { << get_bids_response_->ShortDebugString(); if (!any_successful_bids) { - PS_VLOG(3, log_context_) + PS_LOG(WARNING, log_context_) << "Finishing the GetBids RPC with an error, since there are " "no successful bids returned by the bidding service"; benchmarking_logger_->End(); @@ -244,8 +244,8 @@ void GetBidsUnaryReactor::Execute() { if (num_bidding_calls == 0) { // This is unlikely to happen since we already have this check in place // in SFE. - PS_VLOG(3, log_context_) << "No protected audience or protected app " - "signals input found in the request"; + PS_LOG(ERROR, log_context_) << "No protected audience or protected app " + "signals input found in the request"; benchmarking_logger_->End(); FinishWithStatus(grpc::Status(grpc::INVALID_ARGUMENT, kMissingInputs)); return; @@ -279,7 +279,7 @@ void GetBidsUnaryReactor::MayGetProtectedSignalsBids() { if (!raw_request_.has_protected_app_signals_buyer_input() || !raw_request_.protected_app_signals_buyer_input() .has_protected_app_signals()) { - PS_VLOG(3, log_context_) + PS_VLOG(kNoisyWarn, log_context_) << "No protected app buyer signals input found, skipping fetching bids " "for protected app signals"; return; @@ -350,13 +350,13 @@ void GetBidsUnaryReactor::MayGetProtectedSignalsBids() { void GetBidsUnaryReactor::MayGetProtectedAudienceBids() { if (!config_.is_protected_audience_enabled) { - PS_VLOG(3, log_context_) + PS_VLOG(kNoisyWarn, log_context_) << "Protected Audience is not enabled, skipping bids fetching for PA"; return; } if (raw_request_.buyer_input().interest_groups().empty()) { - PS_VLOG(3, log_context_) + PS_VLOG(kNoisyWarn, log_context_) << "No interest groups found, skipping bidding for protected audience"; return; } diff --git a/services/common/clients/code_dispatcher/BUILD b/services/common/clients/code_dispatcher/BUILD index 244bdcbd..087bf1a9 100644 --- a/services/common/clients/code_dispatcher/BUILD +++ b/services/common/clients/code_dispatcher/BUILD @@ -21,6 +21,7 @@ cc_library( srcs = ["v8_dispatcher.cc"], hdrs = ["v8_dispatcher.h"], deps = [ + ":request_context", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/status", "@com_google_absl//absl/synchronization", @@ -68,3 +69,16 @@ cc_test( "@com_google_googletest//:gtest_main", ], ) + +cc_library( + name = "request_context", + srcs = [ + "request_context.cc", + ], + hdrs = [ + "request_context.h", + ], + deps = [ + "@google_privacysandbox_servers_common//src/logger:request_context_impl", + ], +) diff --git a/services/common/clients/code_dispatcher/request_context.cc b/services/common/clients/code_dispatcher/request_context.cc new file mode 100644 index 00000000..5968c258 --- /dev/null +++ b/services/common/clients/code_dispatcher/request_context.cc @@ -0,0 +1,57 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/common/clients/code_dispatcher/request_context.h" + +namespace privacy_sandbox::bidding_auction_servers { + +RomaRequestContext::RomaRequestContext( + const absl::btree_map& context_map, + const privacy_sandbox::server_common::ConsentedDebugConfiguration& + debug_config) + : request_logging_context_(context_map, debug_config) {} + +const privacy_sandbox::server_common::log::ContextImpl& +RomaRequestContext::GetLogContext() const { + return request_logging_context_; +} + +RomaRequestSharedContext::RomaRequestSharedContext( + const std::shared_ptr& roma_request_context) + : roma_request_context_(roma_request_context) {} + +absl::StatusOr> +RomaRequestSharedContext::GetRomaRequestContext() const { + std::shared_ptr shared_context = + roma_request_context_.lock(); + if (!shared_context) { + return absl::UnavailableError("RomaRequestContext is not available"); + } + + return shared_context; +} + +RomaRequestContextFactory::RomaRequestContextFactory( + const absl::btree_map& context_map, + const privacy_sandbox::server_common::ConsentedDebugConfiguration& + debug_config) { + roma_request_context_ = + std::make_shared(context_map, debug_config); +} + +RomaRequestSharedContext RomaRequestContextFactory::Create() { + return RomaRequestSharedContext(roma_request_context_); +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/clients/code_dispatcher/request_context.h b/services/common/clients/code_dispatcher/request_context.h new file mode 100644 index 00000000..708b9a86 --- /dev/null +++ b/services/common/clients/code_dispatcher/request_context.h @@ -0,0 +1,88 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_COMMON_CLIENTS_CODE_DISPATCHER_REQUEST_CONTEXT_H_ +#define SERVICES_COMMON_CLIENTS_CODE_DISPATCHER_REQUEST_CONTEXT_H_ + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "src/logger/request_context_impl.h" + +namespace privacy_sandbox::bidding_auction_servers { + +// RomaRequestContext holds B&A server level request context, which can be used +// for logging and metrics in roma callbacks. +class RomaRequestContext { + public: + RomaRequestContext( + const absl::btree_map& context_map, + const privacy_sandbox::server_common::ConsentedDebugConfiguration& + debug_config); + + const privacy_sandbox::server_common::log::ContextImpl& GetLogContext() const; + + private: + privacy_sandbox::server_common::log::ContextImpl request_logging_context_; +}; + +class RomaRequestContextFactory; + +// Shared RomaRequestContext that can be used as a part of the Roma worker +// dispatch request metadata. +class RomaRequestSharedContext { + public: + RomaRequestSharedContext() {} + + absl::StatusOr> GetRomaRequestContext() + const; + + friend class RomaRequestContextFactory; + + private: + explicit RomaRequestSharedContext( + const std::shared_ptr& roma_request_context); + std::weak_ptr roma_request_context_; +}; + +// RomaRequestContextFactory holds a RomaRequestContext. Shared copies of this +// context are wrapped by RomaRequestSharedContext and then can be passed to +// Roma workers. +class RomaRequestContextFactory { + public: + RomaRequestContextFactory( + const absl::btree_map& context_map, + const privacy_sandbox::server_common::ConsentedDebugConfiguration& + debug_config); + RomaRequestSharedContext Create(); + + RomaRequestContextFactory(RomaRequestContextFactory&& other) = delete; + RomaRequestContextFactory& operator=(RomaRequestContextFactory&& other) = + delete; + RomaRequestContextFactory(const RomaRequestContextFactory&) = delete; + RomaRequestContextFactory& operator=(const RomaRequestContextFactory&) = + delete; + + private: + std::shared_ptr roma_request_context_; +}; + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_COMMON_CLIENTS_CODE_DISPATCHER_REQUEST_CONTEXT_H_ diff --git a/services/common/clients/code_dispatcher/v8_dispatcher.h b/services/common/clients/code_dispatcher/v8_dispatcher.h index eb66f8bf..eb9676c9 100644 --- a/services/common/clients/code_dispatcher/v8_dispatcher.h +++ b/services/common/clients/code_dispatcher/v8_dispatcher.h @@ -20,24 +20,27 @@ #include #include "absl/status/status.h" +#include "services/common/clients/code_dispatcher/request_context.h" #include "src/roma/interface/roma.h" #include "src/roma/roma_service/roma_service.h" namespace privacy_sandbox::bidding_auction_servers { -// The following aliases are part of the exported API so the +// The following aliases are part of the exported API so the // user does not have to explicitly import the underlying library. -using DispatchRequest = google::scp::roma::InvocationSharedRequest<>; +using DispatchRequest = + google::scp::roma::InvocationSharedRequest; using DispatchResponse = google::scp::roma::ResponseObject; using DispatchDoneCallback = google::scp::roma::Callback; -using DispatchService = google::scp::roma::sandbox::roma_service::RomaService<>; +using DispatchService = google::scp::roma::sandbox::roma_service::RomaService< + RomaRequestSharedContext>; using BatchDispatchDoneCallback = google::scp::roma::BatchCallback; // The DispatchConfig controls the number of worker processes, the number of // threads per worker process, the IPC shared memory size (in bytes), and // the max amount of tasks in the work queue before requests are rejected. // Default values of {0, 0, 0, 0} allow the underlying library to choose // these values as necessary. -using DispatchConfig = google::scp::roma::Config<>; +using DispatchConfig = DispatchService::Config; // This class is a wrapper around Roma, a library which provides an interface // for multi-process javascript and wasm execution in V8. @@ -45,7 +48,7 @@ class V8Dispatcher { public: explicit V8Dispatcher(DispatchConfig&& config = DispatchConfig()); - ~V8Dispatcher(); + virtual ~V8Dispatcher(); // Init the dispatcher. Note that this call may bring up multiple processes, // which can be slow and should only happen on server startup. diff --git a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc index 4a46321f..0ff2b6e5 100644 --- a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc +++ b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc @@ -83,7 +83,7 @@ absl::Status BuyerKeyValueAsyncHttpClient::Execute( absl::StatusOr resultStr) mutable { if (resultStr.ok()) { PS_VLOG(kKVLog) << "BuyerKeyValueAsyncHttpClient Success Response:\n" - << resultStr.value() << "\n"; + << resultStr.value(); size_t response_size = resultStr->size(); std::unique_ptr resultUPtr = std::make_unique(GetBuyerValuesOutput( diff --git a/services/common/code_dispatch/code_dispatch_reactor.h b/services/common/code_dispatch/code_dispatch_reactor.h index 6d046788..a4cb0b1a 100644 --- a/services/common/code_dispatch/code_dispatch_reactor.h +++ b/services/common/code_dispatch/code_dispatch_reactor.h @@ -53,7 +53,7 @@ class CodeDispatchReactor : public grpc::ServerUnaryReactor { crypto_client_(crypto_client) { PS_VLOG(5) << "Encryption is enabled, decrypting request now"; if (DecryptRequest()) { - PS_VLOG(3) << "Decrypted request: " << raw_request_.DebugString(); + PS_VLOG(5) << "Decrypted request: " << raw_request_.DebugString(); } else { PS_LOG(ERROR) << "Failed to decrypt the request"; } diff --git a/services/common/encryption/BUILD b/services/common/encryption/BUILD index ff4cda19..ce28853e 100644 --- a/services/common/encryption/BUILD +++ b/services/common/encryption/BUILD @@ -48,6 +48,7 @@ cc_library( deps = [ "//services/common/clients/config:config_client", "//services/common/constants:common_service_flags", + "//services/common/util:request_response_constants", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", diff --git a/services/common/encryption/key_fetcher_factory.cc b/services/common/encryption/key_fetcher_factory.cc index 73b9a21d..351240bf 100644 --- a/services/common/encryption/key_fetcher_factory.cc +++ b/services/common/encryption/key_fetcher_factory.cc @@ -26,6 +26,7 @@ #include "services/common/clients/config/trusted_server_config_client.h" #include "services/common/constants/common_service_flags.h" +#include "services/common/util/request_response_constants.h" #include "src/concurrent/event_engine_executor.h" #include "src/core/lib/event_engine/default_event_engine.h" #include "src/encryption/key_fetcher/fake_key_fetcher_manager.h" @@ -103,7 +104,8 @@ std::unique_ptr CreateKeyFetcherManager( config_client.GetStringParameter(SECONDARY_COORDINATOR_REGION); if (config_client.HasParameter(GCP_PRIMARY_WORKLOAD_IDENTITY_POOL_PROVIDER)) { - PS_VLOG(3) << "Found GCP Workload Identity Pool Provider, proceeding..."; + PS_VLOG(kNoisyInfo) + << "Found GCP Workload Identity Pool Provider, proceeding..."; primary.gcp_private_key_vending_service_cloudfunction_url = config_client.GetStringParameter( GCP_PRIMARY_KEY_SERVICE_CLOUD_FUNCTION_URL); diff --git a/services/common/test/mocks.h b/services/common/test/mocks.h index 4dc7402b..422f9aec 100644 --- a/services/common/test/mocks.h +++ b/services/common/test/mocks.h @@ -334,7 +334,7 @@ class KVServiceMock : public kv_server::v2::KeyValueService::CallbackService { class MockV8Dispatcher : public V8Dispatcher { public: - virtual ~MockV8Dispatcher() = default; + ~MockV8Dispatcher() override = default; MOCK_METHOD(absl::Status, Init, ()); MOCK_METHOD(absl::Status, Stop, ()); MOCK_METHOD(absl::Status, LoadSync, diff --git a/services/common/util/BUILD b/services/common/util/BUILD index 04ff037f..219b5872 100644 --- a/services/common/util/BUILD +++ b/services/common/util/BUILD @@ -262,6 +262,7 @@ cc_library( srcs = ["async_task_tracker.cc"], hdrs = ["async_task_tracker.h"], deps = [ + "//services/common/util:request_response_constants", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/log:check", "@com_google_absl//absl/strings:str_format", diff --git a/services/common/util/async_task_tracker.cc b/services/common/util/async_task_tracker.cc index 28d8cb23..ba4667e7 100644 --- a/services/common/util/async_task_tracker.cc +++ b/services/common/util/async_task_tracker.cc @@ -18,6 +18,7 @@ #include "absl/log/check.h" #include "absl/strings/str_cat.h" +#include "services/common/util/request_response_constants.h" namespace privacy_sandbox::bidding_auction_servers { @@ -95,10 +96,9 @@ void AsyncTaskTracker::SetNumTasksToTrack(int num_tasks_to_track) { empty_tasks_count_ = 0; skipped_tasks_count_ = 0; error_tasks_count_ = 0; - PS_VLOG(3, log_context_) << "Reset of task tracker to track: " - << num_tasks_to_track - << " number of tasks done. New tracker: " - << ToString(); + PS_VLOG(kStats, log_context_) + << "Reset of task tracker to track: " << num_tasks_to_track + << " number of tasks done. New tracker: " << ToString(); } bool AsyncTaskTracker::AnyTaskSuccessfullyCompleted() { diff --git a/services/common/util/binary_http_utils.h b/services/common/util/binary_http_utils.h index df0f10a2..9e00e15d 100644 --- a/services/common/util/binary_http_utils.h +++ b/services/common/util/binary_http_utils.h @@ -85,8 +85,8 @@ absl::StatusOr FromBinaryHTTP( return absl::InvalidArgumentError( "Unable to convert the Binary HTTP response to proto"); } - PS_VLOG(kSuccess) << "Converted the http request to proto: " - << response_proto.DebugString(); + PS_VLOG(kNoisyInfo) << "Converted the http request to proto: " + << response_proto.DebugString(); } return response_proto; } diff --git a/services/common/util/reporting_util.cc b/services/common/util/reporting_util.cc index a08e8704..9b3d6049 100644 --- a/services/common/util/reporting_util.cc +++ b/services/common/util/reporting_util.cc @@ -45,7 +45,7 @@ void MayVlogAdTechCodeLogs(const rapidjson::Document& document, auto logs_it = document.FindMember(log_type.c_str()); if (logs_it != document.MemberEnd()) { for (const auto& log : logs_it->value.GetArray()) { - PS_VLOG(kNoisyInfo, log_context) << log_type << ": " << log.GetString(); + PS_VLOG(kUdfLog, log_context) << log_type << ": " << log.GetString(); } } } diff --git a/services/common/util/request_response_constants.h b/services/common/util/request_response_constants.h index edd168b9..3843b27a 100644 --- a/services/common/util/request_response_constants.h +++ b/services/common/util/request_response_constants.h @@ -142,14 +142,15 @@ enum class AuctionType : int { kProtectedAudience, kProtectedAppSignals }; inline constexpr int kPlain = 1; // plaintext B&A request and response served inline constexpr int kNoisyWarn = 2; // non-critical error, use PS_LOG(ERROR, *) for critical error -inline constexpr int kOriginated = - 3; // plaintext B&A request and response originated from server -inline constexpr int kKVLog = 3; // KV request response -inline constexpr int kDispatch = 4; // UDF dispatch request and response +inline constexpr int kUdfLog = 3; inline constexpr int kSuccess = 3; -inline constexpr int kNoisyInfo = 3; -inline constexpr int kEncrypted = 4; +inline constexpr int kNoisyInfo = 4; +inline constexpr int kDispatch = 4; // UDF dispatch request and response +inline constexpr int kOriginated = + 5; // plaintext B&A request and response originated from server +inline constexpr int kKVLog = 5; // KV request response inline constexpr int kStats = 5; // Stats log like time , byte size, etc. +inline constexpr int kEncrypted = 6; } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_COMMON_UTIL_REQUEST_RESPONSE_CONSTANTS_H_ diff --git a/services/inference_sidecar/common/BUILD b/services/inference_sidecar/common/BUILD index f725970e..95c8911e 100644 --- a/services/inference_sidecar/common/BUILD +++ b/services/inference_sidecar/common/BUILD @@ -47,6 +47,7 @@ cc_library( "//proto:inference_sidecar_cc_grpc_proto", "//proto:inference_sidecar_cc_proto", "//sandbox:sandbox_worker", + "//utils:cpu", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/log", "@com_google_absl//absl/log:absl_log", diff --git a/services/inference_sidecar/common/grpc_sidecar.cc b/services/inference_sidecar/common/grpc_sidecar.cc index ccf9cd27..083f600a 100644 --- a/services/inference_sidecar/common/grpc_sidecar.cc +++ b/services/inference_sidecar/common/grpc_sidecar.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -36,6 +37,7 @@ #include "sandbox/sandbox_worker.h" #include "sandboxed_api/sandbox2/comms.h" #include "src/util/status_macro/status_util.h" +#include "utils/cpu.h" namespace privacy_sandbox::bidding_auction_servers::inference { namespace { @@ -76,6 +78,14 @@ class InferenceServiceImpl final : public InferenceService::Service { } // namespace +absl::Status SetCpuAffinity(const InferenceSidecarRuntimeConfig& config) { + if (config.cpuset().empty()) { + return absl::OkStatus(); + } + std::vector cpuset(config.cpuset().begin(), config.cpuset().end()); + return SetCpuAffinity(cpuset); +} + absl::Status Run(const InferenceSidecarRuntimeConfig& config) { SandboxWorker worker; grpc::ServerBuilder builder; diff --git a/services/inference_sidecar/common/grpc_sidecar.h b/services/inference_sidecar/common/grpc_sidecar.h index 2629d584..af9faee4 100644 --- a/services/inference_sidecar/common/grpc_sidecar.h +++ b/services/inference_sidecar/common/grpc_sidecar.h @@ -20,6 +20,8 @@ namespace privacy_sandbox::bidding_auction_servers::inference { +absl::Status SetCpuAffinity(const InferenceSidecarRuntimeConfig& config); + // Runs a simple gRPC server. It is thread safe. absl::Status Run(const InferenceSidecarRuntimeConfig& config); diff --git a/services/inference_sidecar/common/inference_sidecar_main.cc b/services/inference_sidecar/common/inference_sidecar_main.cc index c3b0985a..d193bd36 100644 --- a/services/inference_sidecar/common/inference_sidecar_main.cc +++ b/services/inference_sidecar/common/inference_sidecar_main.cc @@ -29,6 +29,10 @@ int main(int argc, char** argv) { CHECK(google::protobuf::util::JsonStringToMessage(argv[0], &config).ok()) << "Could not parse inference sidecar runtime config JsonString to a " "proto message."; + CHECK(privacy_sandbox::bidding_auction_servers::inference::SetCpuAffinity( + config) + .ok()) + << "Could not set CPU affinity."; if (absl::Status run_status = privacy_sandbox::bidding_auction_servers::inference::Run(config); !run_status.ok()) { diff --git a/services/inference_sidecar/common/proto/inference_sidecar.proto b/services/inference_sidecar/common/proto/inference_sidecar.proto index 465e09e1..823b15a4 100644 --- a/services/inference_sidecar/common/proto/inference_sidecar.proto +++ b/services/inference_sidecar/common/proto/inference_sidecar.proto @@ -28,12 +28,16 @@ service InferenceService { message PredictRequest { // Input data. bytes input = 1; + // Should consented logs be collected for the given predict request. + bool is_consented = 2; } // Response for PredictRequest on a successful run. message PredictResponse { // Output data. bytes output = 1; + // Consented debugging log. + InferenceDebugInfo debug_info = 2; } message ModelSpec { @@ -68,4 +72,15 @@ message InferenceSidecarRuntimeConfig { // Specifies the inference backend module required. // Currently supports "test", "tensorflow_v2_14_0", "pytorch_v2_1_1" string module_name = 3; + + // Specifies a list of CPU IDs to set the CPU affinity for the sidecar. + // The sidecar process will run only on the specified CPUs. + // If `cpuset` is empty, the CPU affinity is not used. + repeated int32 cpuset = 4; +} + +// Proto to store consented debugging logs. It's passed back with +// PredictResponse to the caller of the Predict RPC. +message InferenceDebugInfo { + repeated string logs = 1; } diff --git a/services/inference_sidecar/common/test_constants.h b/services/inference_sidecar/common/test_constants.h index 81b0cede..23a462fa 100644 --- a/services/inference_sidecar/common/test_constants.h +++ b/services/inference_sidecar/common/test_constants.h @@ -21,8 +21,12 @@ namespace privacy_sandbox::bidding_auction_servers::inference { -constexpr absl::string_view kRuntimeConfig = - R"json({"num_interop_threads": 4, "num_intraop_threads": 5, "module_name": "test"})json"; +constexpr absl::string_view kRuntimeConfig = R"json({ + "num_interop_threads": 4, + "num_intraop_threads": 5, + "module_name": "test", + "cpuset": [0, 1] +})json"; } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/utils/BUILD b/services/inference_sidecar/common/utils/BUILD index 48b48918..d05277b3 100644 --- a/services/inference_sidecar/common/utils/BUILD +++ b/services/inference_sidecar/common/utils/BUILD @@ -92,3 +92,24 @@ cc_test( "@com_google_googletest//:gtest_main", ], ) + +cc_library( + name = "log", + hdrs = ["log.h"], + deps = [ + "//proto:inference_sidecar_cc_proto", + "@com_google_absl//absl/log", + "@com_google_absl//absl/synchronization", + ], +) + +cc_test( + name = "log_test", + size = "small", + srcs = ["log_test.cc"], + deps = [ + ":log", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/services/inference_sidecar/common/utils/log.h b/services/inference_sidecar/common/utils/log.h new file mode 100644 index 00000000..695ed836 --- /dev/null +++ b/services/inference_sidecar/common/utils/log.h @@ -0,0 +1,83 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_INFERENCE_SIDECAR_COMMON_UTILS_LOG_H_ +#define SERVICES_INFERENCE_SIDECAR_COMMON_UTILS_LOG_H_ + +#include +#include +#include + +#include "absl/log/log_sink.h" +#include "absl/synchronization/mutex.h" +#include "proto/inference_sidecar.pb.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { + +// The INFERENCE_LOG marcro is used to log to an InferenceDebugInfo proto. +#define INFERENCE_LOG(level, log_context) \ + ABSL_LOG(level).ToSinkOnly(log_context.LogSink()) + +// This class wraps a consented debugging log sink which redirect logs to an +// InferenceDebugInfo proto. When ABSL_LOG is invoked on the log sink, logging +// can be redirected to it based on user consent. We currently return the +// InferenceDebugInfo proto with a PredictResponse to the caller of +// InferenceService +class LogContext { + public: + LogContext(absl::AnyInvocable debug_info, + bool is_consented) + : log_sink_(std::make_unique(std::move(debug_info), + is_consented)) {} + + LogContext() : log_sink_(std::make_unique()) {} + + absl::LogSink* LogSink() const { return log_sink_.get(); } + + private: + class DebugResponseSinkImpl : public absl::LogSink { + public: + DebugResponseSinkImpl(absl::AnyInvocable debug_info, + bool is_consented) + : debug_info_(std::move(debug_info)), is_consented_(is_consented) {} + + void Send(const absl::LogEntry& entry) override + ABSL_LOCKS_EXCLUDED(mutex_) { + if (is_consented_) { + absl::MutexLock lock(&mutex_); + // Resolves Tensorflow proto version compatibility. + debug_info_()->add_logs(std::string(entry.text_message_with_prefix())); + } + } + + void Flush() override {} + + private: + absl::AnyInvocable debug_info_; + const bool is_consented_; + absl::Mutex mutex_; + }; + + class NoOpSinkImpl : public absl::LogSink { + public: + void Send(const absl::LogEntry& entry) override {} + + void Flush() override {} + }; + std::unique_ptr log_sink_; +}; + +} // namespace privacy_sandbox::bidding_auction_servers::inference + +#endif // SERVICES_INFERENCE_SIDECAR_COMMON_UTILS_LOG_H_ diff --git a/services/inference_sidecar/common/utils/log_test.cc b/services/inference_sidecar/common/utils/log_test.cc new file mode 100644 index 00000000..039e8db5 --- /dev/null +++ b/services/inference_sidecar/common/utils/log_test.cc @@ -0,0 +1,64 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "utils/log.h" + +#include +#include + +#include "absl/log/absl_log.h" +#include "proto/inference_sidecar.pb.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { +namespace { + +using ::testing::HasSubstr; + +constexpr absl::string_view kLogContent = "log content"; + +TEST(LogTest, LogIfConsented) { + PredictResponse predict_response; + LogContext log_context( + [&predict_response]() { return predict_response.mutable_debug_info(); }, + true); + + INFERENCE_LOG(INFO, log_context) << kLogContent; + ASSERT_EQ(predict_response.debug_info().logs_size(), 1); + EXPECT_THAT(predict_response.debug_info().logs(0), HasSubstr(kLogContent)); +} + +TEST(LogTest, LogNothingIfNoConsent) { + PredictResponse predict_response; + LogContext log_context( + [&predict_response]() { return predict_response.mutable_debug_info(); }, + false); + + INFERENCE_LOG(INFO, log_context) << kLogContent; + EXPECT_EQ(predict_response.debug_info().logs_size(), 0); +} + +TEST(LogTest, CanLogMultipleTimes) { + PredictResponse predict_response; + LogContext log_context( + [&predict_response]() { return predict_response.mutable_debug_info(); }, + true); + + for (int i = 0; i < 10; ++i) { + INFERENCE_LOG(INFO, log_context) << kLogContent; + } + EXPECT_EQ(predict_response.debug_info().logs_size(), 10); +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/seller_frontend_service/select_ad_reactor.cc b/services/seller_frontend_service/select_ad_reactor.cc index ade3435d..6233aa14 100644 --- a/services/seller_frontend_service/select_ad_reactor.cc +++ b/services/seller_frontend_service/select_ad_reactor.cc @@ -404,7 +404,7 @@ void SelectAdReactor::Execute() { OnScoreAdsDone(std::make_unique()); return; } - PS_VLOG(3, log_context_) << "No client / Adtech server errors found"; + PS_VLOG(kNoisyInfo, log_context_) << "No client / Adtech server errors found"; benchmarking_logger_->Begin(); @@ -501,8 +501,9 @@ SelectAdReactor::CreateGetBidsRequest(const std::string& buyer_ig_owner, if (!is_protected_audience_enabled_ && get_bids_request->mutable_buyer_input()->interest_groups_size() > 0) { - PS_VLOG(3) << "Clearing interest groups in the input since protected " - "audience support is disabled"; + PS_VLOG(kNoisyWarn) + << "Clearing interest groups in the input since protected " + "audience support is disabled"; get_bids_request->mutable_buyer_input()->clear_interest_groups(); } return get_bids_request; @@ -646,7 +647,7 @@ void SelectAdReactor::OnAllBidsDone(bool any_successful_bids) { LogIfError( metric_context_->AccumulateMetric( 1, metric::kSfeSelectAdNoSuccessfulBid)); - PS_VLOG(3, log_context_) + PS_LOG(WARNING, log_context_) << "Finishing the SelectAdRequest RPC with an error"; FinishWithStatus(grpc::Status(grpc::INTERNAL, kInternalServerError)); @@ -899,7 +900,7 @@ void SelectAdReactor::ScoreAds() { OnScoreAdsDone(std::make_unique()); return; } - PS_VLOG(kOriginated, log_context_) << "\nScoreAdsRawRequest:\n" + PS_VLOG(kOriginated, log_context_) << "ScoreAdsRawRequest:\n" << raw_request->DebugString(); auto auction_request = @@ -951,7 +952,7 @@ void SelectAdReactor::OnScoreAdsDone( response) { std::optional high_score; if (HaveAdServerVisibleErrors()) { - PS_VLOG(3, log_context_) + PS_LOG(WARNING, log_context_) << "Finishing the SelectAdRequest RPC with ad server visible error"; FinishWithStatus(grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, diff --git a/services/seller_frontend_service/select_ad_reactor_app.cc b/services/seller_frontend_service/select_ad_reactor_app.cc index 525cbee3..394f2abf 100644 --- a/services/seller_frontend_service/select_ad_reactor_app.cc +++ b/services/seller_frontend_service/select_ad_reactor_app.cc @@ -180,7 +180,7 @@ void SelectAdReactorForApp::MayPopulateProtectedAppSignalsBuyerInput( return; } - PS_VLOG(3, log_context_) + PS_VLOG(kNoisyInfo, log_context_) << "Found protected signals in buyer input, passing them to get bids"; auto* protected_app_signals_buyer_input = get_bids_raw_request->mutable_protected_app_signals_buyer_input(); @@ -193,7 +193,7 @@ void SelectAdReactorForApp::MayPopulateProtectedAppSignalsBuyerInput( auto& per_buyer_config = request_->auction_config().per_buyer_config(); auto buyer_config_it = per_buyer_config.find(buyer); if (buyer_config_it == per_buyer_config.end()) { - PS_VLOG(3, log_context_) << "No buyer config found for: " << buyer; + PS_VLOG(kNoisyInfo, log_context_) << "No buyer config found for: " << buyer; return; } @@ -201,9 +201,10 @@ void SelectAdReactorForApp::MayPopulateProtectedAppSignalsBuyerInput( if (!buyer_config.has_contextual_protected_app_signals_data() || buyer_config.contextual_protected_app_signals_data() .ad_render_ids_size() == 0) { - PS_VLOG(3, log_context_) << "No PAS ad render ids received via contextual " - "path for buyer: " - << buyer; + PS_VLOG(kNoisyInfo, log_context_) + << "No PAS ad render ids received via contextual " + "path for buyer: " + << buyer; return; } @@ -255,8 +256,9 @@ void SelectAdReactorForApp::MayPopulateProtectedAppSignalsBids( return; } - PS_VLOG(3, log_context_) << "Protected App signals, may add protected app " - "signals bids to score ads request"; + PS_VLOG(kNoisyInfo, log_context_) + << "Protected App signals, may add protected app " + "signals bids to score ads request"; for (const auto& [buyer_owner, get_bid_response] : shared_buyer_bids_map_) { for (int i = 0; i < get_bid_response->protected_app_signals_bids_size(); i++) { diff --git a/services/seller_frontend_service/util/encryption_util.cc b/services/seller_frontend_service/util/encryption_util.cc index 236f912e..aa78a421 100644 --- a/services/seller_frontend_service/util/encryption_util.cc +++ b/services/seller_frontend_service/util/encryption_util.cc @@ -64,8 +64,9 @@ DecryptOHTTPEncapsulatedHpkeCiphertext( absl::StrFormat(kMissingPrivateKey, str_key_id)); } - PS_VLOG(3) << "Private Key Id: " << private_key->key_id << ", Key Hex: " - << absl::BytesToHexString(private_key->private_key); + PS_VLOG(kSuccess) << "Private Key Id: " << private_key->key_id + << ", Key Hex: " + << absl::BytesToHexString(private_key->private_key); // Decrypt the ciphertext. absl::StatusOr ohttp_request = server_common::DecryptEncapsulatedRequest(*private_key, diff --git a/services/seller_frontend_service/util/web_utils.cc b/services/seller_frontend_service/util/web_utils.cc index 706618e1..ff7f78c6 100644 --- a/services/seller_frontend_service/util/web_utils.cc +++ b/services/seller_frontend_service/util/web_utils.cc @@ -751,8 +751,8 @@ bool IsTypeValid(absl::AnyInvocable is_valid_type, absl::string_view actual_type = kCborDataTypesLookup[item->type]; std::string error = absl::StrFormat(kInvalidTypeError, field_name, expected_type, actual_type); - PS_VLOG(3) << "CBOR type validation failure at: " << location.file_name() - << ":" << location.line(); + PS_VLOG(kNoisyWarn) << "CBOR type validation failure at: " + << location.file_name() << ":" << location.line(); error_accumulator.ReportError(ErrorVisibility::CLIENT_VISIBLE, error, ErrorCode::CLIENT_SIDE); return false; diff --git a/version.txt b/version.txt index 240bba90..0be1fc7d 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.7.0 \ No newline at end of file +3.8.0 \ No newline at end of file