From 332f7e143c7aabb995ffb616cfc248a3efeb0292 Mon Sep 17 00:00:00 2001 From: Salman Date: Tue, 2 Jul 2024 23:19:43 +0000 Subject: [PATCH] Release 3.9.0 (2024-06-25) ### Features * [IsolateBuyerAndSellerCodeExecution] Add wrapper for buyer's reportWin() udf * [reporting] Enable reportResult execution for topLevelSeller * Add chaffing feature flags in SFE/BFE. * add EventMessage to log context * Add request creation timestamp to ProtectedAuctionInput * Add request/response to ExecuteInternal on async clients * Add tee-container-log-redirect option in terraform * Bash script for ASG and Cloud Map Custom HealthChecks * Consented request replace enable_adtech_code_logging in Bidding Server * Create inference model store for model management * Create logging library for inference consented logs * Debug Reporting for Bid Currency * decrease aws/build_and_test duration by ~75% * Feature Flag for TLS in Service Mesh * Force the ML model reset with the probability of 0.1% * Forward per request consented debugging config to roma callback * Implement AWS Cloud Un-Map * Implement new SFE <> BFE request format for chaffing * Implement the probabilistic model reset for PyTorch * Implement the probabilistic model reset for TensorFlow. * Load Test Flag for AWS * Log consented debugging information in inference sidecar * log EventMessage in servers * log udf log in EventMessage for non_prod debug_info * Remove enableAdtechCodeLogging flag value * Remove Envoy Access Logging * Route consented inference requests to a consented model store * send chaff requests from SFE * Service Mesh in AWS * Support CPU isolation in the inference sidecar * update code/cloud build to use tags * Upgrade AWS Provider for Terraform from v3.xx to v4.xx * Use gRPC for AWS Service Mesh Envoy HCs ### Bug Fixes * [IsolateBuyerAndSellerCodeExecution] Add a new class for buyer's reporting code fetch and load * [IsolateBuyerAndSellerCodeExecution] Add a new code wrapper with only scoreAd and reportResult * [IsolateBuyerAndSellerCodeExecution] Add config flag to enable seller and buyer code isolation * [IsolateBuyerAndSellerCodeExecution] Modify seller_udf_manager to fetch and load buyer udfs * [IsolateBuyerAndSellerCodeExecution] Refactor the code fetch files. * add check back in default grpc client * Add DebugInfo pointer for debugging log into RomaRequestContext * Add generation_id to chaff requests * clean up log verbosity 3 * do not try to impersonate service accounts if TEST_MODE=true * Eliminate Terraform Error Message about empty Authority Field * Ensure instance id is set in logs on AWS when not using mesh. * Execute Callback for empty HTTP request vector * make num_chaff_requests not have a static lower bound * scorecard.yaml version updates * Upgrades google terraform plugin to 5.31.0 to fix crash Bug: N/A GitOrigin-RevId: faf77e5cbd6c2ae9f65761250ce95231c5f5ae08 Change-Id: I3e54553186f0ad82ffea0e43fa2ea3fbf22eb19f --- .bazelrc | 1 + .github/workflows/scorecard.yaml | 6 +- BUILD | 18 + CHANGELOG.md | 59 + WORKSPACE | 33 +- api/bidding_auction_servers.proto | 188 +- config.bzl | 10 +- .../terraform/environment/demo/buyer/buyer.tf | 28 +- .../environment/demo/buyer/terraform.tf | 2 +- .../environment/demo/seller/seller.tf | 42 +- .../environment/demo/seller/terraform.tf | 2 +- .../aws/terraform/modules/buyer/service.tf | 170 +- .../terraform/modules/buyer/service_vars.tf | 78 +- .../aws/terraform/modules/seller/service.tf | 173 +- .../terraform/modules/seller/service_vars.tf | 71 +- .../aws/terraform/services/app_mesh/main.tf | 24 + .../terraform/services/app_mesh/outputs.tf | 36 + .../terraform/services/app_mesh/variables.tf | 30 + .../autoscaling/instance_init_script.tftpl | 87 +- .../terraform/services/autoscaling/main.tf | 21 +- .../services/autoscaling/variables.tf | 47 +- .../services/backend_mesh_service/main.tf | 233 + .../services/backend_mesh_service/outputs.tf | 40 + .../backend_mesh_service/variables.tf | 120 + .../services/frontend_mesh_service/main.tf | 177 + .../services/frontend_mesh_service/outputs.tf | 30 + .../frontend_mesh_service/variables.tf | 115 + .../services/iam_role_policies/main.tf | 47 + .../services/load_balancing/variables.tf | 2 +- .../services/security_group_rules/main.tf | 18 + .../terraform/environment/demo/buyer/buyer.tf | 15 +- .../environment/demo/buyer/terraform.tf | 4 +- .../environment/demo/seller/seller.tf | 8 +- .../environment/demo/seller/terraform.tf | 4 +- .../gcp/terraform/modules/buyer/service.tf | 2 +- .../gcp/terraform/modules/seller/service.tf | 2 +- .../packaging/aws/auction_service/BUILD | 2 - .../packaging/aws/bidding_service/BUILD | 33 +- .../aws/bidding_service/test/structure.yaml | 4 + production/packaging/aws/build_and_test | 82 +- .../aws/buyer_frontend_service/BUILD | 2 - production/packaging/aws/codebuild/README.md | 36 +- .../packaging/aws/codebuild/buildspec.yaml | 3 +- .../codebuild/sync_bidding_auction_repo.yaml | 1 + production/packaging/aws/common/ami/BUILD | 3 + .../aws/common/ami/envoy_networking.sh | 273 + production/packaging/aws/common/ami/hc.bash | 200 + .../packaging/aws/common/ami/health.proto | 63 + .../packaging/aws/common/ami/image.pkr.hcl | 26 +- .../aws/seller_frontend_service/BUILD | 2 - .../packaging/build_and_test_all_in_docker | 28 +- .../packaging/gcp/auction_service/BUILD | 2 - .../packaging/gcp/bidding_service/BUILD | 33 +- .../gcp/bidding_service/test/structure.yaml | 4 + .../gcp/buyer_frontend_service/BUILD | 2 - .../packaging/gcp/cloud_build/README.md | 19 +- .../packaging/gcp/cloud_build/cloudbuild.yaml | 5 +- .../sync_bidding_auction_repo.yaml | 44 + .../gcp/seller_frontend_service/BUILD | 2 - services/auction_service/BUILD | 100 +- services/auction_service/auction_main.cc | 17 +- .../auction_service_integration_test.cc | 201 +- .../auction_service_integration_test_util.cc | 258 + .../auction_service_integration_test_util.h | 69 + ...tion_service_reporting_integration_test.cc | 87 + .../score_ads_reactor_benchmarks.cc | 8 +- .../buyer_adtech_reporting_wrapper.h | 75 - services/auction_service/code_wrapper/BUILD | 66 +- .../buyer_reporting_test_constants.h | 136 + .../buyer_reporting_udf_wrapper.cc | 27 + .../buyer_reporting_udf_wrapper.h | 91 + .../buyer_reporting_udf_wrapper_test.cc | 30 + .../code_wrapper/seller_code_wrapper.cc | 11 +- .../code_wrapper/seller_code_wrapper.h | 113 - .../code_wrapper/seller_code_wrapper_test.cc | 11 +- .../seller_code_wrapper_test_constants.h | 253 +- .../code_wrapper/seller_udf_wrapper.cc | 33 + .../code_wrapper/seller_udf_wrapper.h | 142 + .../code_wrapper/seller_udf_wrapper_test.cc | 40 + .../seller_udf_wrapper_test_constants.h | 252 + .../auction_service/data/runtime_config.h | 2 + services/auction_service/reporting/BUILD | 2 +- .../auction_service/reporting/buyer/BUILD | 45 + .../buyer/pa_buyer_reporting_manager.h | 43 + .../buyer/pas_buyer_reporting_manager.h | 44 + .../reporting/reporting_helper.cc | 45 +- .../reporting/reporting_helper.h | 36 +- .../reporting/reporting_helper_test.cc | 23 +- .../auction_service/reporting/seller/BUILD | 73 + .../component_seller_reporting_manager.h | 48 + .../seller/seller_reporting_manager.cc | 160 + .../seller/seller_reporting_manager.h | 60 + .../seller/seller_reporting_manager_test.cc | 199 + services/auction_service/score_ads_reactor.cc | 253 +- services/auction_service/score_ads_reactor.h | 22 +- .../auction_service/score_ads_reactor_test.cc | 24 +- .../score_ads_reactor_test_util.h | 7 + ...core_ads_reactor_top_level_auction_test.cc | 254 +- .../seller_code_fetch_manager_test.cc | 264 - services/auction_service/udf_fetcher/BUILD | 196 + .../udf_fetcher/adtech_code_version_util.cc | 46 + .../udf_fetcher/adtech_code_version_util.h | 41 + .../auction_code_fetch_config.proto | 17 +- .../buyer_reporting_fetcher.cc | 2 +- .../buyer_reporting_fetcher.h | 2 +- .../buyer_reporting_udf_fetch_manager.cc | 180 + .../buyer_reporting_udf_fetch_manager.h | 110 + .../buyer_reporting_udf_fetch_manager_test.cc | 288 + .../seller_udf_fetch_manager.cc} | 72 +- .../seller_udf_fetch_manager.h} | 29 +- .../seller_udf_fetch_manager_test.cc | 485 + services/auction_service/utils/BUILD | 2 +- services/auction_service/utils/proto_utils.cc | 16 +- services/auction_service/utils/proto_utils.h | 18 +- .../auction_service/utils/proto_utils_test.cc | 3 +- services/bidding_service/BUILD | 23 +- .../base_generate_bids_reactor.h | 9 +- .../generate_bids_reactor_benchmarks.cc | 11 +- .../bidding_code_fetch_config.proto | 13 +- services/bidding_service/bidding_main.cc | 15 +- .../bidding_service_integration_test.cc | 33 +- .../bidding_service/bidding_service_test.cc | 5 +- .../buyer_code_fetch_manager.cc | 6 +- .../buyer_code_fetch_manager_test.cc | 13 +- services/bidding_service/code_wrapper/BUILD | 2 +- .../code_wrapper/buyer_code_wrapper.cc | 2 +- .../code_wrapper/buyer_code_wrapper.h | 22 +- .../buyer_code_wrapper_test_constants.h | 40 +- .../bidding_service/data/runtime_config.h | 2 - .../bidding_service/egress_cddl_spec/1.0.0 | 99 + .../bidding_service/egress_cddl_spec/BUILD | 15 + .../bidding_service/generate_bids_reactor.cc | 29 +- .../generate_bids_reactor_test.cc | 170 +- .../generate_bids_reactor_test_utils.cc | 21 +- services/bidding_service/inference/BUILD | 3 + .../inference/inference_flags.cc | 2 +- .../inference/inference_utils.cc | 73 +- .../inference/inference_utils.h | 10 + .../inference/inference_utils_test.cc | 27 +- ...ected_app_signals_generate_bids_reactor.cc | 35 +- ..._app_signals_generate_bids_reactor_test.cc | 103 +- services/bidding_service/utils/BUILD | 51 + services/bidding_service/utils/egress.cc | 27 + services/bidding_service/utils/egress.h | 28 + services/bidding_service/utils/egress_test.cc | 325 + .../buyer_frontend_main.cc | 9 +- .../buyer_frontend_service.cc | 7 +- .../buyer_frontend_service_test.cc | 85 +- .../get_bids_unary_reactor.cc | 18 +- .../get_bids_unary_reactor.h | 4 +- .../get_bids_unary_reactor_test.cc | 100 +- .../buyer_frontend_service/runtime_flags.h | 5 +- services/common/blob_fetch/BUILD | 16 +- services/common/blob_fetch/fetch_mode.proto | 31 + services/common/chaffing/BUILD | 30 + services/common/chaffing/transcoding_utils.h | 126 + services/common/clients/BUILD | 2 + services/common/clients/async_client.h | 6 +- services/common/clients/async_grpc/BUILD | 10 + .../bidding_async_grpc_client_stub_test.h | 27 +- .../async_grpc/default_async_grpc_client.cc | 43 + .../async_grpc/default_async_grpc_client.h | 56 +- .../default_async_grpc_client_stub_test.h | 68 +- .../default_async_grpc_client_test.cc | 3 +- .../clients/async_grpc/grpc_client_utils.h | 6 +- .../clients/async_grpc/request_config.h | 35 + services/common/clients/auction_server/BUILD | 3 + .../auction_server/scoring_async_client.cc | 6 +- .../auction_server/scoring_async_client.h | 1 + .../scoring_async_client_test.cc | 9 +- .../bidding_server/bidding_async_client.h | 1 + .../clients/buyer_frontend_server/BUILD | 5 + .../buyer_frontend_async_client.cc | 119 +- .../buyer_frontend_async_client.h | 33 + .../buyer_frontend_async_client_factory.cc | 21 +- .../buyer_frontend_async_client_factory.h | 15 +- ...er_frontend_async_grpc_client_stub_test.cc | 193 + services/common/clients/client_factory.h | 6 + services/common/clients/client_params.h | 27 +- services/common/clients/code_dispatcher/BUILD | 5 +- .../code_dispatcher/request_context.cc | 17 +- .../clients/code_dispatcher/request_context.h | 28 +- .../http/multi_curl_http_fetcher_async.cc | 7 +- .../multi_curl_http_fetcher_async_test.cc | 14 + .../buyer_key_value_async_http_client_test.cc | 3 +- ...seller_key_value_async_http_client_test.cc | 2 +- .../http_kv_server/util/generate_url.h | 5 +- .../clients/kv_server/kv_async_client.cc | 6 +- .../clients/kv_server/kv_async_client.h | 6 +- .../clients/kv_server/kv_async_client_test.cc | 13 +- .../code_fetch/code_fetcher_interface.h | 3 + services/common/constants/BUILD | 10 + services/common/constants/common_constants.h | 30 + services/common/feature_flags.cc | 5 + services/common/feature_flags.h | 1 + services/common/loggers/BUILD | 21 +- services/common/loggers/request_log_context.h | 83 + .../loggers/request_log_context_test.cc | 27 + services/common/metric/error_code.h | 3 + .../common/telemetry/configure_telemetry.h | 2 +- services/common/test/mocks.h | 20 +- services/common/test/random.cc | 39 +- services/common/test/random.h | 10 + services/common/util/BUILD | 35 +- services/common/util/async_task_tracker.cc | 2 +- services/common/util/async_task_tracker.h | 6 +- .../common/util/async_task_tracker_test.cc | 5 +- services/common/util/data_util.h | 40 + services/common/util/data_util_test.cc | 29 + services/common/util/error_accumulator.cc | 2 +- services/common/util/error_accumulator.h | 6 +- services/common/util/error_categories.h | 4 +- services/common/util/json_util.h | 43 +- services/common/util/json_util_test.cc | 39 +- services/common/util/post_auction_signals.h | 4 + services/common/util/reporting_util.cc | 97 +- services/common/util/reporting_util.h | 23 +- services/common/util/reporting_util_test.cc | 67 +- services/inference_sidecar/README.md | 33 +- services/inference_sidecar/common/BUILD | 28 +- .../common/consented_logging_test.cc | 130 + .../inference_sidecar/common/grpc_sidecar.cc | 44 +- .../inference_sidecar/common/grpc_sidecar.h | 7 + .../common/inference_sidecar_main.cc | 5 + services/inference_sidecar/common/model/BUILD | 44 + .../common/model/model_store.h | 156 + .../common/model/model_store_test.cc | 407 + .../inference_sidecar/common/modules/BUILD | 2 + .../common/modules/module_interface.h | 6 +- .../common/modules/test_module.cc | 12 +- .../common/modules/test_module.h | 4 +- .../common/modules/test_module_test.cc | 6 + .../common/proto/inference_sidecar.proto | 22 + .../inference_sidecar/common/test_constants.h | 2 + .../inference_sidecar/common/testdata/BUILD | 2 + .../testdata/models/pytorch_stateful_model.pt | Bin 0 -> 2351 bytes .../common/tools/debug/start_inference | 16 +- services/inference_sidecar/common/utils/log.h | 12 +- .../common/utils/log_test.cc | 12 +- .../modules/pytorch_v2_1_1/BUILD | 16 +- .../modules/pytorch_v2_1_1/pytorch.cc | 158 +- .../modules/pytorch_v2_1_1/pytorch_test.cc | 138 +- .../modules/pytorch_v2_1_1/test_constants.h | 8 +- .../modules/tensorflow_v2_14_0/BUILD | 9 +- .../benchmark_models/stateful/BUILD | 23 + .../benchmark_models/stateful/saved_model.pb | Bin 0 -> 18376 bytes .../variables/variables.data-00000-of-00001 | Bin 0 -> 751 bytes .../stateful/variables/variables.index | Bin 0 -> 195 bytes .../modules/tensorflow_v2_14_0/tensorflow.cc | 186 +- .../tensorflow_v2_14_0/tensorflow_test.cc | 171 +- .../tensorflow_v2_14_0/test_constants.h | 9 +- services/seller_frontend_service/BUILD | 9 +- .../select_ad_reactor_benchmarks.cc | 43 +- ...et_component_auction_ciphertexts_reactor.h | 4 +- .../seller_frontend_service/runtime_flags.h | 5 +- .../schemas/auction_request.json | 5 +- .../schemas/auction_response.json | 119 + .../schemas/examples/auction_response.json | 33 +- .../select_ad_reactor.cc | 325 +- .../select_ad_reactor.h | 31 +- .../select_ad_reactor_app.cc | 1 + .../select_ad_reactor_app_test.cc | 172 +- ...client.cc => select_ad_reactor_invalid.cc} | 41 +- ...d_client.h => select_ad_reactor_invalid.h} | 13 +- .../select_ad_reactor_test.cc | 248 +- .../select_ad_reactor_web.cc | 10 +- .../select_ad_reactor_web_test.cc | 63 +- .../select_auction_result_reactor.cc | 4 +- .../select_auction_result_reactor.h | 2 +- .../select_auction_result_reactor_test.cc | 160 +- .../seller_frontend_main.cc | 6 + .../seller_frontend_service.cc | 10 +- .../seller_frontend_service.h | 6 +- .../seller_frontend_service_test.cc | 136 +- services/seller_frontend_service/util/BUILD | 3 +- .../util/proto_mapping_util.cc | 37 +- .../util/proto_mapping_util.h | 11 +- .../util/proto_mapping_util_test.cc | 5 +- .../util/select_ad_reactor_test_utils.cc | 29 +- .../util/select_ad_reactor_test_utils.h | 12 +- .../util/validation_utils.cc | 2 +- .../util/validation_utils_test.cc | 8 +- .../seller_frontend_service/util/web_utils.h | 13 +- .../util/web_utils_test.cc | 3 +- third_party/cddl/BUILD | 17 + third_party/cddl/BUILD.bazel | 15 + third_party/cddl/Cargo.lock | 1400 +++ third_party/cddl/Cargo.toml | 118 + third_party/cddl/WORKSPACE | 37 + third_party/cddl/cargo-bazel-lock.json | 8114 +++++++++++++++++ third_party/cddl/cddl.BUILD | 57 + third_party/cddl/cddl.h | 32 + third_party/cddl/cddl.patch | 60 + third_party/deps.bzl | 28 + tools/debug/start_auction | 12 +- tools/debug/start_bfe | 7 +- tools/debug/start_bidding | 11 +- tools/debug/start_sfe | 12 +- tools/secure_invoke/secure_invoke_lib.cc | 12 +- tools/secure_invoke/secure_invoke_lib_test.cc | 4 +- version.txt | 2 +- 301 files changed, 21708 insertions(+), 2512 deletions(-) create mode 100644 production/deploy/aws/terraform/services/app_mesh/main.tf create mode 100644 production/deploy/aws/terraform/services/app_mesh/outputs.tf create mode 100644 production/deploy/aws/terraform/services/app_mesh/variables.tf create mode 100644 production/deploy/aws/terraform/services/backend_mesh_service/main.tf create mode 100644 production/deploy/aws/terraform/services/backend_mesh_service/outputs.tf create mode 100644 production/deploy/aws/terraform/services/backend_mesh_service/variables.tf create mode 100644 production/deploy/aws/terraform/services/frontend_mesh_service/main.tf create mode 100644 production/deploy/aws/terraform/services/frontend_mesh_service/outputs.tf create mode 100644 production/deploy/aws/terraform/services/frontend_mesh_service/variables.tf create mode 120000 production/packaging/aws/codebuild/sync_bidding_auction_repo.yaml create mode 100644 production/packaging/aws/common/ami/envoy_networking.sh create mode 100755 production/packaging/aws/common/ami/hc.bash create mode 100644 production/packaging/aws/common/ami/health.proto create mode 100644 production/packaging/gcp/cloud_build/sync_bidding_auction_repo.yaml create mode 100644 services/auction_service/auction_service_integration_test_util.cc create mode 100644 services/auction_service/auction_service_integration_test_util.h create mode 100644 services/auction_service/auction_service_reporting_integration_test.cc delete mode 100644 services/auction_service/buyer_adtech_reporting_wrapper.h create mode 100644 services/auction_service/code_wrapper/buyer_reporting_test_constants.h create mode 100644 services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.cc create mode 100644 services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h create mode 100644 services/auction_service/code_wrapper/buyer_reporting_udf_wrapper_test.cc create mode 100644 services/auction_service/code_wrapper/seller_udf_wrapper.cc create mode 100644 services/auction_service/code_wrapper/seller_udf_wrapper.h create mode 100644 services/auction_service/code_wrapper/seller_udf_wrapper_test.cc create mode 100644 services/auction_service/code_wrapper/seller_udf_wrapper_test_constants.h create mode 100644 services/auction_service/reporting/buyer/BUILD create mode 100644 services/auction_service/reporting/buyer/pa_buyer_reporting_manager.h create mode 100644 services/auction_service/reporting/buyer/pas_buyer_reporting_manager.h create mode 100644 services/auction_service/reporting/seller/BUILD create mode 100644 services/auction_service/reporting/seller/component_seller_reporting_manager.h create mode 100644 services/auction_service/reporting/seller/seller_reporting_manager.cc create mode 100644 services/auction_service/reporting/seller/seller_reporting_manager.h create mode 100644 services/auction_service/reporting/seller/seller_reporting_manager_test.cc delete mode 100644 services/auction_service/seller_code_fetch_manager_test.cc create mode 100644 services/auction_service/udf_fetcher/BUILD create mode 100644 services/auction_service/udf_fetcher/adtech_code_version_util.cc create mode 100644 services/auction_service/udf_fetcher/adtech_code_version_util.h rename services/auction_service/{ => udf_fetcher}/auction_code_fetch_config.proto (87%) rename services/auction_service/{code_wrapper => udf_fetcher}/buyer_reporting_fetcher.cc (98%) rename services/auction_service/{code_wrapper => udf_fetcher}/buyer_reporting_fetcher.h (97%) create mode 100644 services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.cc create mode 100644 services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.h create mode 100644 services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager_test.cc rename services/auction_service/{seller_code_fetch_manager.cc => udf_fetcher/seller_udf_fetch_manager.cc} (68%) rename services/auction_service/{seller_code_fetch_manager.h => udf_fetcher/seller_udf_fetch_manager.h} (77%) create mode 100644 services/auction_service/udf_fetcher/seller_udf_fetch_manager_test.cc create mode 100644 services/bidding_service/egress_cddl_spec/1.0.0 create mode 100644 services/bidding_service/egress_cddl_spec/BUILD create mode 100644 services/bidding_service/utils/BUILD create mode 100644 services/bidding_service/utils/egress.cc create mode 100644 services/bidding_service/utils/egress.h create mode 100644 services/bidding_service/utils/egress_test.cc create mode 100644 services/common/blob_fetch/fetch_mode.proto create mode 100644 services/common/chaffing/BUILD create mode 100644 services/common/chaffing/transcoding_utils.h create mode 100644 services/common/clients/async_grpc/default_async_grpc_client.cc create mode 100644 services/common/clients/async_grpc/request_config.h create mode 100644 services/common/constants/common_constants.h create mode 100644 services/common/loggers/request_log_context.h create mode 100644 services/common/loggers/request_log_context_test.cc create mode 100644 services/common/util/data_util.h create mode 100644 services/common/util/data_util_test.cc create mode 100644 services/inference_sidecar/common/consented_logging_test.cc create mode 100644 services/inference_sidecar/common/model/BUILD create mode 100644 services/inference_sidecar/common/model/model_store.h create mode 100644 services/inference_sidecar/common/model/model_store_test.cc create mode 100644 services/inference_sidecar/common/testdata/models/pytorch_stateful_model.pt create mode 100644 services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark_models/stateful/BUILD create mode 100644 services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark_models/stateful/saved_model.pb create mode 100644 services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark_models/stateful/variables/variables.data-00000-of-00001 create mode 100644 services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark_models/stateful/variables/variables.index rename services/seller_frontend_service/{select_ad_reactor_invalid_client.cc => select_ad_reactor_invalid.cc} (51%) rename services/seller_frontend_service/{select_ad_reactor_invalid_client.h => select_ad_reactor_invalid.h} (84%) create mode 100644 third_party/cddl/BUILD create mode 100644 third_party/cddl/BUILD.bazel create mode 100644 third_party/cddl/Cargo.lock create mode 100644 third_party/cddl/Cargo.toml create mode 100644 third_party/cddl/WORKSPACE create mode 100644 third_party/cddl/cargo-bazel-lock.json create mode 100644 third_party/cddl/cddl.BUILD create mode 100644 third_party/cddl/cddl.h create mode 100644 third_party/cddl/cddl.patch create mode 100644 third_party/deps.bzl diff --git a/.bazelrc b/.bazelrc index 76541859..43c74980 100644 --- a/.bazelrc +++ b/.bazelrc @@ -92,6 +92,7 @@ build:prod --@google_privacysandbox_servers_common//:build_flavor=prod build:inference_non_prod --//:build_flavor=non_prod build:inference_non_prod --//:inference_build=yes +build:inference_non_prod --//:inference_runtime=all build:inference_non_prod --@google_privacysandbox_servers_common//:build_flavor=non_prod # Address sanitizer, set action_env to segregate cache entries diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml index dbcb6200..d36e50b9 100644 --- a/.github/workflows/scorecard.yaml +++ b/.github/workflows/scorecard.yaml @@ -51,7 +51,7 @@ jobs: persist-credentials: false - name: Run analysis - uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 # v2.1.2 + uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 with: results_file: results.sarif results_format: sarif @@ -73,7 +73,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: Upload artifact - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0 + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: name: SARIF file path: results.sarif @@ -81,6 +81,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: Upload to code-scanning - uses: github/codeql-action/upload-sarif@17573ee1cc1b9d061760f3a006fc4aac4f944fd5 # v2.2.4 + uses: github/codeql-action/upload-sarif@8662eabe0e9f338a07350b7fd050732745f93848 # v2.3.1 with: sarif_file: results.sarif diff --git a/BUILD b/BUILD index a23dc523..86d922f5 100644 --- a/BUILD +++ b/BUILD @@ -173,6 +173,14 @@ config_setting( visibility = ["//visibility:public"], ) +config_setting( + name = "prod_build", + flag_values = { + ":build_flavor": "prod", + }, + visibility = ["//visibility:public"], +) + string_flag( name = "inference_build", build_setting_default = "no", @@ -189,6 +197,7 @@ string_flag( "noop", "pytorch", "tensorflow", + "all", ], ) @@ -218,6 +227,15 @@ config_setting( visibility = ["//visibility:public"], ) +config_setting( + name = "include_all_inference_binaries", + flag_values = { + ":inference_build": "yes", + ":inference_runtime": "all", + }, + visibility = ["//visibility:public"], +) + string_flag( name = "build_for_test", build_setting_default = "non_test", diff --git a/CHANGELOG.md b/CHANGELOG.md index 61040347..731ae11e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,65 @@ All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines. +## 3.9.0 (2024-06-25) + + +### Features + +* [IsolateBuyerAndSellerCodeExecution] Add wrapper for buyer's reportWin() udf +* [reporting] Enable reportResult execution for topLevelSeller +* Add chaffing feature flags in SFE/BFE. +* add EventMessage to log context +* Add request creation timestamp to ProtectedAuctionInput +* Add request/response to ExecuteInternal on async clients +* Add tee-container-log-redirect option in terraform +* Bash script for ASG and Cloud Map Custom HealthChecks +* Consented request replace enable_adtech_code_logging in Bidding Server +* Create inference model store for model management +* Create logging library for inference consented logs +* Debug Reporting for Bid Currency +* decrease aws/build_and_test duration by ~75% +* Feature Flag for TLS in Service Mesh +* Force the ML model reset with the probability of 0.1% +* Forward per request consented debugging config to roma callback +* Implement AWS Cloud Un-Map +* Implement new SFE <> BFE request format for chaffing +* Implement the probabilistic model reset for PyTorch +* Implement the probabilistic model reset for TensorFlow. +* Load Test Flag for AWS +* Log consented debugging information in inference sidecar +* log EventMessage in servers +* log udf log in EventMessage for non_prod debug_info +* Remove enableAdtechCodeLogging flag value +* Remove Envoy Access Logging +* Route consented inference requests to a consented model store +* send chaff requests from SFE +* Service Mesh in AWS +* Support CPU isolation in the inference sidecar +* update code/cloud build to use tags +* Upgrade AWS Provider for Terraform from v3.xx to v4.xx +* Use gRPC for AWS Service Mesh Envoy HCs + + +### Bug Fixes + +* [IsolateBuyerAndSellerCodeExecution] Add a new class for buyer's reporting code fetch and load +* [IsolateBuyerAndSellerCodeExecution] Add a new code wrapper with only scoreAd and reportResult +* [IsolateBuyerAndSellerCodeExecution] Add config flag to enable seller and buyer code isolation +* [IsolateBuyerAndSellerCodeExecution] Modify seller_udf_manager to fetch and load buyer udfs +* [IsolateBuyerAndSellerCodeExecution] Refactor the code fetch files. +* add check back in default grpc client +* Add DebugInfo pointer for debugging log into RomaRequestContext +* Add generation_id to chaff requests +* clean up log verbosity 3 +* do not try to impersonate service accounts if TEST_MODE=true +* Eliminate Terraform Error Message about empty Authority Field +* Ensure instance id is set in logs on AWS when not using mesh. +* Execute Callback for empty HTTP request vector +* make num_chaff_requests not have a static lower bound +* scorecard.yaml version updates +* Upgrades google terraform plugin to 5.31.0 to fix crash + ## 3.8.0 (2024-05-24) diff --git a/WORKSPACE b/WORKSPACE index ca1131c8..65f9a6b1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -23,11 +23,11 @@ http_archive( http_archive( name = "google_privacysandbox_servers_common", - # 2024-05-24 - sha256 = "444f76d69afba08dfe7424dae0ff8afb26bd3f4e3396d76b7bf55668d3afd7c6", - strip_prefix = "data-plane-shared-libraries-3e09f7993cf6e4ef0fd33360b778f993d3cfd9ef", + # 2024-05-28 + sha256 = "a241ad3d8d897599a6acff89fc759d0e858f8466988f7c56c57b0a83b8cdab5d", + strip_prefix = "data-plane-shared-libraries-3e92e75fceb18694f1ce7177e7433824769a89a0", urls = [ - "https://github.com/privacysandbox/data-plane-shared-libraries/archive/3e09f7993cf6e4ef0fd33360b778f993d3cfd9ef.zip", + "https://github.com/privacysandbox/data-plane-shared-libraries/archive/3e92e75fceb18694f1ce7177e7433824769a89a0.zip", ], ) @@ -138,3 +138,28 @@ http_archive( strip_prefix = "libevent-release-2.1.12-stable", urls = ["https://github.com/libevent/libevent/archive/refs/tags/release-2.1.12-stable.zip"], ) + +load("@rules_rust//crate_universe:repositories.bzl", "crate_universe_dependencies") + +crate_universe_dependencies() + +load("//third_party:deps.bzl", cddl_deps = "deps") + +cddl_deps() + +load("@cddl_crate_index//:defs.bzl", cddl_crate_repositories = "crate_repositories") + +cddl_crate_repositories() + +http_archive( + name = "cddl_lib", + build_file = "//third_party/cddl:cddl.BUILD", + patch_args = ["-p1"], + patches = [ + "//third_party:cddl/cddl.patch", + ], + sha256 = "01e04989c6482e851dc22f376f1c2e1cc493e1ae7b808ae78180d539e6939acb", + strip_prefix = "cddl-0.9.4", + urls = ["https://github.com/anweiss/cddl/archive/refs/tags/0.9.4.zip"], + workspace_file = "//third_party/cddl:WORKSPACE", +) diff --git a/api/bidding_auction_servers.proto b/api/bidding_auction_servers.proto index 049ca11e..6f56967e 100644 --- a/api/bidding_auction_servers.proto +++ b/api/bidding_auction_servers.proto @@ -28,10 +28,11 @@ import "src/logger/logger.proto"; message ProtectedAudienceInput { option deprecated = true; // Input per buyer. - // The key in the map corresponds to IGOwner (Interest Group Owner) that - // is the Buyer / DSP. This string that can identify a - // buyer participating in the auction. The value corresponds to plaintext - // BuyerInput ingested by the buyer for bidding. + // The key in the map corresponds to the Buyer / DSP that owns interest + // groups and protected app signals (in case of Android) that are part of + // this request. This string can identify a buyer participating in the + // auction. The value corresponds to plaintext BuyerInput ingested by the + // buyer for bidding. map buyer_input = 1; // Publisher website or app. @@ -63,10 +64,11 @@ message ProtectedAudienceInput { // for the Protected Audience auction. message ProtectedAuctionInput { // Input per buyer. - // The key in the map corresponds to IGOwner (Interest Group Owner) that - // is the Buyer / DSP. This string that can identify a - // buyer participating in the auction. The value corresponds to plaintext - // BuyerInput ingested by the buyer for bidding. + // The key in the map corresponds to the Buyer / DSP that owns interest + // groups and protected app signals (in case of Android) that are part of + // this request. This string can identify a buyer participating in the + // auction. The value corresponds to plaintext BuyerInput ingested by the + // buyer for bidding. map buyer_input = 1; // Publisher website or app. @@ -89,6 +91,13 @@ message ProtectedAuctionInput { // A boolean value which indicates whether temporary unlimited egress should // be enabled. bool enable_unlimited_egress = 6; + + // Optional field (note: this will become a required field in 2025). + // The request timestamp represents when the SSP requests an ad auction to be run. + // For Chrome, the request timestamp will be passed to the browser by the SSP + // when it requests the browser to run an auction. For Android, the request + // timestamp will come from the system clock on the device. + int64 request_timestamp_ms = 7; } // Grouping of data pertaining to protected app signals. @@ -336,6 +345,137 @@ message AuctionResult { // This will be verified with the buyerAndSellerReportingId in the Ad // properties on the browser. string buyer_and_seller_reporting_id = 20; + + // Join candidates for K-Anonymity for the winner. This should include + // key hashes corresponding to the winning ad only. + // Refer https://wicg.github.io/turtledove/#k-anonymity for details around + // how clients call the Join API using the hash. + message KAnonJoinCandidate { + // Protected Audience: + // - SHA-256 hash of the tuple: render_url, interest group owner, reportWin() + // UDF endpoint. + // + // Protected App Signals: + // - SHA-256 hash of render_url, reportWin UDF endpoint. + // + // NOTE: The buyer's reportWin() UDF url endpoint must match with what + // Chrome has. + // Refer to the spec to create key hashes- https://wicg.github.io/turtledove/#query-ad-k-anonymity-count + string ad_render_url_hash = 1; + + // Protected Audience: + // - SHA-256 hash of an ad_component_render_url. + // + // Refer to the spec to create key hashes- https://wicg.github.io/turtledove/#query-component-ad-k-anonymity-count + // + // Note: There is a maximum limit of 40 ad component render urls per + // render url. + // Note: Currently component ads are not in scope of Protected App Signals + // for Android. + repeated string ad_component_render_urls_hash = 2; + + // Protected Audience: + // - SHA-256 hash should include IG owner, ad render url, reportWin() UDF + // url and one of the following: + // - If buyerAndSellerReportingId exists, this hash should include that. + // - Else if buyerReportingId exists, hash should include that. + // - Else IG name should be included in the hash. + // + // Note: By design, an adtech can use either buyerReportingId or + // buyerAndSellerReportingId. + // Note: Currently reporting Ids are not in scope of Protected Audience + // for Android and Protected App Signal for Android. + string reporting_id_hash = 3; + } + + KAnonJoinCandidate k_anon_winner_join_candidates = 21; + + // Positional index of the k-anon winner, if there is a winner returned to + // the client (web browser, Android) for emitting metrics. + // + // Note: Positional index >= 0. + // - If this is equal to 0, that would imply the highest scored bid is also + // K-Anonymous and hence a winner. + // - If this is greater than 0, the positional index would imply the index of + // first K-Anonymous scored bid in a sorted list in decreasing order of scored + // bids. In this case, the highest scored bid that is not K-Anonymous is the + // ghost winner. + // - In case all scored bids fail the K-Anonymity constraint, this would be + // set to -1 since there is no winner. + // - In case all scored bids <= 0, this would be set to -1 since there is no + // winner. + optional int32 k_anon_winner_positional_index = 22; + + // Data for the ghost winner sent back to the client. This should also include + // key-hashes corresponding to the ghost winning ad. + // Refer https://wicg.github.io/turtledove/#k-anonymity for details around + // how clients call the Join API using the hash. + message KAnonGhostWinner { + // Join candidates for the K-Anon ghost winner. + KAnonJoinCandidate k_anon_join_candidates = 1; + + // Interest group index corresponding to buyer / DSP + // that generated the ghost winning bid. + // Note: This is only relevant in case of Protected Audience. + int32 interest_group_index = 2; + + // Origin (Chrome) and domain (Android) of the buyer / DSP who owns + // the ghost winner. + // Protected Audience: This refers to the Interest Group owner. + // Proptected App Signal: This refers to the buyer domain. + string owner = 3; + + // Private aggregation signals for the ghost winner. + // Single seller auctions: This would correspond to a ghost winner + // if available. + // + // Note: Event type is "reserved.loss" and bid rejection reason is 8 + // when K-Anonymity threshold is not. The client should incorporate these static + // values; these won't be sent back. + message GhostWinnerPrivateAggregationSignals { + // 128 bit integer in the form of bytestring. + bytes bucket = 1; + + int32 value = 2; + } + optional GhostWinnerPrivateAggregationSignals ghost_winner_private_aggregation_signals = 4; + + // In case of multiseller auction, the associated data for the ghost winner + // will be returned so that the ghost winning bid can be scored + // during top level auction. This is true if there is a ghost winner + // after component level auction. + // - In case of device orchestrated component auction for web, this data will + // returned to the browser so that the ghost winner can be passed to the top + // level auction on-device. + // - In case of server orchestrated component auction (for web, Android), this + // data will be returned to the top level seller by the component level + // sellers. + message GhostWinnerForTopLevelAuction { + // The ad render url corresponding to ghost winner. + string ad_render_url = 1; + + // Render URLs for ads which are components of the main ghost winning ad. + repeated string ad_component_render_urls = 2; + + // Modified bid price corresponding to ghost winning bid. + float modified_bid = 3; + + // Optional. Indicates the currency used for the bid price corresponding + // to the ghost winner (expressed as ISO 4217 alpha code). + string bid_currency = 4; + + // Arbitrary metadata associated with ghost winner to pass to the + // top-level seller during top level auction. + string ad_metadata = 5; + + // BuyerAndSellerReportingId of the ghost winning Ad. + // This will be verified with the buyerAndSellerReportingId in the Ad + // properties on the browser. + string buyer_and_seller_reporting_id = 6; + } + optional GhostWinnerForTopLevelAuction ghost_winner_for_top_level_auction = 5; + } + repeated KAnonGhostWinner k_anon_ghost_winners = 23; } message GetComponentAuctionCiphertextsRequest { @@ -1437,3 +1577,35 @@ message DebugReportUrls { // If undefined or malformed url it will be ignored. string auction_debug_loss_url = 2; } + +// message to store the server request/ response +message EventMessage { + repeated string udf_log = 1; + + // sfe + SelectAdRequest select_ad_request = 2; + oneof protected_input { + ProtectedAuctionInput protected_auction = 3; + ProtectedAudienceInput protected_audience = 4; + } + AuctionResult auction_result = 5; + + // bfe + GetBidsRequest get_bid_request = 11; + GetBidsRequest.GetBidsRawRequest get_bid_raw_request = 12; + GetBidsResponse.GetBidsRawResponse get_bid_raw_response = 13; + + // bidding + GenerateBidsRequest generate_bid_request = 21; + GenerateBidsRequest.GenerateBidsRawRequest generate_bid_raw_request = 22; + GenerateBidsResponse.GenerateBidsRawResponse generate_bid_raw_response = 23; + + GenerateProtectedAppSignalsBidsRequest generate_app_signal_request = 24; + GenerateProtectedAppSignalsBidsRequest.GenerateProtectedAppSignalsBidsRawRequest generate_app_signal_raw_request = 25; + GenerateProtectedAppSignalsBidsResponse.GenerateProtectedAppSignalsBidsRawResponse generate_app_signal_raw_response = 26; + + // auction + ScoreAdsRequest score_ad_request = 31; + ScoreAdsRequest.ScoreAdsRawRequest score_ad_raw_request = 32; + ScoreAdsResponse.ScoreAdsRawResponse score_ad_raw_response = 33; +} diff --git a/config.bzl b/config.bzl index 5d394b1d..d11b5719 100644 --- a/config.bzl +++ b/config.bzl @@ -14,12 +14,12 @@ """Top-level config-related variables.""" -LOG_ENV_VARS = select({ - "//:non_prod_build": {"GLOG_v": "10"}, - "//conditions:default": {"GLOG_v": "0"}, -}) - ENABLE_CORE_DUMPS_DEFINES = select({ "//:non_prod_build": ["PS_ENABLE_CORE_DUMPS=true"], "//conditions:default": ["PS_ENABLE_CORE_DUMPS=false"], }) + +IS_PROD_BUILD_DEFINES = select({ + "//:prod_build": ["PS_IS_PROD_BUILD=true"], + "//conditions:default": ["PS_IS_PROD_BUILD=false"], +}) diff --git a/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf b/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf index 2b2eb6d8..be148af4 100644 --- a/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf +++ b/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf @@ -58,7 +58,7 @@ module "buyer" { ENABLE_BIDDING_SERVICE_BENCHMARK = "" # Example: "false" BIDDING_SERVER_ADDR = "" # Example: "dns:///bidding1.com:443" - BUYER_KV_SERVER_ADDR = "" # Example: "https://googleads.g.doubleclick.net/td/bts" + BUYER_KV_SERVER_ADDR = "" # Example: "https://kvserver.com/trusted-signals" TEE_AD_RETRIEVAL_KV_SERVER_ADDR = "" # Example: "xds:///ad-retrieval-host" TEE_KV_SERVER_ADDR = "" # Example: "xds:///kv-service-host" AD_RETRIEVAL_TIMEOUT_MS = "60000" @@ -86,7 +86,6 @@ module "buyer" { # "urlFetchPeriodMs": 13000000, # "urlFetchTimeoutMs": 30000, # "enableBuyerDebugUrlGeneration": true, - # "enableAdtechCodeLogging": false, # "prepareDataForAdsRetrievalJsUrl": "", # "prepareDataForAdsRetrievalWasmHelperUrl": "", # }" @@ -97,20 +96,23 @@ module "buyer" { # Note: turning on this flag will lead to higher memory consumption for AdTech code execution # and additional latency for parsing the logs. + # Coordinator-based attestation flags. + # These flags are production-ready and you do not need to change them. + PUBLIC_KEY_ENDPOINT = "https://publickeyservice.pa.aws.privacysandboxservices.com/.well-known/protected-auction/v1/public-keys" + PRIMARY_COORDINATOR_PRIVATE_KEY_ENDPOINT = "https://privatekeyservice-a.pa-3.aws.privacysandboxservices.com/v1alpha/encryptionKeys" + SECONDARY_COORDINATOR_PRIVATE_KEY_ENDPOINT = "https://privatekeyservice-b.pa-4.aws.privacysandboxservices.com/v1alpha/encryptionKeys" + PRIMARY_COORDINATOR_REGION = "us-east-1" + SECONDARY_COORDINATOR_REGION = "us-east-1" + PRIVATE_KEY_CACHE_TTL_SECONDS = "3974400" + KEY_REFRESH_FLOW_RUN_FREQUENCY_SECONDS = "20000" # Reach out to the Privacy Sandbox B&A team to enroll with Coordinators and update the following flag values. # More information on enrollment can be found here: https://github.com/privacysandbox/fledge-docs/blob/main/bidding_auction_services_api.md#enroll-with-coordinators # Coordinator-based attestation flags: - PUBLIC_KEY_ENDPOINT = "" # Example: "https://test.cloudfront.net/v1alpha/publicKeys" - PRIMARY_COORDINATOR_PRIVATE_KEY_ENDPOINT = "" # Example: "https://test.execute-api.us-east-1.amazonaws.com/stage/v1alpha/encryptionKeys" - SECONDARY_COORDINATOR_PRIVATE_KEY_ENDPOINT = "" # Example: "https://test.execute-api.us-east-1.amazonaws.com/stage/v1alpha/encryptionKeys" - PRIMARY_COORDINATOR_ACCOUNT_IDENTITY = "" # Example: "arn:aws:iam::574738241111:role/mp-prim-ba_574738241111_coordinator_assume_role" - SECONDARY_COORDINATOR_ACCOUNT_IDENTITY = "" # Example: "arn:aws:iam::574738241111:role/mp-sec-ba_574738241111_coordinator_assume_role" - PRIMARY_COORDINATOR_REGION = "" # Example: "us-east-1" - SECONDARY_COORDINATOR_REGION = "" # Example: "us-east-1" - PRIVATE_KEY_CACHE_TTL_SECONDS = "" # Example: "3974400" (46 days) - KEY_REFRESH_FLOW_RUN_FREQUENCY_SECONDS = "" # Example: "10800" - MAX_ALLOWED_SIZE_DEBUG_URL_BYTES = "" # Example: "65536" - MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB = "" # Example: "3000" + PRIMARY_COORDINATOR_ACCOUNT_IDENTITY = "" # Example: "arn:aws:iam::811625435250:role/a__coordinator_assume_role" + SECONDARY_COORDINATOR_ACCOUNT_IDENTITY = "" # Example: "arn:aws:iam::891377198286:role/b__coordinator_assume_role" + + MAX_ALLOWED_SIZE_DEBUG_URL_BYTES = "" # Example: "65536" + MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB = "" # Example: "3000" INFERENCE_SIDECAR_BINARY_PATH = "" # Example: "/server/bin/inference_sidecar" INFERENCE_MODEL_BUCKET_NAME = "" # Example: "" diff --git a/production/deploy/aws/terraform/environment/demo/buyer/terraform.tf b/production/deploy/aws/terraform/environment/demo/buyer/terraform.tf index aa14c562..873865b8 100644 --- a/production/deploy/aws/terraform/environment/demo/buyer/terraform.tf +++ b/production/deploy/aws/terraform/environment/demo/buyer/terraform.tf @@ -18,7 +18,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 3.0" + version = "~> 4.0" } } } diff --git a/production/deploy/aws/terraform/environment/demo/seller/seller.tf b/production/deploy/aws/terraform/environment/demo/seller/seller.tf index 7c5f4546..eba608c3 100644 --- a/production/deploy/aws/terraform/environment/demo/seller/seller.tf +++ b/production/deploy/aws/terraform/environment/demo/seller/seller.tf @@ -59,9 +59,9 @@ module "seller" { GET_BID_RPC_TIMEOUT_MS = "" # Example: "60000" KEY_VALUE_SIGNALS_FETCH_RPC_TIMEOUT_MS = "" # Example: "60000" SCORE_ADS_RPC_TIMEOUT_MS = "" # Example: "60000" - SELLER_ORIGIN_DOMAIN = "" # Example: "https://securepubads.g.doubleclick.net" + SELLER_ORIGIN_DOMAIN = "" # Example: "https://sellerorigin.com" AUCTION_SERVER_HOST = "" # Example: "dns:///auction.seller-frontend.com:443" - KEY_VALUE_SIGNALS_HOST = "" # Example: "https://pubads.g.doubleclick.net/td/sts" + KEY_VALUE_SIGNALS_HOST = "" # Example: "https://keyvaluesignals.com/trusted-signals" BUYER_SERVER_HOSTS = "" # Example: "{ \"https://bid1.com\": { \"url\": \"dns:///bidding1.com:443\", \"cloudPlatform\": \"AWS\" } }" SELLER_CLOUD_PLATFORMS_MAP = "" # Example: "{ \"https://partner-seller1.com\": "GCP", \"https://partner-seller2.com\": "AWS"}" ENABLE_SELLER_FRONTEND_BENCHMARKING = "" # Example: "false" @@ -87,7 +87,6 @@ module "seller" { # execution need to be exported as VLOG. # Note: turning on this flag will lead to higher memory consumption for AdTech code execution # and additional latency for parsing the logs. - # "enableAdtechCodeLogging": false, # "enableReportResultUrlGeneration": true, # "enableReportWinUrlGeneration": true, # "buyerReportWinJsUrls": {"https://buyerA_origin.com":"https://buyerA.com/generateBid.js", @@ -100,25 +99,30 @@ module "seller" { JS_WORKER_QUEUE_LEN = "" # Example: "100". ROMA_TIMEOUT_MS = "" # Example: "10000" ENABLE_REPORT_WIN_INPUT_NOISING = "" # Example: "true" + + # Coordinator-based attestation flags. + # These flags are production-ready and you do not need to change them. + SFE_PUBLIC_KEYS_ENDPOINTS = <_coordinator_assume_role" + SECONDARY_COORDINATOR_ACCOUNT_IDENTITY = "" # Example: "arn:aws:iam::891377198286:role/b__coordinator_assume_role" + + MAX_ALLOWED_SIZE_DEBUG_URL_BYTES = "" # Example: "65536" + MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB = "" # Example: "3000" # TCMalloc related config parameters. # See: https://github.com/google/tcmalloc/blob/master/docs/tuning.md diff --git a/production/deploy/aws/terraform/environment/demo/seller/terraform.tf b/production/deploy/aws/terraform/environment/demo/seller/terraform.tf index aa14c562..873865b8 100644 --- a/production/deploy/aws/terraform/environment/demo/seller/terraform.tf +++ b/production/deploy/aws/terraform/environment/demo/seller/terraform.tf @@ -18,7 +18,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 3.0" + version = "~> 4.0" } } } diff --git a/production/deploy/aws/terraform/modules/buyer/service.tf b/production/deploy/aws/terraform/modules/buyer/service.tf index c0b0c120..e48bac70 100644 --- a/production/deploy/aws/terraform/modules/buyer/service.tf +++ b/production/deploy/aws/terraform/modules/buyer/service.tf @@ -107,12 +107,54 @@ module "buyer_dashboard" { region = var.region } -################ Buyer FrontEnd operator Setup ################ -module "load_balancing_bfe" { +module "buyer_app_mesh" { + # Only create if using service mesh + count = var.use_service_mesh ? 1 : 0 + + source = "../../services/app_mesh" + operator = var.operator + environment = var.environment + vpc_id = module.networking.vpc_id +} + +################ Bidding operator Setup ################ + +module "bidding_mesh_service" { + # Only create if using service mesh + count = var.use_service_mesh ? 1 : 0 + + source = "../../services/backend_mesh_service" + operator = var.operator + environment = var.environment + service = "bidding" + app_mesh_id = module.buyer_app_mesh[0].app_mesh_id + app_mesh_name = module.buyer_app_mesh[0].app_mesh_name + root_domain = var.root_domain + cloud_map_private_dns_namespace_id = module.buyer_app_mesh[0].cloud_map_private_dns_namespace_id + cloud_map_private_dns_namespace_name = module.buyer_app_mesh[0].cloud_map_private_dns_namespace_name + server_instance_role_name = module.iam_roles.instance_role_name + business_org_for_cert_auth = var.business_org_for_cert_auth + country_for_cert_auth = var.country_for_cert_auth + state_for_cert_auth = var.state_for_cert_auth + locality_for_cert_auth = var.locality_for_cert_auth + org_unit_for_cert_auth = var.org_unit_for_cert_auth + service_port = var.server_port + root_domain_zone_id = var.root_domain_zone_id + healthcheck_interval_sec = var.healthcheck_interval_sec + healthcheck_timeout_sec = var.healthcheck_timeout_sec + healthcheck_healthy_threshold = var.healthcheck_healthy_threshold + healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold + use_tls_with_mesh = var.use_tls_with_mesh +} + +module "load_balancing_bidding" { + # Only create if not using service mesh + count = var.use_service_mesh ? 0 : 1 + source = "../../services/load_balancing" environment = var.environment operator = var.operator - service = "bfe" + service = "bidding" certificate_arn = var.certificate_arn elb_subnet_ids = module.networking.public_subnet_ids server_port = var.server_port @@ -123,35 +165,72 @@ module "load_balancing_bfe" { healthcheck_healthy_threshold = var.healthcheck_healthy_threshold healthcheck_interval_sec = var.healthcheck_interval_sec healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold + # Recommended not to change. Ensures internal VPC load balancers for traffic over private network. + internal = true } -module "autoscaling_bfe" { - source = "../../services/autoscaling" - environment = var.environment - operator = var.operator - enclave_debug_mode = var.enclave_debug_mode - service = "bfe" - autoscaling_subnet_ids = module.networking.private_subnet_ids - instance_ami_id = var.bfe_instance_ami_id - instance_security_group_id = module.security_groups.instance_security_group_id - instance_type = var.bfe_instance_type - target_group_arns = module.load_balancing_bfe.target_group_arns - autoscaling_desired_capacity = var.bfe_autoscaling_desired_capacity - autoscaling_max_size = var.bfe_autoscaling_max_size - autoscaling_min_size = var.bfe_autoscaling_min_size - instance_profile_arn = module.iam_roles.instance_profile_arn - enclave_cpu_count = var.bfe_enclave_cpu_count - enclave_memory_mib = var.bfe_enclave_memory_mib +module "autoscaling_bidding" { + source = "../../services/autoscaling" + environment = var.environment + operator = var.operator + enclave_debug_mode = var.enclave_debug_mode + service = "bidding" + autoscaling_subnet_ids = module.networking.private_subnet_ids + instance_ami_id = var.bidding_instance_ami_id + instance_security_group_id = module.security_groups.instance_security_group_id + instance_type = var.bidding_instance_type + target_group_arns = var.use_service_mesh ? [] : module.load_balancing_bidding[0].target_group_arns + autoscaling_desired_capacity = var.bidding_autoscaling_desired_capacity + autoscaling_max_size = var.bidding_autoscaling_max_size + autoscaling_min_size = var.bidding_autoscaling_min_size + instance_profile_arn = module.iam_roles.instance_profile_arn + enclave_cpu_count = var.bidding_enclave_cpu_count + enclave_memory_mib = var.bidding_enclave_memory_mib + cloud_map_service_id = var.use_service_mesh ? module.bidding_mesh_service[0].cloud_map_service_id : "" + region = var.region + app_mesh_name = var.use_service_mesh ? module.buyer_app_mesh[0].app_mesh_name : "" + virtual_node_name = var.use_service_mesh ? module.bidding_mesh_service[0].virtual_node_name : "" + healthcheck_interval_sec = var.healthcheck_interval_sec + healthcheck_timeout_sec = var.healthcheck_timeout_sec + healthcheck_healthy_threshold = var.healthcheck_healthy_threshold + healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold + healthcheck_grace_period_sec = var.healthcheck_grace_period_sec } -################ Bidding operator Setup ################ +################ Buyer FrontEnd operator Setup ################ +module "bfe_mesh_service" { + # Only create if using service mesh + count = var.use_service_mesh ? 1 : 0 -module "load_balancing_bidding" { + source = "../../services/frontend_mesh_service" + operator = var.operator + environment = var.environment + service = "bfe" + app_mesh_id = module.buyer_app_mesh[0].app_mesh_id + app_mesh_name = module.buyer_app_mesh[0].app_mesh_name + root_domain = var.root_domain + cloud_map_private_dns_namespace_id = module.buyer_app_mesh[0].cloud_map_private_dns_namespace_id + cloud_map_private_dns_namespace_name = module.buyer_app_mesh[0].cloud_map_private_dns_namespace_name + server_instance_role_name = module.iam_roles.instance_role_name + backend_virtual_service_name = module.bidding_mesh_service[0].virtual_service_name + backend_virtual_service_port = var.server_port + backend_virtual_service_private_certificate_arn = (var.use_tls_with_mesh) ? module.bidding_mesh_service[0].acmpca_certificate_authority_arn : "" # This will be empty string for var.use_tls_with_mesh = false anyways. + service_port = var.server_port + backend_service = "bidding" + root_domain_zone_id = var.root_domain_zone_id + healthcheck_interval_sec = var.healthcheck_interval_sec + healthcheck_timeout_sec = var.healthcheck_timeout_sec + healthcheck_healthy_threshold = var.healthcheck_healthy_threshold + healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold + use_tls_with_mesh = var.use_tls_with_mesh +} + +module "load_balancing_bfe" { source = "../../services/load_balancing" environment = var.environment operator = var.operator - service = "bidding" + service = "bfe" certificate_arn = var.certificate_arn elb_subnet_ids = module.networking.public_subnet_ids server_port = var.server_port @@ -162,27 +241,34 @@ module "load_balancing_bidding" { healthcheck_healthy_threshold = var.healthcheck_healthy_threshold healthcheck_interval_sec = var.healthcheck_interval_sec healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold - # Recommended not to change. Ensures internal VPC load balancers for traffic over private network. - internal = true } -module "autoscaling_bidding" { - source = "../../services/autoscaling" - environment = var.environment - operator = var.operator - enclave_debug_mode = var.enclave_debug_mode - service = "bidding" - autoscaling_subnet_ids = module.networking.private_subnet_ids - instance_ami_id = var.bidding_instance_ami_id - instance_security_group_id = module.security_groups.instance_security_group_id - instance_type = var.bidding_instance_type - target_group_arns = module.load_balancing_bidding.target_group_arns - autoscaling_desired_capacity = var.bidding_autoscaling_desired_capacity - autoscaling_max_size = var.bidding_autoscaling_max_size - autoscaling_min_size = var.bidding_autoscaling_min_size - instance_profile_arn = module.iam_roles.instance_profile_arn - enclave_cpu_count = var.bidding_enclave_cpu_count - enclave_memory_mib = var.bidding_enclave_memory_mib +module "autoscaling_bfe" { + source = "../../services/autoscaling" + environment = var.environment + operator = var.operator + enclave_debug_mode = var.enclave_debug_mode + service = "bfe" + autoscaling_subnet_ids = module.networking.private_subnet_ids + instance_ami_id = var.bfe_instance_ami_id + instance_security_group_id = module.security_groups.instance_security_group_id + instance_type = var.bfe_instance_type + target_group_arns = module.load_balancing_bfe.target_group_arns + autoscaling_desired_capacity = var.bfe_autoscaling_desired_capacity + autoscaling_max_size = var.bfe_autoscaling_max_size + autoscaling_min_size = var.bfe_autoscaling_min_size + instance_profile_arn = module.iam_roles.instance_profile_arn + enclave_cpu_count = var.bfe_enclave_cpu_count + enclave_memory_mib = var.bfe_enclave_memory_mib + cloud_map_service_id = var.use_service_mesh ? module.bfe_mesh_service[0].cloud_map_service_id : "" + region = var.region + app_mesh_name = var.use_service_mesh ? module.buyer_app_mesh[0].app_mesh_name : "" + virtual_node_name = var.use_service_mesh ? module.bfe_mesh_service[0].virtual_node_name : "" + healthcheck_interval_sec = var.healthcheck_interval_sec + healthcheck_timeout_sec = var.healthcheck_timeout_sec + healthcheck_healthy_threshold = var.healthcheck_healthy_threshold + healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold + healthcheck_grace_period_sec = var.healthcheck_grace_period_sec } diff --git a/production/deploy/aws/terraform/modules/buyer/service_vars.tf b/production/deploy/aws/terraform/modules/buyer/service_vars.tf index e69e0f82..0aaad56d 100644 --- a/production/deploy/aws/terraform/modules/buyer/service_vars.tf +++ b/production/deploy/aws/terraform/modules/buyer/service_vars.tf @@ -19,8 +19,8 @@ variable "environment" { description = "Assigned environment name to group related resources." type = string validation { - condition = length(var.environment) <= 10 - error_message = "Due to current naming scheme limitations, environment must not be longer than 10." + condition = length(var.environment) <= 3 + error_message = "Due to current naming scheme limitations, environment must not be longer than 3." } } variable "region" { @@ -34,6 +34,16 @@ variable "operator" { type = string } +variable "use_service_mesh" { + description = "use mesh if true, else if false use load balancers" + type = bool +} + +variable "use_tls_with_mesh" { + type = bool + description = "Whether to use TLS-encrypted communication between service mesh envoy sidecars." +} + # Variables related to network, dns and certs configuration. variable "vpc_cidr_block" { description = "CIDR range for the VPC where auction server will be deployed." @@ -57,24 +67,20 @@ variable "certificate_arn" { variable "bfe_instance_type" { description = "Hardware and OS configuration for the BFE EC2 instance." type = string - default = "c6i.2xlarge" # Recommend at least c6i.12xlarge for AdTechs testing or load testing. } variable "bidding_instance_type" { description = "Hardware and OS configuration for the Bidding EC2 instance." type = string - default = "c6i.2xlarge" # Recommend at least c6i.12xlarge for AdTechs testing or load testing. } variable "bfe_instance_ami_id" { description = "Buyer FrontEnd operator Amazon Machine Image to run on EC2 instance." type = string - default = "ami-0ea7735ce85ec9cf5" } variable "bidding_instance_ami_id" { description = "Bidding operator Amazon Machine Image to run on EC2 instance." type = string - default = "ami-0f2f28fc0914f6575" } # Variables related to server configuration. @@ -86,51 +92,41 @@ variable "server_port" { variable "bfe_enclave_cpu_count" { description = "The number of vcpus to allocate to the BFE enclave." type = number - default = 6 } variable "bfe_enclave_memory_mib" { description = "Amount of memory to allocate to the BFE enclave." type = number - default = 12000 } variable "bidding_enclave_cpu_count" { description = "The number of vcpus to allocate to the Bidding enclave." type = number - default = 6 } variable "bidding_enclave_memory_mib" { description = "Amount of memory to allocate to the Bidding enclave." type = number - default = 12000 } variable "bfe_autoscaling_desired_capacity" { - type = number - default = 1 + type = number } variable "bfe_autoscaling_max_size" { - type = number - default = 1 + type = number } variable "bfe_autoscaling_min_size" { - type = number - default = 1 + type = number } variable "bidding_autoscaling_desired_capacity" { - type = number - default = 1 + type = number } variable "bidding_autoscaling_max_size" { - type = number - default = 1 + type = number } variable "bidding_autoscaling_min_size" { - type = number - default = 1 + type = number } # Variables related to AWS backend services @@ -159,6 +155,19 @@ variable "healthcheck_interval_sec" { type = number default = 7 } + +variable "healthcheck_timeout_sec" { + description = "Amount of time to wait for a health check response in seconds." + type = number + default = 5 +} + +variable "healthcheck_grace_period_sec" { + description = "Amount of time to wait for service inside enclave to start up before starting health checks, in seconds." + type = number + default = 60 +} + variable "healthcheck_healthy_threshold" { description = "Consecutive health check successes required to be considered healthy." type = number @@ -193,3 +202,28 @@ variable "runtime_flags" { type = map(string) description = "Buyer runtime flags. Must exactly match flags specified in /services/(bidding_service|buyer_frontend_service)/runtime_flags.h" } + +variable "business_org_for_cert_auth" { + description = "Name of your business organization, for the private certificate authority" + type = string +} + +variable "country_for_cert_auth" { + description = "Country of your business organization, for the private certificate authority" + type = string +} + +variable "state_for_cert_auth" { + description = "State or province where your business organization is located, for the private certificate authority" + type = string +} + +variable "org_unit_for_cert_auth" { + description = "Name of your particular unit in your business organization, for the private certificate authority" + type = string +} + +variable "locality_for_cert_auth" { + description = "Locality where your business organization is located, for the private certificate authority" + type = string +} diff --git a/production/deploy/aws/terraform/modules/seller/service.tf b/production/deploy/aws/terraform/modules/seller/service.tf index 58948eb9..7a35f751 100644 --- a/production/deploy/aws/terraform/modules/seller/service.tf +++ b/production/deploy/aws/terraform/modules/seller/service.tf @@ -178,13 +178,57 @@ resource "aws_lb_listener_rule" "public_alb_listener_http2_rule" { } } +module "seller_app_mesh" { + # Only create if using service mesh + count = var.use_service_mesh ? 1 : 0 + + source = "../../services/app_mesh" + operator = var.operator + environment = var.environment + vpc_id = module.networking.vpc_id +} + ####### Envoy-Specific Resources STOP ####### -module "load_balancing_sfe" { +################ Auction operator Setup ################ + + +module "auction_mesh_service" { + # Only create if using service mesh + count = var.use_service_mesh ? 1 : 0 + + source = "../../services/backend_mesh_service" + operator = var.operator + environment = var.environment + service = "auction" + app_mesh_id = module.seller_app_mesh[0].app_mesh_id + app_mesh_name = module.seller_app_mesh[0].app_mesh_name + root_domain = var.root_domain + cloud_map_private_dns_namespace_id = module.seller_app_mesh[0].cloud_map_private_dns_namespace_id + cloud_map_private_dns_namespace_name = module.seller_app_mesh[0].cloud_map_private_dns_namespace_name + server_instance_role_name = module.iam_roles.instance_role_name + business_org_for_cert_auth = var.business_org_for_cert_auth + country_for_cert_auth = var.country_for_cert_auth + state_for_cert_auth = var.state_for_cert_auth + locality_for_cert_auth = var.locality_for_cert_auth + org_unit_for_cert_auth = var.org_unit_for_cert_auth + service_port = var.server_port + root_domain_zone_id = var.root_domain_zone_id + healthcheck_interval_sec = var.healthcheck_interval_sec + healthcheck_timeout_sec = var.healthcheck_timeout_sec + healthcheck_healthy_threshold = var.healthcheck_healthy_threshold + healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold + use_tls_with_mesh = var.use_tls_with_mesh +} + +module "load_balancing_auction" { + # Only create if not using service mesh + count = var.use_service_mesh ? 0 : 1 + source = "../../services/load_balancing" environment = var.environment operator = var.operator - service = "sfe" + service = "auction" certificate_arn = var.certificate_arn elb_subnet_ids = module.networking.public_subnet_ids server_port = var.server_port @@ -195,36 +239,74 @@ module "load_balancing_sfe" { healthcheck_healthy_threshold = var.healthcheck_healthy_threshold healthcheck_interval_sec = var.healthcheck_interval_sec healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold + # Recommended not to change. Ensures internal VPC load balancers for traffic over private network. + internal = true } -module "autoscaling_sfe" { - source = "../../services/autoscaling" - environment = var.environment - operator = var.operator - enclave_debug_mode = var.enclave_debug_mode - service = "sfe" - autoscaling_subnet_ids = module.networking.private_subnet_ids - instance_ami_id = var.sfe_instance_ami_id - instance_security_group_id = module.security_groups.instance_security_group_id - instance_type = var.sfe_instance_type - target_group_arns = concat(module.load_balancing_sfe.target_group_arns, [aws_lb_target_group.alb_http2_target_group.arn]) - autoscaling_desired_capacity = var.sfe_autoscaling_desired_capacity - autoscaling_max_size = var.sfe_autoscaling_max_size - autoscaling_min_size = var.sfe_autoscaling_min_size - instance_profile_arn = module.iam_roles.instance_profile_arn - enclave_cpu_count = var.sfe_enclave_cpu_count - enclave_memory_mib = var.sfe_enclave_memory_mib +module "autoscaling_auction" { + source = "../../services/autoscaling" + environment = var.environment + operator = var.operator + enclave_debug_mode = var.enclave_debug_mode + service = "auction" + autoscaling_subnet_ids = module.networking.private_subnet_ids + instance_ami_id = var.auction_instance_ami_id + instance_security_group_id = module.security_groups.instance_security_group_id + instance_type = var.auction_instance_type + target_group_arns = var.use_service_mesh ? [] : module.load_balancing_auction[0].target_group_arns + autoscaling_desired_capacity = var.auction_autoscaling_desired_capacity + autoscaling_max_size = var.auction_autoscaling_max_size + autoscaling_min_size = var.auction_autoscaling_min_size + instance_profile_arn = module.iam_roles.instance_profile_arn + enclave_cpu_count = var.auction_enclave_cpu_count + enclave_memory_mib = var.auction_enclave_memory_mib + cloud_map_service_id = var.use_service_mesh ? module.auction_mesh_service[0].cloud_map_service_id : "" + region = var.region + app_mesh_name = var.use_service_mesh ? module.seller_app_mesh[0].app_mesh_name : "" + virtual_node_name = var.use_service_mesh ? module.auction_mesh_service[0].virtual_node_name : "" + healthcheck_interval_sec = var.healthcheck_interval_sec + healthcheck_timeout_sec = var.healthcheck_timeout_sec + healthcheck_healthy_threshold = var.healthcheck_healthy_threshold + healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold + healthcheck_grace_period_sec = var.healthcheck_grace_period_sec } -################ Auction operator Setup ################ - +################ SFE operator Setup ################ + + +module "sfe_mesh_service" { + # Only create if using service mesh + count = var.use_service_mesh ? 1 : 0 + + source = "../../services/frontend_mesh_service" + operator = var.operator + environment = var.environment + service = "sfe" + app_mesh_id = module.seller_app_mesh[0].app_mesh_id + app_mesh_name = module.seller_app_mesh[0].app_mesh_name + root_domain = var.root_domain + cloud_map_private_dns_namespace_id = module.seller_app_mesh[0].cloud_map_private_dns_namespace_id + cloud_map_private_dns_namespace_name = module.seller_app_mesh[0].cloud_map_private_dns_namespace_name + server_instance_role_name = module.iam_roles.instance_role_name + backend_virtual_service_name = module.auction_mesh_service[0].virtual_service_name + backend_virtual_service_port = var.server_port + backend_virtual_service_private_certificate_arn = (var.use_tls_with_mesh) ? module.auction_mesh_service[0].acmpca_certificate_authority_arn : "" # This will be empty string for var.use_tls_with_mesh = false anyways + service_port = var.server_port + backend_service = "auction" + root_domain_zone_id = var.root_domain_zone_id + healthcheck_interval_sec = var.healthcheck_interval_sec + healthcheck_timeout_sec = var.healthcheck_timeout_sec + healthcheck_healthy_threshold = var.healthcheck_healthy_threshold + healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold + use_tls_with_mesh = var.use_tls_with_mesh +} -module "load_balancing_auction" { +module "load_balancing_sfe" { source = "../../services/load_balancing" environment = var.environment operator = var.operator - service = "auction" + service = "sfe" certificate_arn = var.certificate_arn elb_subnet_ids = module.networking.public_subnet_ids server_port = var.server_port @@ -235,27 +317,34 @@ module "load_balancing_auction" { healthcheck_healthy_threshold = var.healthcheck_healthy_threshold healthcheck_interval_sec = var.healthcheck_interval_sec healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold - # Recommended not to change. Ensures internal VPC load balancers for traffic over private network. - internal = true } -module "autoscaling_auction" { - source = "../../services/autoscaling" - environment = var.environment - operator = var.operator - enclave_debug_mode = var.enclave_debug_mode - service = "auction" - autoscaling_subnet_ids = module.networking.private_subnet_ids - instance_ami_id = var.auction_instance_ami_id - instance_security_group_id = module.security_groups.instance_security_group_id - instance_type = var.auction_instance_type - target_group_arns = module.load_balancing_auction.target_group_arns - autoscaling_desired_capacity = var.auction_autoscaling_desired_capacity - autoscaling_max_size = var.auction_autoscaling_max_size - autoscaling_min_size = var.auction_autoscaling_min_size - instance_profile_arn = module.iam_roles.instance_profile_arn - enclave_cpu_count = var.auction_enclave_cpu_count - enclave_memory_mib = var.auction_enclave_memory_mib +module "autoscaling_sfe" { + source = "../../services/autoscaling" + environment = var.environment + operator = var.operator + enclave_debug_mode = var.enclave_debug_mode + service = "sfe" + autoscaling_subnet_ids = module.networking.private_subnet_ids + instance_ami_id = var.sfe_instance_ami_id + instance_security_group_id = module.security_groups.instance_security_group_id + instance_type = var.sfe_instance_type + target_group_arns = concat(module.load_balancing_sfe.target_group_arns, [aws_lb_target_group.alb_http2_target_group.arn]) + autoscaling_desired_capacity = var.sfe_autoscaling_desired_capacity + autoscaling_max_size = var.sfe_autoscaling_max_size + autoscaling_min_size = var.sfe_autoscaling_min_size + instance_profile_arn = module.iam_roles.instance_profile_arn + enclave_cpu_count = var.sfe_enclave_cpu_count + enclave_memory_mib = var.sfe_enclave_memory_mib + cloud_map_service_id = var.use_service_mesh ? module.sfe_mesh_service[0].cloud_map_service_id : "" + region = var.region + app_mesh_name = var.use_service_mesh ? module.seller_app_mesh[0].app_mesh_name : "" + virtual_node_name = var.use_service_mesh ? module.sfe_mesh_service[0].virtual_node_name : "" + healthcheck_interval_sec = var.healthcheck_interval_sec + healthcheck_timeout_sec = var.healthcheck_timeout_sec + healthcheck_healthy_threshold = var.healthcheck_healthy_threshold + healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold + healthcheck_grace_period_sec = var.healthcheck_grace_period_sec } diff --git a/production/deploy/aws/terraform/modules/seller/service_vars.tf b/production/deploy/aws/terraform/modules/seller/service_vars.tf index 5bce1340..81891ac2 100644 --- a/production/deploy/aws/terraform/modules/seller/service_vars.tf +++ b/production/deploy/aws/terraform/modules/seller/service_vars.tf @@ -28,6 +28,11 @@ variable "region" { type = string } +variable "use_service_mesh" { + description = "use mesh if true, else if false use load balancers" + type = bool +} + # TODO(b/260100326): update key to service_operator variable "operator" { description = "operator name" @@ -57,24 +62,20 @@ variable "certificate_arn" { variable "sfe_instance_type" { description = "Hardware and OS configuration for the SFE EC2 instance." type = string - default = "c6i.2xlarge" # Recommend at least c6i.12xlarge for AdTechs testing or load testing. } variable "auction_instance_type" { description = "Hardware and OS configuration for the Auction EC2 instance." type = string - default = "c6i.2xlarge" # Recommend at least c6i.12xlarge for AdTechs testing or load testing. } variable "sfe_instance_ami_id" { description = "Seller FrontEnd operator Amazon Machine Image to run on EC2 instance." type = string - default = "ami-0ff8ad2fa8512a078" } variable "auction_instance_ami_id" { description = "Auction operator Amazon Machine Image to run on EC2 instance." type = string - default = "ami-0ea85f493f16aba3c" } # Variables related to server configuration. @@ -94,54 +95,44 @@ variable "server_port" { variable "sfe_enclave_cpu_count" { description = "The number of vcpus to allocate to the SFE enclave." type = number - default = 6 } variable "sfe_enclave_memory_mib" { description = "Amount of memory to allocate to the SFE enclave." type = number - default = 12000 } variable "auction_enclave_cpu_count" { description = "The number of vcpus to allocate to the Auction enclave." type = number - default = 6 } variable "auction_enclave_memory_mib" { description = "Amount of memory to allocate to the Auction enclave." type = number - default = 12000 } variable "sfe_autoscaling_desired_capacity" { - type = number - default = 1 + type = number } variable "sfe_autoscaling_max_size" { - type = number - default = 1 + type = number } variable "sfe_autoscaling_min_size" { - type = number - default = 1 + type = number } variable "auction_autoscaling_desired_capacity" { - type = number - default = 1 + type = number } variable "auction_autoscaling_max_size" { - type = number - default = 1 + type = number } variable "auction_autoscaling_min_size" { - type = number - default = 1 + type = number } @@ -177,6 +168,16 @@ variable "healthcheck_interval_sec" { type = number default = 7 } +variable "healthcheck_timeout_sec" { + description = "Amount of time to wait for a health check response in seconds." + type = number + default = 5 +} +variable "healthcheck_grace_period_sec" { + description = "Amount of time to wait for service inside enclave to start up before starting health checks, in seconds." + type = number + default = 60 +} variable "healthcheck_healthy_threshold" { description = "Consecutive health check successes required to be considered healthy." type = number @@ -211,3 +212,33 @@ variable "runtime_flags" { type = map(string) description = "Seller runtime flags. Must exactly match flags specified in /services/(auction_service|seller_frontend_service)/runtime_flags.h" } + +variable "business_org_for_cert_auth" { + description = "Name of your business organization, for the private certificate authority" + type = string +} + +variable "country_for_cert_auth" { + description = "Country of your business organization, for the private certificate authority" + type = string +} + +variable "state_for_cert_auth" { + description = "State or province where your business organization is located, for the private certificate authority" + type = string +} + +variable "org_unit_for_cert_auth" { + description = "Name of your particular unit in your business organization, for the private certificate authority" + type = string +} + +variable "locality_for_cert_auth" { + description = "Locality where your business organization is located, for the private certificate authority" + type = string +} + +variable "use_tls_with_mesh" { + type = bool + description = "Whether to use TLS-encrypted communication between service mesh envoy sidecars." +} diff --git a/production/deploy/aws/terraform/services/app_mesh/main.tf b/production/deploy/aws/terraform/services/app_mesh/main.tf new file mode 100644 index 00000000..4bff2f96 --- /dev/null +++ b/production/deploy/aws/terraform/services/app_mesh/main.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "aws_appmesh_mesh" "app_mesh" { + name = "${var.operator}-${var.environment}-app-mesh" +} + +resource "aws_service_discovery_private_dns_namespace" "cloud_map_private_dns_namespace" { + name = "${var.operator}-${var.environment}-cloud-map-private-dns-namespace" + vpc = var.vpc_id +} diff --git a/production/deploy/aws/terraform/services/app_mesh/outputs.tf b/production/deploy/aws/terraform/services/app_mesh/outputs.tf new file mode 100644 index 00000000..2078dc8d --- /dev/null +++ b/production/deploy/aws/terraform/services/app_mesh/outputs.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +output "app_mesh_name" { + description = "The name of the app mesh." + value = aws_appmesh_mesh.app_mesh.name +} + +output "app_mesh_id" { + description = "The ID of the app mesh." + value = aws_appmesh_mesh.app_mesh.id +} + +output "cloud_map_private_dns_namespace_id" { + description = "ID of the cloud map namespace" + value = aws_service_discovery_private_dns_namespace.cloud_map_private_dns_namespace.id +} + +output "cloud_map_private_dns_namespace_name" { + description = "Name of the cloud map namespace" + value = aws_service_discovery_private_dns_namespace.cloud_map_private_dns_namespace.name +} diff --git a/production/deploy/aws/terraform/services/app_mesh/variables.tf b/production/deploy/aws/terraform/services/app_mesh/variables.tf new file mode 100644 index 00000000..24d02da9 --- /dev/null +++ b/production/deploy/aws/terraform/services/app_mesh/variables.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "operator" { + description = "Assigned name of an operator in Bidding & Auction system, i.e. seller1, buyer1, buyer2." + type = string +} + +variable "environment" { + description = "Assigned environment name to group related resources." + type = string +} + +variable "vpc_id" { + description = "ID of the VPC in which we will make the DNS namespace of this cloud map" + type = string +} diff --git a/production/deploy/aws/terraform/services/autoscaling/instance_init_script.tftpl b/production/deploy/aws/terraform/services/autoscaling/instance_init_script.tftpl index b30db2ee..70c148f5 100644 --- a/production/deploy/aws/terraform/services/autoscaling/instance_init_script.tftpl +++ b/production/deploy/aws/terraform/services/autoscaling/instance_init_script.tftpl @@ -20,11 +20,64 @@ docker run --rm \ # load balancer health checks will eventually kill this instance. curl --retry 6 --retry-all-errors localhost:9902/ready +if [[ -n "${app_mesh_name}" && -n "${virtual_node_name}" ]]; then + # Authenticate with the Envoy Amazon ECR repository in the Region that you want + # your Docker client to pull the image from. + aws ecr get-login-password \ + --region ${region} | + docker login \ + --username AWS \ + --password-stdin 840364872350.dkr.ecr.${region}.amazonaws.com + + # Start the App Mesh Envoy container. + sudo docker run --detach --env APPMESH_RESOURCE_ARN=mesh/${app_mesh_name}/virtualNode/${virtual_node_name} \ + -v /tmp:/tmp \ + -u 1337 --network host public.ecr.aws/appmesh/aws-appmesh-envoy:v1.29.4.0-prod +fi + +# Install grpcurl +cd /tmp +wget -q https://github.com/fullstorydev/grpcurl/releases/download/v1.9.1/grpcurl_1.9.1_linux_amd64.rpm +sudo rpm -i grpcurl_1.9.1_linux_amd64.rpm +cd - + +# Grab the metadata needed for registering instance. +# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html +IP_ADDRESS=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) +INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) + +if [[ -n "${cloud_map_service_id}" && -n "${region}" ]]; then + # Actually register the present EC2 with the cloud map. + register_instance_out=$(aws servicediscovery register-instance \ + --service-id ${cloud_map_service_id} \ + --instance-id $INSTANCE_ID \ + --attributes "AWS_INSTANCE_IPV4="$IP_ADDRESS \ + --region ${region} 2>&1) + while [[ "$?" -gt 0 ]] && [[ "$register_instance_out" =~ "not authorized to perform" ]]; do + echo "Registering service instance failed ... This can be transient and thus trying again in 2 seconds" + echo "Observed failure: $register_instance_out" + sleep 2 + register_instance_out=$(aws servicediscovery register-instance \ + --service-id ${cloud_map_service_id} \ + --instance-id $INSTANCE_ID \ + --attributes "AWS_INSTANCE_IPV4="$IP_ADDRESS \ + --region ${region} 2>&1) + done +fi + +if [[ -n "${app_mesh_name}" && -n "${virtual_node_name}" && -n "${cloud_map_service_id}" && -n "${region}" ]]; then + echo "Will wait for service mesh envoy proxy to come up" + while [ "$(curl localhost:9901/ready)" != "LIVE" ]; do + echo "Service mesh envoy proxy is not ready.. will check again in 1 second" + sleep 1 + done +fi + # Make sure nitro enclave allocator service is stopped systemctl stop nitro-enclaves-allocator.service # Allocate resources available to enclave -declare -r -x ALLOCATOR_YAML=/etc/nitro_enclaves/allocator.yaml +declare -r -x ALLOCATOR_YAML=/etc/nitro_enclaves/allocator.yaml cat >"$${ALLOCATOR_YAML}" < "/amazon-cloudwatch-agent.json" + cat <"/amazon-cloudwatch-agent.json" { "agent": { "metrics_collection_interval": 10 @@ -62,7 +115,7 @@ if [[ "${enclave_debug_mode}" == "true" ]]; then { "file_path": "${enclave_log_path}", "log_group_name": "${service}-tee-logs", - "log_stream_name": "${environment}-{instance_id}", + "log_stream_name": "${environment}-$INSTANCE_ID", "timestamp_format": "%Y-%m-%d %H:%M:%S" } ] @@ -80,10 +133,34 @@ EOF nitro-cli run-enclave \ --cpu-count ${enclave_cpu_count} --memory ${enclave_memory_mib} \ --eif-path /opt/privacysandbox/server_enclave_image.eif \ - --enclave-cid 16 --attach-console > ${enclave_log_path} + --enclave-cid 16 --attach-console >${enclave_log_path} & else nitro-cli run-enclave \ --cpu-count ${enclave_cpu_count} --memory ${enclave_memory_mib} \ --eif-path /opt/privacysandbox/server_enclave_image.eif \ - --enclave-cid 16 + --enclave-cid 16 & +fi + +SECONDS_TRIED=0 + +echo "Will wait for ${healthcheck_grace_period_sec} seconds for the service to come up" +while ! grpcurl --plaintext localhost:50051 list; do + echo "Service/Vsock proxy is not reachable.. will retry in 1 second" + ((SECONDS_TRIED++)) + if (( SECONDS_TRIED > ${healthcheck_grace_period_sec} )) + then + echo "Timing out: tried for ${healthcheck_grace_period_sec} seconds and the service and its vsock proxy are still not reachable." + break + fi + sleep 1 +done + +if [[ -n "${app_mesh_name}" && -n "${virtual_node_name}" && -n "${cloud_map_service_id}" && -n "${region}" ]]; then + bash /opt/privacysandbox/hc.bash -p /opt/privacysandbox -n health.proto -a localhost:50051 -i ${healthcheck_interval_sec} -t ${healthcheck_timeout_sec} -h ${healthcheck_healthy_threshold} -u ${healthcheck_unhealthy_threshold} -e $INSTANCE_ID -g 0 -r ${region} -s ${cloud_map_service_id} & + + # Run Cloud Un-Map to catch and de-register any instances shut down by EC2 ASG HCs, or which otherwise did not de-register themselves. + sudo docker run --detach spreaker/aws-cloud-unmap --service-id ${cloud_map_service_id} --service-region ${region} --instances-region ${region} + + echo "Setting up iptables to route traffic via service mesh / envoy" + sudo bash -x /opt/privacysandbox/envoy_networking.sh -s ${service} fi diff --git a/production/deploy/aws/terraform/services/autoscaling/main.tf b/production/deploy/aws/terraform/services/autoscaling/main.tf index 74fbdbd9..755b1f01 100644 --- a/production/deploy/aws/terraform/services/autoscaling/main.tf +++ b/production/deploy/aws/terraform/services/autoscaling/main.tf @@ -34,12 +34,21 @@ resource "aws_launch_template" "instance_launch_template" { user_data = base64encode(templatefile( "${path.module}/instance_init_script.tftpl", { - enclave_memory_mib = var.enclave_memory_mib, - enclave_cpu_count = var.enclave_cpu_count, - service = var.service, - environment = var.environment, - enclave_log_path = var.enclave_log_path, - enclave_debug_mode = var.enclave_debug_mode + enclave_memory_mib = var.enclave_memory_mib, + enclave_cpu_count = var.enclave_cpu_count, + service = var.service, + environment = var.environment, + cloud_map_service_id = var.cloud_map_service_id, + enclave_log_path = var.enclave_log_path, + enclave_debug_mode = var.enclave_debug_mode + region = var.region + app_mesh_name = var.app_mesh_name + virtual_node_name = var.virtual_node_name + healthcheck_interval_sec = var.healthcheck_interval_sec + healthcheck_timeout_sec = var.healthcheck_timeout_sec + healthcheck_healthy_threshold = var.healthcheck_healthy_threshold + healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold + healthcheck_grace_period_sec = var.healthcheck_grace_period_sec })) metadata_options { diff --git a/production/deploy/aws/terraform/services/autoscaling/variables.tf b/production/deploy/aws/terraform/services/autoscaling/variables.tf index c765af79..d23c5b96 100644 --- a/production/deploy/aws/terraform/services/autoscaling/variables.tf +++ b/production/deploy/aws/terraform/services/autoscaling/variables.tf @@ -73,7 +73,7 @@ variable "enclave_cpu_count" { } variable "service" { - description = "One of: bidding, auction, buyer-frontend, seller-frontend" + description = "One of: bidding, auction, bfe, sfe" type = string } @@ -88,3 +88,48 @@ variable "enclave_log_path" { type = string default = "/output.log" } + +variable "cloud_map_service_id" { + description = "The ID of the service discovery service" + type = string +} + +variable "region" { + description = "AWS region in which services have been created" + type = string +} + +variable "app_mesh_name" { + description = "Name of the AWS App Mesh in which this service will communicate." + type = string +} + +variable "virtual_node_name" { + description = "Name of the App Mesh Virtual Node of which instance in this ASG will be a part." + type = string +} + +variable "healthcheck_interval_sec" { + description = "Amount of time between health check intervals in seconds." + type = number +} + +variable "healthcheck_timeout_sec" { + description = "Amount of time to wait for a health check response in seconds." + type = number +} + +variable "healthcheck_healthy_threshold" { + description = "Consecutive health check successes required to be considered healthy." + type = number +} + +variable "healthcheck_unhealthy_threshold" { + description = "Consecutive health check failures required to be considered unhealthy." + type = number +} + +variable "healthcheck_grace_period_sec" { + description = "Amount of time to wait for service inside enclave to start up before starting health checks, in seconds." + type = number +} diff --git a/production/deploy/aws/terraform/services/backend_mesh_service/main.tf b/production/deploy/aws/terraform/services/backend_mesh_service/main.tf new file mode 100644 index 00000000..c24338d5 --- /dev/null +++ b/production/deploy/aws/terraform/services/backend_mesh_service/main.tf @@ -0,0 +1,233 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "aws_service_discovery_service" "cloud_map_service" { + name = "${var.service}-${var.operator}-${var.environment}-cloud-map-service.${var.root_domain}" + + dns_config { + namespace_id = var.cloud_map_private_dns_namespace_id + + dns_records { + ttl = 10 + type = "A" + } + } + + health_check_custom_config { + failure_threshold = 1 + } +} + +data "aws_partition" "current" { + # Only create if using TLS with service mesh + count = var.use_tls_with_mesh ? 1 : 0 +} + +// Create a root certificate authority (CA) in ACM. +resource "aws_acmpca_certificate_authority" "acmpca_certificate_authority" { + # Only create if using TLS with service mesh + count = var.use_tls_with_mesh ? 1 : 0 + + type = "ROOT" + + certificate_authority_configuration { + key_algorithm = "RSA_2048" + signing_algorithm = "SHA256WITHRSA" + + subject { + common_name = var.root_domain + organization = var.business_org_for_cert_auth + country = var.country_for_cert_auth + state = var.state_for_cert_auth + organizational_unit = var.org_unit_for_cert_auth + locality = var.locality_for_cert_auth + } + } +} + +// Self-sign the root CA. +resource "aws_acmpca_certificate" "acmpca_certificate" { + # Only create if using TLS with service mesh + count = var.use_tls_with_mesh ? 1 : 0 + + certificate_authority_arn = aws_acmpca_certificate_authority.acmpca_certificate_authority[0].arn + certificate_signing_request = aws_acmpca_certificate_authority.acmpca_certificate_authority[0].certificate_signing_request + signing_algorithm = "SHA256WITHRSA" + + template_arn = "arn:${data.aws_partition.current[0].partition}:acm-pca:::template/RootCACertificate/V1" + + validity { + type = "YEARS" + value = 10 + } +} + +// Import the signed certiticate as the root CA. +resource "aws_acmpca_certificate_authority_certificate" "acmpca_certificate_authority_certificate" { + # Only create if using TLS with service mesh + count = var.use_tls_with_mesh ? 1 : 0 + + certificate_authority_arn = aws_acmpca_certificate_authority.acmpca_certificate_authority[0].arn + + certificate = aws_acmpca_certificate.acmpca_certificate[0].certificate + certificate_chain = aws_acmpca_certificate.acmpca_certificate[0].certificate_chain +} + +// Allow the ACM service to automatically renew certificates issued by a PCA +resource "aws_acmpca_permission" "acmpca_permission" { + # Only create if using TLS with service mesh + count = var.use_tls_with_mesh ? 1 : 0 + + certificate_authority_arn = aws_acmpca_certificate_authority.acmpca_certificate_authority[0].arn + actions = ["IssueCertificate", "GetCertificate", "ListPermissions"] + principal = "acm.amazonaws.com" +} + +resource "aws_acm_certificate" "acm_certificate" { + # Only create if using TLS with service mesh + count = var.use_tls_with_mesh ? 1 : 0 + + certificate_authority_arn = aws_acmpca_certificate_authority.acmpca_certificate_authority[0].arn + + domain_name = "*.${var.root_domain}" + subject_alternative_names = [ + "*.${var.root_domain}", + "*.${var.root_domain}.${var.cloud_map_private_dns_namespace_name}" + ] + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_appmesh_virtual_node" "appmesh_virtual_node_with_tls" { + # Only create if using TLS with service mesh + count = var.use_tls_with_mesh ? 1 : 0 + + name = "${var.service}-${var.operator}-${var.environment}-appmesh-virtual-node" + mesh_name = var.app_mesh_id + spec { + listener { + port_mapping { + port = var.service_port + protocol = "grpc" + } + + tls { + certificate { + acm { + certificate_arn = aws_acm_certificate.acm_certificate[0].arn + } + } + mode = "PERMISSIVE" + } + + health_check { + protocol = "grpc" + healthy_threshold = var.healthcheck_healthy_threshold + unhealthy_threshold = var.healthcheck_unhealthy_threshold + timeout_millis = var.healthcheck_timeout_sec * 1000 + interval_millis = var.healthcheck_interval_sec * 1000 + } + } + + service_discovery { + aws_cloud_map { + service_name = aws_service_discovery_service.cloud_map_service.name + namespace_name = var.cloud_map_private_dns_namespace_name + } + } + } +} + +resource "aws_appmesh_virtual_node" "appmesh_virtual_node_sans_tls" { + # Only create if NOT using TLS with service mesh + count = var.use_tls_with_mesh ? 0 : 1 + + name = "${var.service}-${var.operator}-${var.environment}-appmesh-virtual-node" + mesh_name = var.app_mesh_id + spec { + listener { + port_mapping { + port = var.service_port + protocol = "grpc" + } + + health_check { + protocol = "grpc" + healthy_threshold = var.healthcheck_healthy_threshold + unhealthy_threshold = var.healthcheck_unhealthy_threshold + timeout_millis = var.healthcheck_timeout_sec * 1000 + interval_millis = var.healthcheck_interval_sec * 1000 + } + } + + service_discovery { + aws_cloud_map { + service_name = aws_service_discovery_service.cloud_map_service.name + namespace_name = var.cloud_map_private_dns_namespace_name + } + } + } +} + +resource "aws_appmesh_virtual_service" "appmesh_virtual_service" { + name = "${var.service}-${var.operator}-${var.environment}-appmesh-virtual-service.${var.root_domain}" + mesh_name = var.app_mesh_name + spec { + provider { + virtual_node { + virtual_node_name = (var.use_tls_with_mesh) ? aws_appmesh_virtual_node.appmesh_virtual_node_with_tls[0].name : aws_appmesh_virtual_node.appmesh_virtual_node_sans_tls[0].name + } + } + } +} + +resource "aws_route53_record" "mesh_node_record" { + name = aws_appmesh_virtual_service.appmesh_virtual_service.name + type = "A" + // In seconds + ttl = 300 + zone_id = var.root_domain_zone_id + // Any non-loopback IP will do, this record just needs to exist, not go anywhere (should be overrifed by appmesh). + records = ["10.10.10.10"] +} + +data "aws_iam_policy_document" "virtual_node_policy_document" { + statement { + actions = [ + "appmesh:StreamAggregatedResources" + ] + resources = [ + (var.use_tls_with_mesh) ? aws_appmesh_virtual_node.appmesh_virtual_node_with_tls[0].arn : aws_appmesh_virtual_node.appmesh_virtual_node_sans_tls[0].arn + ] + } +} + +resource "aws_iam_policy" "app_mesh_node_policy" { + name = format("%s-%s-%s-virtualNodePolicy", var.service, var.operator, var.environment) + policy = data.aws_iam_policy_document.virtual_node_policy_document.json +} + +resource "aws_iam_role_policy_attachment" "app_mesh_node_policy_to_ec2_attachment" { + role = var.server_instance_role_name + policy_arn = aws_iam_policy.app_mesh_node_policy.arn +} + +resource "aws_iam_role_policy_attachment" "amazon_ec2_container_registry_read_only_to_ec2_attachment" { + role = var.server_instance_role_name + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" +} diff --git a/production/deploy/aws/terraform/services/backend_mesh_service/outputs.tf b/production/deploy/aws/terraform/services/backend_mesh_service/outputs.tf new file mode 100644 index 00000000..f24f3193 --- /dev/null +++ b/production/deploy/aws/terraform/services/backend_mesh_service/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_map_service_id" { + description = "The ID of the service discovery service" + value = aws_service_discovery_service.cloud_map_service.id +} + +output "virtual_node_name" { + description = "The name of the virtual node." + value = (var.use_tls_with_mesh) ? aws_appmesh_virtual_node.appmesh_virtual_node_with_tls[0].name : aws_appmesh_virtual_node.appmesh_virtual_node_sans_tls[0].name +} + +output "virtual_service_name" { + description = "The name of the virtual service." + value = aws_appmesh_virtual_service.appmesh_virtual_service.name +} + +output "acmpca_certificate_authority_arn" { + description = "ARN of the AWS Certificate manager private certificate authority created here." + value = var.use_tls_with_mesh ? aws_acmpca_certificate_authority.acmpca_certificate_authority[0].arn : "" +} + +output "cloud_map_service_name" { + description = "The name of the service discovery service" + value = aws_service_discovery_service.cloud_map_service.name +} diff --git a/production/deploy/aws/terraform/services/backend_mesh_service/variables.tf b/production/deploy/aws/terraform/services/backend_mesh_service/variables.tf new file mode 100644 index 00000000..6a9f798e --- /dev/null +++ b/production/deploy/aws/terraform/services/backend_mesh_service/variables.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "operator" { + description = "Assigned name of an operator in Bidding & Auction system, i.e. seller1, buyer1, buyer2." + type = string +} + +variable "environment" { + description = "Assigned environment name to group related resources." + type = string +} + +variable "service" { + description = "One of: bidding, auction, bfe, sfe" + type = string +} + +variable "app_mesh_id" { + description = "ID of the app mesh to which we are adding this cloud map" + type = string +} + +variable "app_mesh_name" { + description = "name of the app mesh to which we are adding this cloud map" + type = string +} + +variable "cloud_map_private_dns_namespace_id" { + description = "ID of the cloud map private DNS namespace in which we are making this cloud map" + type = string +} + +variable "cloud_map_private_dns_namespace_name" { + description = "Name of the cloud map private DNS namespace in which we are making this cloud map" + type = string +} + +variable "root_domain" { + description = "Root domain for APIs." + type = string +} + +variable "server_instance_role_name" { + description = "Role for server EC2 instance profile." + type = string +} + +variable "business_org_for_cert_auth" { + description = "Name of your business organization, for the private certificate authority" + type = string +} + +variable "country_for_cert_auth" { + description = "Country of your business organization, for the private certificate authority" + type = string +} + +variable "state_for_cert_auth" { + description = "State or province where your business organization is located, for the private certificate authority" + type = string +} + +variable "org_unit_for_cert_auth" { + description = "Name of your particular unit in your business organization, for the private certificate authority" + type = string +} + +variable "locality_for_cert_auth" { + description = "Locality where your business organization is located, for the private certificate authority" + type = string +} + +variable "service_port" { + description = "Port on which this service recieves outbound traffic" + type = number +} + +variable "root_domain_zone_id" { + description = "Zone id for the root domain." + type = string +} + +variable "healthcheck_interval_sec" { + description = "Amount of time between health check intervals in seconds." + type = number +} + +variable "healthcheck_timeout_sec" { + description = "Amount of time to wait for a health check response in seconds." + type = number +} + +variable "healthcheck_healthy_threshold" { + description = "Consecutive health check successes required to be considered healthy." + type = number +} + +variable "healthcheck_unhealthy_threshold" { + description = "Consecutive health check failures required to be considered unhealthy." + type = number +} + +variable "use_tls_with_mesh" { + type = bool + description = "Whether to use TLS-encrypted communication between service mesh envoy sidecars." +} diff --git a/production/deploy/aws/terraform/services/frontend_mesh_service/main.tf b/production/deploy/aws/terraform/services/frontend_mesh_service/main.tf new file mode 100644 index 00000000..c472da30 --- /dev/null +++ b/production/deploy/aws/terraform/services/frontend_mesh_service/main.tf @@ -0,0 +1,177 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "aws_service_discovery_service" "cloud_map_service" { + name = "${var.service}-${var.operator}-${var.environment}-cloud-map-service.${var.root_domain}" + + dns_config { + namespace_id = var.cloud_map_private_dns_namespace_id + + dns_records { + ttl = 10 + type = "A" + } + } + + health_check_custom_config { + failure_threshold = 1 + } +} + +resource "aws_appmesh_virtual_node" "appmesh_virtual_node_with_tls" { + # Only create this version of the node if using TLS. + count = (var.use_tls_with_mesh) ? 1 : 0 + + name = "${var.service}-${var.operator}-${var.environment}-appmesh-virtual-node" + mesh_name = var.app_mesh_id + spec { + backend { + virtual_service { + virtual_service_name = var.backend_virtual_service_name + } + } + + backend_defaults { + client_policy { + tls { + enforce = true + validation { + trust { + acm { + certificate_authority_arns = [var.backend_virtual_service_private_certificate_arn] + } + } + subject_alternative_names { + match { + exact = [ + "${var.service}-${var.operator}-${var.environment}-appmesh-virtual-service.${var.root_domain}.${var.cloud_map_private_dns_namespace_name}", + "${var.backend_service}-${var.operator}-${var.environment}-appmesh-virtual-service.${var.root_domain}.${var.cloud_map_private_dns_namespace_name}" + ] + } + } + } + } + } + } + + listener { + port_mapping { + port = 443 + protocol = "grpc" + } + + health_check { + protocol = "grpc" + healthy_threshold = var.healthcheck_healthy_threshold + unhealthy_threshold = var.healthcheck_unhealthy_threshold + timeout_millis = var.healthcheck_timeout_sec * 1000 + interval_millis = var.healthcheck_interval_sec * 1000 + } + } + + service_discovery { + aws_cloud_map { + service_name = aws_service_discovery_service.cloud_map_service.name + namespace_name = var.cloud_map_private_dns_namespace_name + } + } + } +} + +resource "aws_appmesh_virtual_node" "appmesh_virtual_node_sans_tls" { + # Only create this version of the node if NOT using TLS. + count = (var.use_tls_with_mesh) ? 0 : 1 + + name = "${var.service}-${var.operator}-${var.environment}-appmesh-virtual-node" + mesh_name = var.app_mesh_id + spec { + backend { + virtual_service { + virtual_service_name = var.backend_virtual_service_name + } + } + + listener { + port_mapping { + port = 443 + protocol = "grpc" + } + + health_check { + protocol = "grpc" + healthy_threshold = var.healthcheck_healthy_threshold + unhealthy_threshold = var.healthcheck_unhealthy_threshold + timeout_millis = var.healthcheck_timeout_sec * 1000 + interval_millis = var.healthcheck_interval_sec * 1000 + } + } + + service_discovery { + aws_cloud_map { + service_name = aws_service_discovery_service.cloud_map_service.name + namespace_name = var.cloud_map_private_dns_namespace_name + } + } + } +} + +resource "aws_appmesh_virtual_service" "appmesh_virtual_service" { + name = "${var.service}-${var.operator}-${var.environment}-appmesh-virtual-service.${var.root_domain}" + mesh_name = var.app_mesh_name + spec { + provider { + virtual_node { + virtual_node_name = (var.use_tls_with_mesh) ? aws_appmesh_virtual_node.appmesh_virtual_node_with_tls[0].name : aws_appmesh_virtual_node.appmesh_virtual_node_sans_tls[0].name + } + } + } +} + +resource "aws_route53_record" "mesh_node_record" { + name = aws_appmesh_virtual_service.appmesh_virtual_service.name + type = "A" + // In seconds + ttl = 300 + zone_id = var.root_domain_zone_id + // Any non-loopback IP will do, this record just needs to exist, not go anywhere (should be overrifed by appmesh). + records = ["10.10.10.10"] +} + +data "aws_iam_policy_document" "virtual_node_policy_document" { + statement { + actions = [ + "appmesh:StreamAggregatedResources" + ] + resources = [ + (var.use_tls_with_mesh) ? aws_appmesh_virtual_node.appmesh_virtual_node_with_tls[0].arn : aws_appmesh_virtual_node.appmesh_virtual_node_sans_tls[0].arn + ] + } +} + +resource "aws_iam_policy" "app_mesh_node_policy" { + name = format("%s-%s-%s-virtualNodePolicy", var.service, var.operator, var.environment) + policy = data.aws_iam_policy_document.virtual_node_policy_document.json +} + +resource "aws_iam_role_policy_attachment" "app_mesh_node_policy_to_ec2_attachment" { + role = var.server_instance_role_name + policy_arn = aws_iam_policy.app_mesh_node_policy.arn +} + +resource "aws_iam_role_policy_attachment" "amazon_ec2_container_registry_read_only_to_ec2_attachment" { + role = var.server_instance_role_name + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" +} diff --git a/production/deploy/aws/terraform/services/frontend_mesh_service/outputs.tf b/production/deploy/aws/terraform/services/frontend_mesh_service/outputs.tf new file mode 100644 index 00000000..0813f3d9 --- /dev/null +++ b/production/deploy/aws/terraform/services/frontend_mesh_service/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_map_service_id" { + description = "The ID of the service discovery service" + value = aws_service_discovery_service.cloud_map_service.id +} + +output "virtual_node_name" { + description = "The name of the virtual node." + value = (var.use_tls_with_mesh) ? aws_appmesh_virtual_node.appmesh_virtual_node_with_tls[0].name : aws_appmesh_virtual_node.appmesh_virtual_node_sans_tls[0].name +} + +output "virtual_service_name" { + description = "The name of the virtual service." + value = aws_appmesh_virtual_service.appmesh_virtual_service.name +} diff --git a/production/deploy/aws/terraform/services/frontend_mesh_service/variables.tf b/production/deploy/aws/terraform/services/frontend_mesh_service/variables.tf new file mode 100644 index 00000000..21e51cdf --- /dev/null +++ b/production/deploy/aws/terraform/services/frontend_mesh_service/variables.tf @@ -0,0 +1,115 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "operator" { + description = "Assigned name of an operator in Bidding & Auction system, i.e. seller1, buyer1, buyer2." + type = string +} + +variable "environment" { + description = "Assigned environment name to group related resources." + type = string +} + +variable "service" { + description = "One of: bidding, auction, bfe, sfe" + type = string +} + +variable "app_mesh_id" { + description = "ID of the app mesh to which we are adding this cloud map" + type = string +} + +variable "app_mesh_name" { + description = "name of the app mesh to which we are adding this cloud map" + type = string +} + +variable "cloud_map_private_dns_namespace_id" { + description = "ID of the cloud map private DNS namespace in which we are making this cloud map" + type = string +} + +variable "cloud_map_private_dns_namespace_name" { + description = "Name of the cloud map private DNS namespace in which we are making this cloud map" + type = string +} + +variable "root_domain" { + description = "Root domain for APIs." + type = string +} + +variable "server_instance_role_name" { + description = "Role for server EC2 instance profile." + type = string +} + +variable "backend_virtual_service_name" { + description = "Name of the AppMesh virtual service to which this service sends outbound traffic" + type = string +} + +variable "backend_virtual_service_port" { + description = "Port of the AppMesh virtual service to which this service sends outbound traffic" + type = number +} + +variable "backend_virtual_service_private_certificate_arn" { + description = "ARN of the PCA cert of the AppMesh virtual service to which this service sends outbound traffic" + type = string +} + +variable "service_port" { + description = "Port on which this service recieves outbound traffic" + type = number +} + +variable "backend_service" { + description = "name of backend service to which this service reaches out. For example, for BFE, this should be 'bidding', or for SFE this should be 'auction'" + type = string +} + +variable "root_domain_zone_id" { + description = "Zone id for the root domain." + type = string +} + +variable "healthcheck_interval_sec" { + description = "Amount of time between health check intervals in seconds." + type = number +} + +variable "healthcheck_timeout_sec" { + description = "Amount of time to wait for a health check response in seconds." + type = number +} + +variable "healthcheck_healthy_threshold" { + description = "Consecutive health check successes required to be considered healthy." + type = number +} + +variable "healthcheck_unhealthy_threshold" { + description = "Consecutive health check failures required to be considered unhealthy." + type = number +} + +variable "use_tls_with_mesh" { + type = bool + description = "Whether to use TLS-encrypted communication between service mesh envoy sidecars." +} diff --git a/production/deploy/aws/terraform/services/iam_role_policies/main.tf b/production/deploy/aws/terraform/services/iam_role_policies/main.tf index 048cfcf3..e19ee74c 100644 --- a/production/deploy/aws/terraform/services/iam_role_policies/main.tf +++ b/production/deploy/aws/terraform/services/iam_role_policies/main.tf @@ -83,7 +83,14 @@ data "aws_iam_policy_document" "instance_policy_doc" { resources = ["*"] } statement { + sid = "AllowInstancesToRegisterInstance" actions = [ + "servicediscovery:RegisterInstance", + "route53:CreateHealthCheck", + "route53:GetHealthCheck", + "route53:UpdateHealthCheck", + "route53:ChangeResourceRecordSets", + "ec2:DescribeInstances", "s3:ListBucket", "s3:GetBucketLocation", "s3:GetObject", @@ -91,6 +98,46 @@ data "aws_iam_policy_document" "instance_policy_doc" { effect = "Allow" resources = ["*"] } + statement { + sid = "AllowInstancesToSetInstanceHealthForASGandCloudMap" + actions = [ + "autoscaling:SetInstanceHealth", + "servicediscovery:UpdateInstanceCustomHealthStatus", + "servicediscovery:DeregisterInstance", + ] + effect = "Allow" + resources = ["*"] + } + statement { + sid = "AllowTLSAuth" + actions = [ + "acm:DescribeCertificate", + "acm-pca:DescribeCertificateAuthority", + "acm:ExportCertificate", + "acm-pca:GetCertificateAuthorityCertificate" + ] + effect = "Allow" + resources = ["*"] + } + statement { + sid = "RunCloudUnMap" + effect = "Allow" + actions = [ + "ec2:DescribeInstances", + "servicediscovery:ListInstances", + "servicediscovery:DeregisterInstance", + "route53:GetHealthCheck", + "route53:DeleteHealthCheck", + "route53:UpdateHealthCheck", + ] + resources = ["*"] + } + statement { + sid = "UpdateDnsWhileDeregisteringServiceInstancesForCloudUnMap" + effect = "Allow" + actions = ["route53:ChangeResourceRecordSets"] + resources = ["*"] + } } resource "aws_iam_policy" "instance_policy" { diff --git a/production/deploy/aws/terraform/services/load_balancing/variables.tf b/production/deploy/aws/terraform/services/load_balancing/variables.tf index da02167c..97469061 100644 --- a/production/deploy/aws/terraform/services/load_balancing/variables.tf +++ b/production/deploy/aws/terraform/services/load_balancing/variables.tf @@ -96,6 +96,6 @@ variable "healthcheck_unhealthy_threshold" { } variable "service" { - description = "One of: bidding, auction, buyer-frontend, seller-frontend" + description = "One of: bidding, auction, bfe, sfe" type = string } diff --git a/production/deploy/aws/terraform/services/security_group_rules/main.tf b/production/deploy/aws/terraform/services/security_group_rules/main.tf index 5073d7c9..e8f174e3 100644 --- a/production/deploy/aws/terraform/services/security_group_rules/main.tf +++ b/production/deploy/aws/terraform/services/security_group_rules/main.tf @@ -73,6 +73,15 @@ resource "aws_security_group_rule" "allow_elb_to_ec2_ingress" { source_security_group_id = var.elb_security_group_id } +resource "aws_security_group_rule" "allow_app_mesh_to_ec2_ingress" { + from_port = 50051 + protocol = "TCP" + security_group_id = var.instances_security_group_id + to_port = 50051 + type = "ingress" + cidr_blocks = ["0.0.0.0/0"] +} + resource "aws_security_group_rule" "allow_ssh_to_ec2_ingress" { from_port = 22 protocol = "TCP" @@ -132,6 +141,15 @@ resource "aws_security_group_rule" "allow_ec2_http_egress" { cidr_blocks = ["0.0.0.0/0"] } +resource "aws_security_group_rule" "allow_ec2_app_mesh_egress" { + from_port = 50051 + protocol = "TCP" + security_group_id = var.instances_security_group_id + to_port = 50051 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] +} + # Ingress and egress rules for backend vpc interface endpoints. resource "aws_security_group_rule" "allow_ec2_to_vpce_ingress" { from_port = 443 diff --git a/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf b/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf index 10cb10dc..3f30ff9f 100644 --- a/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf +++ b/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf @@ -57,7 +57,7 @@ module "buyer" { TEST_MODE = "false" # Do not change unless you are testing without key fetching. ENABLE_BIDDING_SERVICE_BENCHMARK = "" # Example: "false" - BUYER_KV_SERVER_ADDR = "" # Example: "https://googleads.g.doubleclick.net/td/bts" + BUYER_KV_SERVER_ADDR = "" # Example: "https://kvserver.com/trusted-signals" TEE_AD_RETRIEVAL_KV_SERVER_ADDR = "" # Example: "xds:///ad-retrieval-host" TEE_KV_SERVER_ADDR = "" # Example: "xds:///kv-service-host" AD_RETRIEVAL_TIMEOUT_MS = "" # Example: "60000" @@ -84,7 +84,6 @@ module "buyer" { # "urlFetchPeriodMs": 13000000, # "urlFetchTimeoutMs": 30000, # "enableBuyerDebugUrlGeneration": true, - # "enableAdtechCodeLogging": false, # "prepareDataForAdsRetrievalJsUrl": "", # "prepareDataForAdsRetrievalWasmHelperUrl": "", # }" @@ -119,9 +118,15 @@ module "buyer" { MAX_ALLOWED_SIZE_DEBUG_URL_BYTES = "" # Example: "65536" MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB = "" # Example: "3000" - INFERENCE_SIDECAR_BINARY_PATH = "" # Example: "/server/bin/inference_sidecar" - INFERENCE_MODEL_BUCKET_NAME = "" # Example: "" - INFERENCE_MODEL_BUCKET_PATHS = "" # Example: "," + INFERENCE_SIDECAR_BINARY_PATH = "" # Example: "/server/bin/inference_sidecar" + INFERENCE_MODEL_BUCKET_NAME = "" # Example: "" + INFERENCE_MODEL_BUCKET_PATHS = "" # Example: "," + INFERENCE_SIDECAR_RUNTIME_CONFIG = "" # Example: + # "{ + # "num_interop_threads": 4, + # "num_intraop_threads": 4, + # "module_name": "tensorflow_v2_14_0", + # }" # TCMalloc related config parameters. # See: https://github.com/google/tcmalloc/blob/master/docs/tuning.md diff --git a/production/deploy/gcp/terraform/environment/demo/buyer/terraform.tf b/production/deploy/gcp/terraform/environment/demo/buyer/terraform.tf index 0c677b4d..e9ba10da 100644 --- a/production/deploy/gcp/terraform/environment/demo/buyer/terraform.tf +++ b/production/deploy/gcp/terraform/environment/demo/buyer/terraform.tf @@ -18,11 +18,11 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = "5.0.0" + version = "5.31.0" } google = { source = "hashicorp/google" - version = "5.0.0" + version = "5.31.0" } } } diff --git a/production/deploy/gcp/terraform/environment/demo/seller/seller.tf b/production/deploy/gcp/terraform/environment/demo/seller/seller.tf index 67bad96d..326a38e8 100644 --- a/production/deploy/gcp/terraform/environment/demo/seller/seller.tf +++ b/production/deploy/gcp/terraform/environment/demo/seller/seller.tf @@ -61,8 +61,8 @@ module "seller" { GET_BID_RPC_TIMEOUT_MS = "" # Example: "60000" KEY_VALUE_SIGNALS_FETCH_RPC_TIMEOUT_MS = "" # Example: "60000" SCORE_ADS_RPC_TIMEOUT_MS = "" # Example: "60000" - SELLER_ORIGIN_DOMAIN = "" # Example: "https://securepubads.g.doubleclick.net" - KEY_VALUE_SIGNALS_HOST = "" # Example: "https://pubads.g.doubleclick.net/td/sts" + SELLER_ORIGIN_DOMAIN = "" # Example: "https://sellerorigin.com" + KEY_VALUE_SIGNALS_HOST = "" # Example: "https://keyvaluesignals.com/trusted-signals" BUYER_SERVER_HOSTS = "" # Example: "{ \"https://example-bidder.com\": { \"url\": \"dns:///bidding-service-host:443\", \"cloudPlatform\": \"GCP\" } }" SELLER_CLOUD_PLATFORMS_MAP = "" # Example: "{ \"https://partner-seller1.com\": "GCP", \"https://partner-seller2.com\": "AWS"}" ENABLE_SELLER_FRONTEND_BENCHMARKING = "" # Example: "false" @@ -80,7 +80,6 @@ module "seller" { # "urlFetchPeriodMs": 13000000, # "urlFetchTimeoutMs": 30000, # "enableSellerDebugUrlGeneration": true, - # "enableAdtechCodeLogging": false, # "enableReportResultUrlGeneration": true, # "enableReportWinUrlGeneration": true, # "buyerReportWinJsUrls": {"https://buyerA_origin.com":"https://buyerA.com/generateBid.js", @@ -103,7 +102,8 @@ module "seller" { PUBLIC_KEY_ENDPOINT = "https://publickeyservice.pa.gcp.privacysandboxservices.com/.well-known/protected-auction/v1/public-keys" SFE_PUBLIC_KEYS_ENDPOINTS = < --without-shared-cache Containers will not mount ${HOME}/.cache/bazel - --server-image Server docker image tarfile - --service-path [REQUIRED] One of: auction_service, bidding_service, buyer_frontend_service, seller_frontend_service + --service [REQUIRED] At least one of: auction_service, bidding_service, buyer_frontend_service, seller_frontend_service. Use multiple times to specify more than one service. --with-ami AMI region in which to build and store AWS AMIs. Use multiple times to specify more than one region --aws-image-tag Custom image tag for AMI image. Applied to AMI as tag value for key 'build_env'. @@ -54,27 +53,21 @@ USAGE declare -a AMI_REGIONS declare BUILD_FLAVOR=prod declare AWS_IMAGE_TAG="" +declare SERVICE_LIST=() while [[ $# -gt 0 ]]; do case "$1" in - --server-image) - SERVER_IMAGE="$2" - shift - shift - ;; - --service-path) - SERVICE="$2" - shift - shift + --service) + SERVICE_LIST+=("$2") + shift 2 || usage ;; --with-ami) AMI_REGIONS+=("$2") - shift - shift + shift 2 || usage ;; --aws-image-tag) AWS_IMAGE_TAG="$2" - shift 2 + shift 2 || usage ;; --verbose) set -o xtrace @@ -110,25 +103,36 @@ mkdir -p "${DIST}"/aws chmod 770 "${DIST}" "${DIST}"/aws printf "==== build AWS artifacts using build-amazonlinux2 =====\n" -# build nitro enclave image, collect eif artifacts -readonly IMAGE_URI=bazel/production/packaging/aws/${SERVICE} -IMAGE_TAG=$(mktemp --dry-run temp-XXXXXX) -readonly ENCLAVE_NAME=${SERVICE} -builder::cbuild_al2 $" -set -o errexit -# extract server docker image into local docker client and retag it -docker load -i ${SERVER_IMAGE} -docker tag ${IMAGE_URI}:server_docker_image ${IMAGE_URI}:${IMAGE_TAG} -nitro-cli build-enclave \ - --docker-uri ${IMAGE_URI}:${IMAGE_TAG} \ - --output-file /tmp/${ENCLAVE_NAME}.eif \ - > /tmp/${ENCLAVE_NAME}_${BUILD_FLAVOR}.json -if [[ $? -eq 0 ]]; then - mv /tmp/${ENCLAVE_NAME}.* dist/aws/ - mv /tmp/${ENCLAVE_NAME}_${BUILD_FLAVOR}.* dist/aws/ -fi -" -docker image rm "${IMAGE_URI}:${IMAGE_TAG}" +declare server_image="" +declare image_uri="" +declare image_tag="" +for service in "${SERVICE_LIST[@]}"; do + server_image="dist/debian/${service}_image.tar" + if [[ ! -s ${WORKSPACE}/${server_image} ]]; then + printf "Error: docker image tar file not found: %s\n" "${server_image}" >&2 + exit 1 + fi + + # build nitro enclave image, collect eif artifacts + image_uri="bazel/production/packaging/aws/${service}" + image_tag=$(mktemp --dry-run temp-XXXXXX) + + builder::cbuild_al2 $" + set -o errexit + # extract server docker image into local docker client and retag it + docker load -i ${server_image} + docker tag ${image_uri}:server_docker_image ${image_uri}:${image_tag} + nitro-cli build-enclave \ + --docker-uri ${image_uri}:${image_tag} \ + --output-file /tmp/${service}.eif \ + > /tmp/${service}_${BUILD_FLAVOR}.json + if [[ $? -eq 0 ]]; then + mv /tmp/${service}.* dist/aws/ + mv /tmp/${service}_${BUILD_FLAVOR}.* dist/aws/ + fi + " + docker image rm "${image_uri}:${image_tag}" +done builder::cbuild_al2 $" trap _collect_logs EXIT @@ -164,17 +168,21 @@ if [[ -n ${AMI_REGIONS[0]} ]]; then unzip -o -d "${DIST}"/aws "${DIST}"/aws/artifacts_proto.zip printf "==== build AWS AMI (using packer) =====\n" regions="$(arr_to_string_list AMI_REGIONS)" - build_version="$(git -C ${WORKSPACE} describe --tags --always || echo no-git-version)-${BUILD_FLAVOR}" - builder::cbuild_al2 " + build_version="$(git -C ${WORKSPACE} describe --tags --always || echo no-git-version)" + for service in "${SERVICE_LIST[@]}"; do + builder::cbuild_al2 " set -o errexit packer build \ -var=regions='${regions}' \ - -var=service=${SERVICE} \ + -var=service=${service} \ -var=git_commit=$(git rev-parse HEAD) \ + -var=build_flavor=${BUILD_FLAVOR} \ -var=build_version=${build_version} \ -var=build_env=${AWS_IMAGE_TAG} \ -var=distribution_dir=dist/aws \ -var=workspace=/src/workspace \ production/packaging/aws/common/ami/image.pkr.hcl -" +" & + done + wait fi diff --git a/production/packaging/aws/buyer_frontend_service/BUILD b/production/packaging/aws/buyer_frontend_service/BUILD index 4aef7e20..4e90b59b 100644 --- a/production/packaging/aws/buyer_frontend_service/BUILD +++ b/production/packaging/aws/buyer_frontend_service/BUILD @@ -26,7 +26,6 @@ load( ) load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") -load("//:config.bzl", "LOG_ENV_VARS") pkg_files( name = "server_executables", @@ -90,7 +89,6 @@ container_image( entrypoint = [ "/busybox/sh", ], - env = LOG_ENV_VARS, layers = [ ":server_binary_layer", ] + select({ diff --git a/production/packaging/aws/codebuild/README.md b/production/packaging/aws/codebuild/README.md index 5e96ffdb..e4a7cd2b 100644 --- a/production/packaging/aws/codebuild/README.md +++ b/production/packaging/aws/codebuild/README.md @@ -123,12 +123,22 @@ Set Github as your source provider, and use the public repository: `https://github.com/privacysandbox/bidding-auction-servers.git`. No other source repo config is necessary. -However, if you want to enable webhook events (after selecting 'Repository in my GitHub account', -webhook events are not available for the 'Public repository' option) and run an automatic build for -every release, you will have to clone the public repo into a Github repo under your ownership and -keep your clone up to date with the public repo. If you prefer to avoid cloning the public -repository, any time you want a new build for Bidding and Auction serivces you must follow the steps -[here](#start-build). +#### Webhooks + +Webhook events are not available for the 'Public repository' option. If you want to run an automatic +build for every release, you will have to clone the public repo into a Github repo under your +ownership and keep your clone up to date with the public repo. If you prefer to avoid cloning the +public repository, any time you want a new build for Bidding and Auction serivces you must follow +the steps [here](#start-build). + +If you do choose to use webhook events, select 'Repository in my Github account', and in the +'Primary source webhook events' section select 'Rebuild every time a code change is pushed to this +repository' for a 'Single build'. Then, add a 'Start a build' filter group of event type `PUSH`, +type `HEAD_REF`, and pattern `^refs/tags/.*`. This will build on every release tag. + +Make sure that your Github fork, if updated automatically, also fetches the tags from the upstream +repo -- that way, you can build directly from the semantically versioned tags. See +[here](sync_bidding_auction_repo.yaml) for an example Github Action that handles syncing. ### Environment @@ -154,9 +164,17 @@ Additional configuration: 1. Environment variables: ```plaintext - key: AMI_REGION value: - key: DOCKERHUB_USERNAME value: - key: DOCKERHUB_PASSWORD_SECRET_NAME value: + key: AMI_REGION + value: + + key: DOCKERHUB_USERNAME + value: + + key: DOCKERHUB_PASSWORD_SECRET_NAME + value: + + key: BUILD_FLAVOR + value: ``` ### Buildspec diff --git a/production/packaging/aws/codebuild/buildspec.yaml b/production/packaging/aws/codebuild/buildspec.yaml index 94efbaca..195a7378 100644 --- a/production/packaging/aws/codebuild/buildspec.yaml +++ b/production/packaging/aws/codebuild/buildspec.yaml @@ -42,5 +42,6 @@ phases: --service-path auction_service \ --instance aws --platform aws \ --with-ami ${AMI_REGION} \ - --build-flavor prod \ + --build-flavor ${BUILD_FLAVOR} \ + --aws-image-tag ${CODEBUILD_SOURCE_VERSION} \ --no-tests --no-precommit diff --git a/production/packaging/aws/codebuild/sync_bidding_auction_repo.yaml b/production/packaging/aws/codebuild/sync_bidding_auction_repo.yaml new file mode 120000 index 00000000..81b8fa15 --- /dev/null +++ b/production/packaging/aws/codebuild/sync_bidding_auction_repo.yaml @@ -0,0 +1 @@ +../../gcp/cloud_build/sync_bidding_auction_repo.yaml \ No newline at end of file diff --git a/production/packaging/aws/common/ami/BUILD b/production/packaging/aws/common/ami/BUILD index 91cc0620..1e265d30 100644 --- a/production/packaging/aws/common/ami/BUILD +++ b/production/packaging/aws/common/ami/BUILD @@ -19,6 +19,9 @@ filegroup( srcs = select({ "//conditions:default": [ "envoy.yaml", + "envoy_networking.sh", + "hc.bash", + "health.proto", "otel_collector_config.yaml", ], }), diff --git a/production/packaging/aws/common/ami/envoy_networking.sh b/production/packaging/aws/common/ami/envoy_networking.sh new file mode 100644 index 00000000..081b3c78 --- /dev/null +++ b/production/packaging/aws/common/ami/envoy_networking.sh @@ -0,0 +1,273 @@ +#!/bin/bash -e +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +while getopts "s:" flag; do + case $flag in + s) # Handle the -s flag with an argument + service=$OPTARG + ;; + \?) + # Handle invalid options + ;; + esac +done + +echo "value of service flag passed in with -s: $service" + +if [[ $service == "bidding" ]] || [[ $service == "auction" ]]; then + echo "service was either 'bidding' or 'auction'" +fi + +# +# Start of configurable options +# + +# Configure the iptables for envoy for the appmesh. +APPMESH_IGNORE_UID="1337" +APPMESH_APP_PORTS="50051" +APPMESH_INGRESS_IGNORED_PORTS="443" +APPMESH_EGRESS_IGNORED_PORTS="443" + +APPMESH_ENVOY_EGRESS_PORT="15001" +APPMESH_ENVOY_INGRESS_PORT="15000" +APPMESH_EGRESS_IGNORED_IP="169.254.169.254,169.254.170.2" + +# Enable IPv6. +[ -z "$APPMESH_ENABLE_IPV6" ] && APPMESH_ENABLE_IPV6="0" + +# Egress traffic from the processess owned by the following UID/GID will be ignored. +if [ -z "$APPMESH_IGNORE_UID" ] && [ -z "$APPMESH_IGNORE_GID" ]; then + echo "Variables APPMESH_IGNORE_UID and/or APPMESH_IGNORE_GID must be set." + echo "Envoy must run under those IDs to be able to properly route it's egress traffic." + exit 1 +fi + +# Port numbers Application and Envoy are listening on. +if [ -z "$APPMESH_ENVOY_EGRESS_PORT" ]; then + echo "APPMESH_ENVOY_EGRESS_PORT must be defined to forward traffic from the application to the proxy." + exit 1 +fi + +# If an app port was specified, then we also need to enforce the proxies ingress port so we know where to forward traffic. +if [ ! -z "$APPMESH_APP_PORTS" ] && [ -z "$APPMESH_ENVOY_INGRESS_PORT" ]; then + echo "APPMESH_ENVOY_INGRESS_PORT must be defined to forward traffic from the APPMESH_APP_PORTS to the proxy." + exit 1 +fi + +# Comma separated list of ports for which egress traffic will be ignored, we always refuse to route SSH traffic. +if [ -z "$APPMESH_EGRESS_IGNORED_PORTS" ]; then + APPMESH_EGRESS_IGNORED_PORTS="22" +else + APPMESH_EGRESS_IGNORED_PORTS="$APPMESH_EGRESS_IGNORED_PORTS,22" +fi + +# +# End of configurable options +# + +function initialize() { + echo "=== Initializing ===" + if [ ! -z "$APPMESH_APP_PORTS" ]; then + iptables -t nat -N APPMESH_INGRESS + if [ "$APPMESH_ENABLE_IPV6" == "1" ]; then + ip6tables -t nat -N APPMESH_INGRESS + fi + fi + iptables -t nat -N APPMESH_EGRESS + if [ "$APPMESH_ENABLE_IPV6" == "1" ]; then + ip6tables -t nat -N APPMESH_EGRESS + fi +} + +function enable_egress_routing() { + # Stuff to ignore + [ ! -z "$APPMESH_IGNORE_UID" ] && \ + iptables -t nat -A APPMESH_EGRESS \ + -m owner --uid-owner $APPMESH_IGNORE_UID \ + -j RETURN + + [ ! -z "$APPMESH_IGNORE_GID" ] && \ + iptables -t nat -A APPMESH_EGRESS \ + -m owner --gid-owner $APPMESH_IGNORE_GID \ + -j RETURN + + [ ! -z "$APPMESH_EGRESS_IGNORED_PORTS" ] && \ + for IGNORED_PORT in $(echo "$APPMESH_EGRESS_IGNORED_PORTS" | tr "," "\n"); do + iptables -t nat -A APPMESH_EGRESS \ + -p tcp \ + -m multiport --dports "$IGNORED_PORT" \ + -j RETURN + done + + if [ "$APPMESH_ENABLE_IPV6" == "1" ]; then + # Stuff to ignore ipv6 + [ ! -z "$APPMESH_IGNORE_UID" ] && \ + ip6tables -t nat -A APPMESH_EGRESS \ + -m owner --uid-owner $APPMESH_IGNORE_UID \ + -j RETURN + + [ ! -z "$APPMESH_IGNORE_GID" ] && \ + ip6tables -t nat -A APPMESH_EGRESS \ + -m owner --gid-owner $APPMESH_IGNORE_GID \ + -j RETURN + + [ ! -z "$APPMESH_EGRESS_IGNORED_PORTS" ] && \ + for IGNORED_PORT in $(echo "$APPMESH_EGRESS_IGNORED_PORTS" | tr "," "\n"); do + ip6tables -t nat -A APPMESH_EGRESS \ + -p tcp \ + -m multiport --dports "$IGNORED_PORT" \ + -j RETURN + done + fi + + # The list can contain both IPv4 and IPv6 addresses. We will loop over this list + # to add every IPv4 address into `iptables` and every IPv6 address into `ip6tables`. + [ ! -z "$APPMESH_EGRESS_IGNORED_IP" ] && \ + for IP_ADDR in $(echo "$APPMESH_EGRESS_IGNORED_IP" | tr "," "\n"); do + if [[ $IP_ADDR =~ .*:.* ]] + then + [ "$APPMESH_ENABLE_IPV6" == "1" ] && \ + ip6tables -t nat -A APPMESH_EGRESS \ + -p tcp \ + -d "$IP_ADDR" \ + -j RETURN + else + iptables -t nat -A APPMESH_EGRESS \ + -p tcp \ + -d "$IP_ADDR" \ + -j RETURN + fi + done + + # Redirect egress traffic destined to application port to envoy. + sudo iptables -t nat -A APPMESH_EGRESS -p tcp --dport 50051 -j REDIRECT \ + --to $APPMESH_ENVOY_EGRESS_PORT + + # Apply APPMESH_EGRESS chain to non local traffic + iptables -t nat -A OUTPUT \ + -p tcp \ + -m addrtype ! --dst-type LOCAL \ + -j APPMESH_EGRESS + + if [ "$APPMESH_ENABLE_IPV6" == "1" ]; then + # Redirect everything that is not ignored ipv6 + ip6tables -t nat -A APPMESH_EGRESS \ + -p tcp \ + -j REDIRECT --to $APPMESH_ENVOY_EGRESS_PORT + # Apply APPMESH_EGRESS chain to non local traffic ipv6 + ip6tables -t nat -A OUTPUT \ + -p tcp \ + -m addrtype ! --dst-type LOCAL \ + -j APPMESH_EGRESS + fi + +} + +function enable_ingress_redirect_routing() { + # Route everything arriving at the application port to Envoy + iptables -t nat -A APPMESH_INGRESS \ + -p tcp \ + -m multiport --dports "$APPMESH_APP_PORTS" \ + -j REDIRECT --to-port "$APPMESH_ENVOY_INGRESS_PORT" + + # Apply AppMesh ingress chain to everything non-local + iptables -t nat -A PREROUTING \ + -p tcp \ + -m addrtype ! --src-type LOCAL \ + -j APPMESH_INGRESS + + if [ "$APPMESH_ENABLE_IPV6" == "1" ]; then + # Route everything arriving at the application port to Envoy ipv6 + ip6tables -t nat -A APPMESH_INGRESS \ + -p tcp \ + -m multiport --dports "$APPMESH_APP_PORTS" \ + -j REDIRECT --to-port "$APPMESH_ENVOY_INGRESS_PORT" + + # Apply AppMesh ingress chain to everything non-local ipv6 + ip6tables -t nat -A PREROUTING \ + -p tcp \ + -m addrtype ! --src-type LOCAL \ + -j APPMESH_INGRESS + fi +} + +function enable_routing() { + echo "=== Enabling routing ===" + enable_egress_routing + if [ ! -z "$APPMESH_APP_PORTS" ]; then + if [[ $service == "bidding" ]] || [[ $service == "auction" ]]; then + echo "=== Enabling ingress routing ===" + enable_ingress_redirect_routing + fi + fi +} + +function disable_routing() { + echo "=== Disabling routing ===" + iptables -t nat -F APPMESH_INGRESS + iptables -t nat -F APPMESH_EGRESS + + if [ "$APPMESH_ENABLE_IPV6" == "1" ]; then + ip6tables -t nat -F APPMESH_INGRESS + ip6tables -t nat -F APPMESH_EGRESS + fi +} + +function dump_status() { + echo "=== iptables FORWARD table ===" + iptables -L -v -n + echo "=== iptables NAT table ===" + iptables -t nat -L -v -n + + if [ "$APPMESH_ENABLE_IPV6" == "1" ]; then + echo "=== ip6tables FORWARD table ===" + ip6tables -L -v -n + echo "=== ip6tables NAT table ===" + ip6tables -t nat -L -v -n + fi +} + +function clean_up() { + disable_routing + ruleNum=$(iptables -L PREROUTING -t nat --line-numbers | grep APPMESH_INGRESS | cut -d " " -f 1) + iptables -t nat -D PREROUTING $ruleNum + + ruleNum=$(iptables -L OUTPUT -t nat --line-numbers | grep APPMESH_EGRESS | cut -d " " -f 1) + iptables -t nat -D OUTPUT $ruleNum + + iptables -t nat -X APPMESH_INGRESS + iptables -t nat -X APPMESH_EGRESS + + if [ "$APPMESH_ENABLE_IPV6" == "1" ]; then + ruleNum=$(ip6tables -L PREROUTING -t nat --line-numbers | grep APPMESH_INGRESS | cut -d " " -f 1) + ip6tables -t nat -D PREROUTING $ruleNum + + ruleNum=$(ip6tables -L OUTPUT -t nat --line-numbers | grep APPMESH_EGRESS | cut -d " " -f 1) + ip6tables -t nat -D OUTPUT $ruleNum + + ip6tables -t nat -X APPMESH_INGRESS + ip6tables -t nat -X APPMESH_EGRESS + fi +} + +function print_config() { + echo "=== Input configuration ===" + env | grep APPMESH_ || true +} + +print_config + +initialize +enable_routing diff --git a/production/packaging/aws/common/ami/hc.bash b/production/packaging/aws/common/ami/hc.bash new file mode 100755 index 00000000..8b65eea5 --- /dev/null +++ b/production/packaging/aws/common/ami/hc.bash @@ -0,0 +1,200 @@ +#!/usr/bin/env bash + +# Example usage: +# bash ./hc.bash -p /usr/local/google/home/akundla/bidding-auction-server/production/packaging/aws/common/ami -n health.proto -a localhost:50051 -i 10 -t 5 -h 2 -u 10 -e i-08756c3a64a78711a -g 2 -r us-west-1 -s srv-ajxdkksp7d5wpmou + +# https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-retries.html +export AWS_RETRY_MODE="standard" +export AWS_MAX_ATTEMPTS=4 + +# Possible statuses: +GRPC_STATUS_UNKNOWN="UNKNOWN" +GRPC_STATUS_SERVING="SERVING" +GRPC_STATUS_NOT_SERVING="NOT_SERVING" +GRPC_STATUS_SERVICE_UNKNOWN="SERVICE_UNKNOWN" + +# Input parameter validation values. +healthcheck_min_frequency_sec=1 +healthcheck_max_frequency_sec=90 +min_checks_threshold=2 +max_checks_threshold=10 + +while getopts "p:n:a:i:t:h:u:e:g:r:s:" flag; do + case $flag in + p) + path_to_folder_of_healthcheck_proto_file=$OPTARG + ;; + n) + healthcheck_proto_file_name=$OPTARG + ;; + a) + address_and_port_of_service=$OPTARG + ;; + i) + if [[ $OPTARG -ge $healthcheck_min_frequency_sec ]] && [[ $OPTARG -le $healthcheck_max_frequency_sec ]] + then + interval_between_hc_sec=$OPTARG + else + echo "Invalid value for flag -i (interval_between_hc_sec): value must be between ${healthcheck_min_frequency_sec} and ${healthcheck_max_frequency_sec}, inclusive." + exit 1 + fi + ;; + t) + if [[ $OPTARG -ge $healthcheck_min_frequency_sec ]] && [[ $OPTARG -le $healthcheck_max_frequency_sec ]] + then + hc_timeout_sec=$OPTARG + else + echo "Invalid value for flag -t (hc_timeout_sec): value must be between ${healthcheck_min_frequency_sec} and ${healthcheck_max_frequency_sec}, inclusive." + exit 1 + fi + ;; + h) + if [[ $OPTARG -ge $min_checks_threshold ]] && [[ $OPTARG -le $max_checks_threshold ]] + then + healthy_threshold=$OPTARG + else + echo "Invalid value for flag -h (healthy_threshold): value must be between ${min_checks_threshold} and ${max_checks_threshold}, inclusive." + exit 1 + fi + ;; + u) + if [[ $OPTARG -ge $min_checks_threshold ]] && [[ $OPTARG -le $max_checks_threshold ]] + then + unhealthy_threshold=$OPTARG + else + echo "Invalid value for flag -u (unhealthy_threshold): value must be between ${min_checks_threshold} and ${max_checks_threshold}, inclusive." + exit 1 + fi + ;; + e) + instance_id=$OPTARG + ;; + g) + startup_grace_period=$OPTARG + ;; + r) + region=$OPTARG + ;; + s) + cloud_map_service_id=$OPTARG + ;; + \?) + # Handle invalid options + echo "Invalid option detected" + exit 1 + ;; + esac +done + +input_param_names=("path_to_folder_of_healthcheck_proto_file" "healthcheck_proto_file_name" "address_and_port_of_service" "interval_between_hc_sec" "hc_timeout_sec" "healthy_threshold" "unhealthy_threshold" "instance_id" "startup_grace_period" "region" "cloud_map_service_id") + +input_param_values=("${path_to_folder_of_healthcheck_proto_file}" "${healthcheck_proto_file_name}" "${address_and_port_of_service}" "${interval_between_hc_sec}" "${hc_timeout_sec}" "${healthy_threshold}" "${unhealthy_threshold}" "${instance_id}" "${startup_grace_period}" "${region}" "${cloud_map_service_id}") + +any_params_missing=false + +for index in ${!input_param_names[@]}; +do + if [[ -z "${input_param_values[$index]}" ]] + then + echo "${input_param_names[$index]} is missing!" + any_params_missing=true + fi +done + +if [[ ${any_params_missing} =~ "true" ]] +then + echo "All params required, exiting." + exit 1 +fi + +# Holds the last n statuses, where n = max(healthy_threshold, unhealthy_threshold) +healthcheck_status_queue=() +hc_stat_queue_max_len=$(( $healthy_threshold > $unhealthy_threshold ? $healthy_threshold : $unhealthy_threshold )) + +last_set_status_healthy=true + +echo "Custom health checking script initialized, waiting for grace period before beginning health checks." + +sleep "${startup_grace_period}" + +echo "Custom health checking script beginning healthchecks now." + +while true +do + current_hc_response=$(grpcurl --plaintext -connect-timeout=${hc_timeout_sec} -max-time=${hc_timeout_sec} -import-path=${path_to_folder_of_healthcheck_proto_file} -proto=${healthcheck_proto_file_name} ${address_and_port_of_service} grpc.health.v1.Health/Check) + + if [[ $current_hc_response == *$GRPC_STATUS_SERVING* ]] + then + healthcheck_status_queue+=(true) + # if the server has been set to UNEAHTLHY in the cloud map, it can be set to HEALTHY again. But if it has been condemned in the ASG, it is shutting down; even if the server recovers it will still shut down. + if [[ "${last_set_status_healthy}" =~ true ]] + then + aws servicediscovery update-instance-custom-health-status --service-id $cloud_map_service_id --instance-id $instance_id --region $region --status HEALTHY + fi + else + healthcheck_status_queue+=(false) + echo "Server is not serving; fails custom health check running on machine!" + echo "Last status between the quotes if present: '$($current_hc_response)'" + aws servicediscovery update-instance-custom-health-status --service-id $cloud_map_service_id --instance-id $instance_id --region $region --status UNHEALTHY + echo "Set to UNHEALTHY in cloud map" + fi + + # Keep queue at max length and no larger. + if [[ "${#healthcheck_status_queue[@]}" -gt hc_stat_queue_max_len ]] + then + # Remove first element + healthcheck_status_queue=("${healthcheck_status_queue[@]:1}") + fi + + current_hc_stat_queue_len=${#healthcheck_status_queue[@]} + + if [[ ${healthcheck_status_queue[-1]} =~ "true" ]] && [[ current_hc_stat_queue_len -ge $healthy_threshold ]] + then + start_i="$(( $current_hc_stat_queue_len - $healthy_threshold ))" + end_i="$(( $current_hc_stat_queue_len - 1 ))" + can_set_server_as_healthy=true + for i in $(seq $start_i $end_i) + do + if [[ ${healthcheck_status_queue[$i]} =~ "false" ]] + then + can_set_server_as_healthy=false + fi + done + if [[ ${can_set_server_as_healthy} =~ "true" ]] && [[ "${last_set_status_healthy}" =~ "false" ]] + then + aws autoscaling set-instance-health --instance-id $instance_id --region $region --health-status Healthy + last_set_status_healthy=true + echo "Just made ASG HC Call to set server to healthy" + fi + fi + + if [[ ${healthcheck_status_queue[-1]} =~ "false" ]] && [[ current_hc_stat_queue_len -ge $unhealthy_threshold ]] + then + echo "Can check if server is un-healthy." + start_i="$(( $current_hc_stat_queue_len - $unhealthy_threshold ))" + end_i="$(( $current_hc_stat_queue_len - 1 ))" + can_set_server_as_unhealthy=true + for i in $(seq $start_i $end_i) + do + if [[ ${healthcheck_status_queue[$i]} =~ "true" ]] + then + can_set_server_as_unhealthy=false + fi + echo "healthcheck_status_queue[$i]: ${healthcheck_status_queue[$i]}" + done + echo "Can set server as unhealthy: ${can_set_server_as_unhealthy}" + if [[ ${can_set_server_as_unhealthy} =~ "true" ]] && [[ "${last_set_status_healthy}" =~ "true" ]] + then + # We're about to flag this instance to be killed by ASG, so it must be de-registered from the cloud map - and we need to de-register it NOW, before the machine is shut down. + aws servicediscovery deregister-instance --instance-id $instance_id --service-id $cloud_map_service_id --region $region + echo "Just made Cloud Map Call to de-register instance" + # Now set the instance to be grought down and replaced by ASG. + aws autoscaling set-instance-health --instance-id $instance_id --region $region --health-status Unhealthy + last_set_status_healthy=false + echo "Just made ASG HC Call to set server to Unhealthy" + fi + fi + + # Sleep no matter what. + sleep "${interval_between_hc_sec}s" +done diff --git a/production/packaging/aws/common/ami/health.proto b/production/packaging/aws/common/ami/health.proto new file mode 100644 index 00000000..38843ff1 --- /dev/null +++ b/production/packaging/aws/common/ami/health.proto @@ -0,0 +1,63 @@ +// Copyright 2015 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto + +syntax = "proto3"; + +package grpc.health.v1; + +option csharp_namespace = "Grpc.Health.V1"; +option go_package = "google.golang.org/grpc/health/grpc_health_v1"; +option java_multiple_files = true; +option java_outer_classname = "HealthProto"; +option java_package = "io.grpc.health.v1"; + +message HealthCheckRequest { + string service = 1; +} + +message HealthCheckResponse { + enum ServingStatus { + UNKNOWN = 0; + SERVING = 1; + NOT_SERVING = 2; + SERVICE_UNKNOWN = 3; // Used only by the Watch method. + } + ServingStatus status = 1; +} + +service Health { + // If the requested service is unknown, the call will fail with status + // NOT_FOUND. + rpc Check(HealthCheckRequest) returns (HealthCheckResponse); + + // Performs a watch for the serving status of the requested service. + // The server will immediately send back a message indicating the current + // serving status. It will then subsequently send a new message whenever + // the service's serving status changes. + // + // If the requested service is unknown when the call is received, the + // server will send a message setting the serving status to + // SERVICE_UNKNOWN but will *not* terminate the call. If at some + // future point, the serving status of the service becomes known, the + // server will send a new message with the service's serving status. + // + // If the call terminates with status UNIMPLEMENTED, then clients + // should assume this method is not supported and should not retry the + // call. If the call terminates with any other status (including OK), + // clients should retry the call with appropriate exponential backoff. + rpc Watch(HealthCheckRequest) returns (stream HealthCheckResponse); +} diff --git a/production/packaging/aws/common/ami/image.pkr.hcl b/production/packaging/aws/common/ami/image.pkr.hcl index 74414753..9ce88be3 100644 --- a/production/packaging/aws/common/ami/image.pkr.hcl +++ b/production/packaging/aws/common/ami/image.pkr.hcl @@ -28,6 +28,12 @@ variable "git_commit" { variable "build_version" { type = string + description = "git tag if available" +} + +variable "build_flavor" { + type = string + description = "prod or non_prod" } variable "build_env" { @@ -78,7 +84,7 @@ locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") } # build blocks. A build block runs provisioners and post-processors on a # source. source "amazon-ebs" "dataserver" { - ami_name = "${var.service}-${local.timestamp}" + ami_name = "${var.service}-${var.build_version}-${var.build_flavor}-${local.timestamp}" instance_type = "m5.xlarge" region = var.regions[0] ami_regions = var.regions @@ -94,6 +100,7 @@ source "amazon-ebs" "dataserver" { tags = { git_commit = var.git_commit build_version = var.build_version + build_flavor = var.build_flavor build_env = var.build_env } ssh_username = "ec2-user" @@ -119,6 +126,18 @@ build { source = join("/", [var.distribution_dir, "otel_collector_config.yaml"]) destination = "/tmp/otel_collector_config.yaml" } + provisioner "file" { + source = join("/", [var.distribution_dir, "envoy_networking.sh"]) + destination = "/tmp/envoy_networking.sh" + } + provisioner "file" { + source = join("/", [var.distribution_dir, "hc.bash"]) + destination = "/tmp/hc.bash" + } + provisioner "file" { + source = join("/", [var.distribution_dir, "health.proto"]) + destination = "/tmp/health.proto" + } provisioner "file" { source = join("/", [var.distribution_dir, "/${var.service}.eif"]) destination = "/tmp/server_enclave_image.eif" @@ -140,7 +159,10 @@ mkdir -p /opt/privacysandbox mv /tmp/proxy /opt/privacysandbox/proxy mv /tmp/server_enclave_image.eif /opt/privacysandbox/server_enclave_image.eif mv /tmp/otel_collector_config.yaml /opt/privacysandbox/otel_collector_config.yaml -chmod 555 /opt/privacysandbox/{proxy,server_enclave_image.eif,otel_collector_config.yaml} +mv /tmp/envoy_networking.sh /opt/privacysandbox/envoy_networking.sh +mv /tmp/hc.bash /opt/privacysandbox/hc.bash +mv /tmp/health.proto /opt/privacysandbox/health.proto +chmod 555 /opt/privacysandbox/{proxy,server_enclave_image.eif,otel_collector_config.yaml,envoy_networking.sh,hc.bash,health.proto} EOF destination = "/tmp/setup_files" } diff --git a/production/packaging/aws/seller_frontend_service/BUILD b/production/packaging/aws/seller_frontend_service/BUILD index 4cafa1f8..9df37cf0 100644 --- a/production/packaging/aws/seller_frontend_service/BUILD +++ b/production/packaging/aws/seller_frontend_service/BUILD @@ -26,7 +26,6 @@ load( ) load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") -load("//:config.bzl", "LOG_ENV_VARS") pkg_files( name = "server_executables", @@ -90,7 +89,6 @@ container_image( entrypoint = [ "/busybox/sh", ], - env = LOG_ENV_VARS, layers = [ ":server_binary_layer", ] + select({ diff --git a/production/packaging/build_and_test_all_in_docker b/production/packaging/build_and_test_all_in_docker index 0e617070..674ccdaa 100755 --- a/production/packaging/build_and_test_all_in_docker +++ b/production/packaging/build_and_test_all_in_docker @@ -67,7 +67,7 @@ usage: $0 --service-path [REQUIRED] Recognized values: auction_service, bidding_service, buyer_frontend_service, seller_frontend_service. Use multiple times to specify more than one service. - --instance [REQUIRED] Recognized values: local, aws + --instance [REQUIRED] Recognized values: local, gcp, aws --platform [REQUIRED] Recognized values: gcp, aws --e2e Run e2e mode --build-flavor Recognized values: prod, non_prod, inference_non_prod. Default: prod @@ -260,19 +260,18 @@ function invoke_targets() { " } -function build_service_for_aws() { - declare -r service="$1" +function build_services_for_aws() { + declare -n services="$1" declare -n extra_args="$2" - # note: use relative path to dist - declare -r docker_image=dist/debian/${service}_image.tar - if ! [[ -s ${WORKSPACE}/${docker_image} ]]; then - printf "Error: docker image tar file not found: %s\n" "${docker_image}" &>/dev/stderr - exit 1 - fi - # shellcheck disable=SC2086 + + declare -a service_flags=() + for service in "${services[@]}"; do + service_flags+=("--service" "$service") + done + + # Call build_and_test with service and image flags "${SCRIPT_DIR}"/aws/build_and_test ${VERBOSE_ARG} \ - --server-image "${docker_image}" \ - --service-path "${service}" \ + ${service_flags[@]} \ --build-flavor ${BUILD_FLAVOR} \ "${extra_args[@]}" } @@ -324,10 +323,7 @@ if [[ ${NO_PLATFORM_BUILD} -eq 0 ]]; then if [[ ${VERBOSE} -eq 1 ]]; then EXTRA_ARGS+=("--verbose") fi - - for svc in "${SERVICES[@]}"; do - build_service_for_aws "${svc}" EXTRA_ARGS - done + build_services_for_aws SERVICES EXTRA_ARGS ;; gcp) source "${SCRIPT_DIR}"/gcp/lib_gcp_artifacts.sh diff --git a/production/packaging/gcp/auction_service/BUILD b/production/packaging/gcp/auction_service/BUILD index 3ff9579d..21c77c41 100644 --- a/production/packaging/gcp/auction_service/BUILD +++ b/production/packaging/gcp/auction_service/BUILD @@ -25,7 +25,6 @@ load( ) load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") -load("//:config.bzl", "LOG_ENV_VARS") pkg_files( name = "server_executables", @@ -75,7 +74,6 @@ container_image( entrypoint = [ "/server/bin/server", ], - env = LOG_ENV_VARS, labels = {"tee.launch_policy.log_redirect": "always"}, layers = [ ":server_binary_layer", diff --git a/production/packaging/gcp/bidding_service/BUILD b/production/packaging/gcp/bidding_service/BUILD index 7c949e00..5480eae1 100644 --- a/production/packaging/gcp/bidding_service/BUILD +++ b/production/packaging/gcp/bidding_service/BUILD @@ -25,7 +25,6 @@ load( ) load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") -load("//:config.bzl", "LOG_ENV_VARS") pkg_files( name = "server_executables", @@ -34,14 +33,18 @@ pkg_files( "//services/bidding_service:server", "@com_github_grpc_grpc//test/cpp/util:grpc_cli", ] + select({ + "//:include_all_inference_binaries": [ + "@pytorch_v2_1_1//:artifacts/inference_sidecar_pytorch_v2_1_1", + "@tensorflow_v2_14_0//:artifacts/inference_sidecar_tensorflow_v2_14_0", + ], "//:inference_noop": [ "@inference_common//:inference_sidecar_test_target", ], "//:inference_pytorch": [ - "@pytorch_v2_1_1//:artifacts/inference_sidecar", + "@pytorch_v2_1_1//:artifacts/inference_sidecar_pytorch_v2_1_1", ], "//:inference_tensorflow": [ - "@tensorflow_v2_14_0//:artifacts/inference_sidecar", + "@tensorflow_v2_14_0//:artifacts/inference_sidecar_tensorflow_v2_14_0", ], "//conditions:default": [], }), @@ -76,6 +79,28 @@ container_layer( ], ) +cddl_specs = [ + "//services/bidding_service:packaged_cddl_specs", +] + +pkg_zip( + name = "cddl_specs", + srcs = cddl_specs, +) + +pkg_tar( + name = "cddl_specs_tar", + srcs = cddl_specs, +) + +container_layer( + name = "cddl_specs_layer", + directory = "/", + tars = [ + ":cddl_specs_tar", + ], +) + container_image( name = "server_docker_image", base = select({ @@ -88,9 +113,9 @@ container_image( entrypoint = [ "sh", ], - env = LOG_ENV_VARS, labels = {"tee.launch_policy.log_redirect": "always"}, layers = [ + ":cddl_specs_layer", ":server_binary_layer", ] + select({ "//:e2e_build": [ diff --git a/production/packaging/gcp/bidding_service/test/structure.yaml b/production/packaging/gcp/bidding_service/test/structure.yaml index 0019582e..9c4698e8 100644 --- a/production/packaging/gcp/bidding_service/test/structure.yaml +++ b/production/packaging/gcp/bidding_service/test/structure.yaml @@ -31,5 +31,9 @@ fileExistenceTests: path: /etc/ssl/certs/ca-certificates.crt shouldExist: true + - name: egress_cddl_spec_v1.0.0 + path: /egress_cddl_spec/1.0.0 + shouldExist: true + licenseTests: - debian: true diff --git a/production/packaging/gcp/buyer_frontend_service/BUILD b/production/packaging/gcp/buyer_frontend_service/BUILD index afb20ce7..4a39763c 100644 --- a/production/packaging/gcp/buyer_frontend_service/BUILD +++ b/production/packaging/gcp/buyer_frontend_service/BUILD @@ -25,7 +25,6 @@ load( ) load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") -load("//:config.bzl", "LOG_ENV_VARS") pkg_files( name = "server_executables", @@ -70,7 +69,6 @@ container_image( "/server/bin/init_server_basic", ], entrypoint = ["sh"], - env = LOG_ENV_VARS, labels = {"tee.launch_policy.log_redirect": "always"}, layers = [ ":server_binary_layer", diff --git a/production/packaging/gcp/cloud_build/README.md b/production/packaging/gcp/cloud_build/README.md index a9258aa2..c88032eb 100644 --- a/production/packaging/gcp/cloud_build/README.md +++ b/production/packaging/gcp/cloud_build/README.md @@ -25,7 +25,10 @@ First, follow the steps to [connect a Github repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen) and create a host connection. You will need to clone the [Bidding and Auction repo](https://github.com/privacysandbox/bidding-auction-servers) to your own -Github account before you can connect it to your GCP project's Cloud Build. +Github account before you can connect it to your GCP project's Cloud Build. Make sure that your +fork, if updated automatically, also fetches the tags from the upstream repo -- that way, you can +build directly from the semantically versioned tags. See [here](sync_bidding_auction_repo.yaml) for +an example Github Action that handles syncing. #### Configuring an Image Repo @@ -51,6 +54,10 @@ You must create a build trigger. Starting with a [Github](https://cloud.google.com/build/docs/triggers#github) trigger is recommended. Please make sure to use a '2nd gen' repository source type. +Recommendation 1: Use a `Push a new tag` Event to build `.*` tags. + +Recommendation 2: Create a separate trigger for each `_BUILD_FLAVOR` (see below). + #### Configuration 1. Type: Cloud Build configuration file (yaml or json) @@ -65,8 +72,14 @@ sure to use a '2nd gen' repository source type. Note: these will override variables in the cloudbuild.yaml. ```plaintext - key: _GCP_IMAGE_REPO value: service images repo URI from prerequisites (default: us-docker.pkg.dev/${PROJECT_ID}/services) - key: _GCP_IMAGE_TAG value: any tag (default: from-cloudbuild) + key: _BUILD_FLAVOR + value: prod or non_prod. While 'prod' allows for attestation against production private keys, non_prod has enhanced logging. + + key: _GCP_IMAGE_REPO + value: service images repo URI from prerequisites (default: us-docker.pkg.dev/${PROJECT_ID}/services) + + key: _GCP_IMAGE_TAG + value: any tag (default: ${GIT_TAG}, only useful if building from a tag directly) ``` 1. Service account: Use the account created [previously](#service-account-permissions). diff --git a/production/packaging/gcp/cloud_build/cloudbuild.yaml b/production/packaging/gcp/cloud_build/cloudbuild.yaml index 030f9b16..0928a0c9 100644 --- a/production/packaging/gcp/cloud_build/cloudbuild.yaml +++ b/production/packaging/gcp/cloud_build/cloudbuild.yaml @@ -32,7 +32,7 @@ steps: --service-path seller_frontend_service \ --service-path auction_service \ --instance gcp --platform gcp \ - --build-flavor prod \ + --build-flavor ${_BUILD_FLAVOR} \ --gcp-image-tag ${_GCP_IMAGE_TAG} \ --gcp-image-repo ${_GCP_IMAGE_REPO} \ --no-tests --no-precommit @@ -41,7 +41,8 @@ substitutions: # CloudBuild Trigger GUI. # See https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values # for more information. - _GCP_IMAGE_TAG: from-cloudbuild # Default. Useful for organization. The build already tags the images with verison information. + _BUILD_FLAVOR: prod # Default. Use non_prod for enhanced logging output. + _GCP_IMAGE_TAG: ${GIT_TAG} # Default. Useful for organization. _GCP_IMAGE_REPO: us-docker.pkg.dev/${PROJECT_ID}/services # Default. Artifact Registry repo to house images for each service. timeout: 10800s options: diff --git a/production/packaging/gcp/cloud_build/sync_bidding_auction_repo.yaml b/production/packaging/gcp/cloud_build/sync_bidding_auction_repo.yaml new file mode 100644 index 00000000..a94c72ea --- /dev/null +++ b/production/packaging/gcp/cloud_build/sync_bidding_auction_repo.yaml @@ -0,0 +1,44 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file contains a Github Action that syncs with +# the Bidding and Auction Servers repo, including tags, +# on a regular schedule. It will push any new tags +# to your repo, and you can use Cloud Build, +# CodeBuild, or any webhook-based system to trigger +# an automatic build based on the new tag. +name: Sync with github.com/privacysandbox/bidding-auction-servers + +on: + schedule: + - cron: '*/30 * * * *' # every 30 minutes + workflow_dispatch: # on button click + +jobs: + sync_code: + runs-on: ubuntu-latest + steps: + - uses: tgymnich/fork-sync@v1.8 + continue-on-error: true + with: + base: main + head: main + - name: Checkout Code + uses: actions/checkout@v3 + if: always() # Always checkout, even if sync fails + - name: Sync Tags with Upstream + if: always() # Always sync tags, even if sync or checkout fails + run: | + git fetch https://github.com/privacysandbox/bidding-auction-servers --tags --force + git push origin --tags --force diff --git a/production/packaging/gcp/seller_frontend_service/BUILD b/production/packaging/gcp/seller_frontend_service/BUILD index c796c81a..0a862608 100644 --- a/production/packaging/gcp/seller_frontend_service/BUILD +++ b/production/packaging/gcp/seller_frontend_service/BUILD @@ -26,7 +26,6 @@ load( ) load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") -load("//:config.bzl", "LOG_ENV_VARS") pkg_files( name = "server_executables", @@ -123,7 +122,6 @@ container_image( "/server/bin/init_server_basic", ], entrypoint = ["sh"], - env = LOG_ENV_VARS, labels = {"tee.launch_policy.log_redirect": "always"}, layers = [ ":server_binary_layer", diff --git a/services/auction_service/BUILD b/services/auction_service/BUILD index 42f7db09..a2f048e9 100644 --- a/services/auction_service/BUILD +++ b/services/auction_service/BUILD @@ -13,8 +13,7 @@ # limitations under the License. load("@bazel_skylib//rules:copy_file.bzl", "copy_file") -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library", "cc_test") -load("@rules_proto//proto:defs.bzl", "proto_library") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") load("//:config.bzl", "ENABLE_CORE_DUMPS_DEFINES") cc_library( @@ -36,16 +35,6 @@ cc_library( visibility = ["//visibility:public"], ) -cc_library( - name = "buyer_adtech_reporting_wrapper", - hdrs = [ - "buyer_adtech_reporting_wrapper.h", - ], - deps = [ - "@com_google_absl//absl/strings", - ], -) - cc_library( name = "auction_service", srcs = [ @@ -93,6 +82,7 @@ cc_library( "//services/common/code_dispatch:code_dispatch_reactor", "//services/common/constants:user_error_strings", "//services/common/encryption:crypto_client_wrapper_interface", + "//services/common/loggers:request_log_context", "//services/common/metric:server_definition", "//services/common/reporters:async_reporter", "//services/common/util:auction_scope_util", @@ -103,7 +93,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@google_privacysandbox_servers_common//src/encryption/key_fetcher/interface:key_fetcher_manager_interface", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", "@google_privacysandbox_servers_common//src/util/status_macro:status_util", "@rapidjson", @@ -146,6 +135,7 @@ cc_test( ":auction_constants", ":score_ads_reactor", ":score_ads_reactor_test_util", + "//services/auction_service/reporting:reporting_helper_test", "//services/common/test:random", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", @@ -187,6 +177,8 @@ cc_test( "//services/auction_service/benchmarking:score_ads_no_op_logger", "//services/auction_service/code_wrapper:seller_code_wrapper", "//services/auction_service/code_wrapper:seller_code_wrapper_test", + "//services/auction_service/code_wrapper:seller_udf_wrapper", + "//services/auction_service/code_wrapper:seller_udf_wrapper_test", "//services/common/constants:common_service_flags", "//services/common/encryption:key_fetcher_factory", "//services/common/encryption:mock_crypto_client_wrapper", @@ -202,73 +194,37 @@ cc_test( ], ) -proto_library( - name = "auction_code_fetch_config_proto", - srcs = ["auction_code_fetch_config.proto"], - deps = [ - "@com_google_googleapis//google/api:annotations_proto", - "@com_google_protobuf//:struct_proto", - ], -) - -cc_proto_library( - name = "auction_code_fetch_config_cc_proto", - visibility = ["//services/auction_service:__subpackages__"], - deps = [":auction_code_fetch_config_proto"], -) - -cc_library( - name = "seller_code_fetch_manager", +cc_test( + name = "auction_service_reporting_integration_test", + size = "large", srcs = [ - "seller_code_fetch_manager.cc", - ], - hdrs = [ - "seller_code_fetch_manager.h", - ], - deps = [ - ":auction_code_fetch_config_cc_proto", - ":auction_constants", - "//services/auction_service/code_wrapper:buyer_reporting_fetcher", - "//services/auction_service/code_wrapper:seller_code_wrapper", - "//services/common/clients/code_dispatcher:v8_dispatcher", - "//services/common/code_fetch:periodic_bucket_fetcher", - "//services/common/code_fetch:periodic_code_fetcher", - "//services/common/util:file_util", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/time", - "@google_privacysandbox_servers_common//src/concurrent:executor", - "@google_privacysandbox_servers_common//src/logger:request_context_logger", - "@google_privacysandbox_servers_common//src/public/core/interface:errors", - "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + "auction_service_integration_test_util.cc", + "auction_service_integration_test_util.h", + "auction_service_reporting_integration_test.cc", ], -) - -cc_test( - name = "seller_code_fetch_manager_test", - size = "small", - srcs = ["seller_code_fetch_manager_test.cc"], deps = [ - ":seller_code_fetch_manager", + ":auction_service", + "//services/auction_service/benchmarking:score_ads_benchmarking_logger", + "//services/auction_service/benchmarking:score_ads_no_op_logger", + "//services/auction_service/code_wrapper:buyer_reporting_udf_wrapper", "//services/auction_service/code_wrapper:seller_code_wrapper", - "//services/common/clients/code_dispatcher:v8_dispatcher", - "//services/common/clients/http:http_fetcher_async", - "//services/common/code_fetch:periodic_bucket_fetcher", - "//services/common/code_fetch:periodic_code_fetcher", + "//services/auction_service/code_wrapper:seller_code_wrapper_test", + "//services/auction_service/code_wrapper:seller_udf_wrapper", + "//services/auction_service/code_wrapper:seller_udf_wrapper_test", + "//services/auction_service/udf_fetcher:adtech_code_version_util", + "//services/common/constants:common_service_flags", + "//services/common/encryption:key_fetcher_factory", + "//services/common/encryption:mock_crypto_client_wrapper", "//services/common/test:mocks", - "@com_google_absl//absl/functional:any_invocable", + "//services/common/test:random", + "//services/common/util:request_response_constants", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/random", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", "@com_google_absl//absl/time", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", - "@google_privacysandbox_servers_common//src/concurrent:executor", - "@google_privacysandbox_servers_common//src/public/core/interface:errors", - "@google_privacysandbox_servers_common//src/public/core/interface:execution_result", - "@google_privacysandbox_servers_common//src/public/cpio/interface:cpio", - "@google_privacysandbox_servers_common//src/public/cpio/interface/blob_storage_client", - "@google_privacysandbox_servers_common//src/public/cpio/mock/blob_storage_client:blob_storage_client_mock", ], ) @@ -282,15 +238,15 @@ cc_binary( malloc = "@com_google_tcmalloc//tcmalloc", visibility = ["//visibility:public"], deps = [ - ":auction_code_fetch_config_cc_proto", ":auction_constants", ":auction_service", - ":seller_code_fetch_manager", "//api:bidding_auction_servers_cc_grpc_proto", "//api:bidding_auction_servers_cc_proto", "//services/auction_service/benchmarking:score_ads_benchmarking_logger", "//services/auction_service/benchmarking:score_ads_no_op_logger", "//services/auction_service/data:runtime_config", + "//services/auction_service/udf_fetcher:auction_code_fetch_config_cc_proto", + "//services/auction_service/udf_fetcher:seller_udf_fetch_manager", "//services/common/clients/config:config_client_util", "//services/common/clients/http:multi_curl_http_fetcher_async", "//services/common/encryption:crypto_client_factory", diff --git a/services/auction_service/auction_main.cc b/services/auction_service/auction_main.cc index ae861683..f0729984 100644 --- a/services/auction_service/auction_main.cc +++ b/services/auction_service/auction_main.cc @@ -32,7 +32,6 @@ #include "grpcpp/ext/proto_server_reflection_plugin.h" #include "grpcpp/grpcpp.h" #include "grpcpp/health_check_service_interface.h" -#include "services/auction_service/auction_code_fetch_config.pb.h" #include "services/auction_service/auction_constants.h" #include "services/auction_service/auction_service.h" #include "services/auction_service/benchmarking/score_ads_benchmarking_logger.h" @@ -40,7 +39,8 @@ #include "services/auction_service/code_wrapper/seller_code_wrapper.h" #include "services/auction_service/data/runtime_config.h" #include "services/auction_service/runtime_flags.h" -#include "services/auction_service/seller_code_fetch_manager.h" +#include "services/auction_service/udf_fetcher/auction_code_fetch_config.pb.h" +#include "services/auction_service/udf_fetcher/seller_udf_fetch_manager.h" #include "services/common/clients/config/trusted_server_config_client.h" #include "services/common/clients/config/trusted_server_config_client_util.h" #include "services/common/clients/http/multi_curl_http_fetcher_async.h" @@ -95,6 +95,8 @@ using ::google::scp::cpio::LogOption; using ::grpc::Server; using ::grpc::ServerBuilder; +bool kEnableSellerAndBuyerUdfIsolation = false; + absl::StatusOr GetConfigClient( absl::string_view config_param_prefix) { TrustedServersConfigClient config_client(GetServiceFlags()); @@ -219,12 +221,13 @@ absl::Status RunServer() { code_fetch_proto.enable_report_win_url_generation(); const bool enable_protected_app_signals = config_client.GetBooleanParameter(ENABLE_PROTECTED_APP_SIGNALS); - + code_fetch_proto.set_enable_seller_and_buyer_udf_isolation( + kEnableSellerAndBuyerUdfIsolation); MultiCurlHttpFetcherAsync http_fetcher = MultiCurlHttpFetcherAsync(executor.get()); HttpFetcherAsync* seller_udf_fetcher = &http_fetcher; HttpFetcherAsync* buyer_reporting_udf_fetcher = &http_fetcher; - SellerCodeFetchManager code_fetch_manager( + SellerUdfFetchManager code_fetch_manager( BlobStorageClientFactory::Create(), executor.get(), seller_udf_fetcher, buyer_reporting_udf_fetcher, &dispatcher, code_fetch_proto, enable_protected_app_signals); @@ -261,7 +264,7 @@ absl::Status RunServer() { }; std::string default_code_version = - code_fetch_proto.fetch_mode() == auction_service::FETCH_MODE_BUCKET + code_fetch_proto.fetch_mode() == blob_fetch::FETCH_MODE_BUCKET ? code_fetch_proto.auction_js_bucket_default_blob() : kScoreAdBlobVersion; @@ -281,7 +284,9 @@ absl::Status RunServer() { config_client.GetIntParameter(MAX_ALLOWED_SIZE_DEBUG_URL_BYTES), .max_allowed_size_all_debug_urls_kb = config_client.GetIntParameter(MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB), - .default_code_version = default_code_version}; + .default_code_version = default_code_version, + .enable_seller_and_buyer_udf_isolation = + kEnableSellerAndBuyerUdfIsolation}; AuctionService auction_service( std::move(score_ads_reactor_factory), CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr), diff --git a/services/auction_service/auction_service_integration_test.cc b/services/auction_service/auction_service_integration_test.cc index 8bb57185..4229af9e 100644 --- a/services/auction_service/auction_service_integration_test.cc +++ b/services/auction_service/auction_service_integration_test.cc @@ -26,6 +26,7 @@ #include "services/auction_service/benchmarking/score_ads_no_op_logger.h" #include "services/auction_service/code_wrapper/seller_code_wrapper.h" #include "services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h" +#include "services/auction_service/code_wrapper/seller_udf_wrapper_test_constants.h" #include "services/common/clients/code_dispatcher/code_dispatch_client.h" #include "services/common/clients/config/trusted_server_config_client.h" #include "services/common/constants/common_service_flags.h" @@ -77,6 +78,13 @@ constexpr char kTestReportWinUrlWithNoising[] = "signalsForWinner=testSignalsForWinner&perBuyerSignals=1,test,2&" "auctionSignals=3,test,4&desirability=undefined"; +constexpr char kTestComponentWinReportingUrl[] = + "http://componentReportingUrl.com"; +constexpr char kTestComponentEvent[] = "click"; +constexpr char kTestComponentInteractionReportingUrl[] = + "http://componentInteraction.com"; +constexpr char kTestComponentSeller[] = "http://componentSeller.com"; + using ::google::protobuf::TextFormat; using AdWithBidMetadata = ScoreAdsRequest::ScoreAdsRawRequest::AdWithBidMetadata; @@ -292,6 +300,16 @@ struct TestBuyerReportingSignals { std::string auction_signals = kTestAuctionSignals; }; +struct TestScoreAdsRequestConfig { + const TestBuyerReportingSignals& test_buyer_reporting_signals; + bool enable_debug_reporting = false; + int desired_ad_count = 90; + absl::string_view top_level_seller = ""; + absl::string_view component_level_seller = ""; + bool is_consented = false; + std::optional buyer_reporting_id; +}; + void BuildScoreAdsRequest( ScoreAdsRequest* request, absl::flat_hash_map* interest_group_to_ad, @@ -331,6 +349,26 @@ void BuildScoreAdsRequest( request->set_key_id(kKeyId); } +void BuildTopLevelAuctionScoreAdsRequest( + ScoreAdsRequest* request, bool enable_debug_reporting = false, + int desired_ad_count = 20, absl::string_view seller = kTestSeller) { + ScoreAdsRequest::ScoreAdsRawRequest raw_request; + std::string generation_id = MakeARandomString(); + for (int i = 0; i < desired_ad_count; i++) { + auto auction_result = MakeARandomComponentAuctionResultWithReportingUrls( + generation_id, absl::StrCat(seller), kTestComponentEvent, + kTestComponentWinReportingUrl, kTestComponentInteractionReportingUrl, + kTestComponentSeller); + *raw_request.mutable_component_auction_results()->Add() = auction_result; + } + if (enable_debug_reporting) { + raw_request.set_enable_debug_reporting(enable_debug_reporting); + } + raw_request.set_seller(seller); + *request->mutable_request_ciphertext() = raw_request.SerializeAsString(); + request->set_key_id(kKeyId); +} + void BuildComponentScoreAdsRequest( ScoreAdsRequest* request, absl::flat_hash_map* interest_group_to_ad, @@ -343,32 +381,36 @@ void BuildComponentScoreAdsRequest( void BuildScoreAdsRequestForReporting( ScoreAdsRequest* request, absl::flat_hash_map* interest_group_to_ad, - const TestBuyerReportingSignals& test_buyer_reporting_signals, - bool enable_debug_reporting = false, int desired_ad_count = 90, - absl::string_view top_level_seller = "", const bool is_consented = false, - absl::string_view buyer_reporting_id = "") { + const TestScoreAdsRequestConfig& test_score_ads_request_config) { ScoreAdsRequest::ScoreAdsRawRequest raw_request; std::string trusted_scoring_signals = R"json({"renderUrls":{"placeholder_url":[123])json"; - for (int i = 0; i < desired_ad_count; i++) { - ScoreAdsRequest::ScoreAdsRawRequest::AdWithBidMetadata ad; - ad = MakeARandomAdWithBidMetadata(/*min_bid=*/1, /*max_bid=*/1); - if (!buyer_reporting_id.empty()) { - ad.set_buyer_reporting_id(buyer_reporting_id); + for (int i = 0; i < test_score_ads_request_config.desired_ad_count; i++) { + auto ad = MakeARandomAdWithBidMetadata(/*min_bid=*/1, /*max_bid=*/1); + if (test_score_ads_request_config.buyer_reporting_id.has_value()) { + ad.set_buyer_reporting_id( + test_score_ads_request_config.buyer_reporting_id.value()); } ad.set_bid_currency(kEurosIsoCode); - ad.set_ad_cost(test_buyer_reporting_signals.ad_cost); - ad.set_modeling_signals(test_buyer_reporting_signals.modeling_signals); - ad.set_join_count(test_buyer_reporting_signals.join_count); + ad.set_ad_cost( + test_score_ads_request_config.test_buyer_reporting_signals.ad_cost); + ad.set_modeling_signals(test_score_ads_request_config + .test_buyer_reporting_signals.modeling_signals); + ad.set_join_count( + test_score_ads_request_config.test_buyer_reporting_signals.join_count); ad.set_interest_group_name( - test_buyer_reporting_signals.interest_group_name); + test_score_ads_request_config.test_buyer_reporting_signals + .interest_group_name); ad.set_interest_group_owner(kDefaultBarInterestGroupOwner); ad.set_render( absl::StrFormat("%s/ads?id=%d", kDefaultBarInterestGroupOwner, i)); - ad.set_recency(test_buyer_reporting_signals.recency); + ad.set_recency( + test_score_ads_request_config.test_buyer_reporting_signals.recency); *raw_request.mutable_ad_bids()->Add() = ad; raw_request.mutable_per_buyer_signals()->try_emplace( - ad.interest_group_owner(), test_buyer_reporting_signals.buyer_signals); + ad.interest_group_owner(), + test_score_ads_request_config.test_buyer_reporting_signals + .buyer_signals); interest_group_to_ad->try_emplace(ad.interest_group_name(), ad); std::string ad_signal = absl::StrFormat( @@ -379,16 +421,21 @@ void BuildScoreAdsRequestForReporting( absl::StrAppend(&trusted_scoring_signals, R"json(},"adComponentRenderUrls":{}})json"); raw_request.set_scoring_signals(trusted_scoring_signals); - if (enable_debug_reporting) { - raw_request.set_enable_debug_reporting(enable_debug_reporting); + if (test_score_ads_request_config.enable_debug_reporting) { + raw_request.set_enable_debug_reporting( + test_score_ads_request_config.enable_debug_reporting); } raw_request.set_seller("http://seller.com"); raw_request.set_publisher_hostname(kPublisherHostname); - raw_request.set_auction_signals(test_buyer_reporting_signals.auction_signals); - raw_request.mutable_consented_debug_config()->set_is_consented(is_consented); + raw_request.set_auction_signals( + test_score_ads_request_config.test_buyer_reporting_signals + .auction_signals); + raw_request.mutable_consented_debug_config()->set_is_consented( + test_score_ads_request_config.is_consented); raw_request.mutable_consented_debug_config()->set_token(kTestConsentToken); - if (!top_level_seller.empty()) { - raw_request.set_top_level_seller(top_level_seller); + if (!test_score_ads_request_config.top_level_seller.empty()) { + raw_request.set_top_level_seller( + test_score_ads_request_config.top_level_seller); } *request->mutable_request_ciphertext() = raw_request.SerializeAsString(); request->set_key_id(kKeyId); @@ -756,14 +803,15 @@ TEST_F(AuctionServiceIntegrationTest, bool enable_adtech_code_logging = true; bool enable_report_result_url_generation = true; bool enable_report_win_url_generation = true; - int desired_ad_count = 90; bool enable_report_win_input_noising = false; TestBuyerReportingSignals test_buyer_reporting_signals; ScoreAdsRequest request; absl::flat_hash_map interest_group_to_ad; - BuildScoreAdsRequestForReporting( - &request, &interest_group_to_ad, test_buyer_reporting_signals, - enable_debug_reporting, desired_ad_count, ""); + TestScoreAdsRequestConfig test_score_ads_request_config = { + .test_buyer_reporting_signals = test_buyer_reporting_signals, + .buyer_reporting_id = ""}; + BuildScoreAdsRequestForReporting(&request, &interest_group_to_ad, + test_score_ads_request_config); ScoreAdsResponse response; SellerCodeWrappingTestHelper( &request, &response, kSellerBaseCode, enable_seller_debug_url_generation, @@ -800,16 +848,15 @@ TEST_F(AuctionServiceIntegrationTest, bool enable_adtech_code_logging = true; bool enable_report_result_url_generation = true; bool enable_report_win_url_generation = true; - int desired_ad_count = 90; bool enable_report_win_input_noising = false; - bool is_consented = false; TestBuyerReportingSignals test_buyer_reporting_signals; ScoreAdsRequest request; absl::flat_hash_map interest_group_to_ad; + TestScoreAdsRequestConfig test_score_ads_request_config = { + .test_buyer_reporting_signals = test_buyer_reporting_signals, + .buyer_reporting_id = kTestBuyerReportingId}; BuildScoreAdsRequestForReporting(&request, &interest_group_to_ad, - test_buyer_reporting_signals, - enable_debug_reporting, desired_ad_count, "", - is_consented, kTestBuyerReportingId); + test_score_ads_request_config); ScoreAdsResponse response; SellerCodeWrappingTestHelper( &request, &response, kSellerBaseCode, enable_seller_debug_url_generation, @@ -846,15 +893,14 @@ TEST_F(AuctionServiceIntegrationTest, ReportingSuccessfulWithNoising) { bool enable_adtech_code_logging = true; bool enable_report_result_url_generation = true; bool enable_report_win_url_generation = true; - int desired_ad_count = 90; bool enable_report_win_input_noising = true; bool enable_protected_app_signals = false; - TestBuyerReportingSignals test_buyer_reporting_signals; ScoreAdsRequest request; absl::flat_hash_map interest_group_to_ad; - BuildScoreAdsRequestForReporting( - &request, &interest_group_to_ad, test_buyer_reporting_signals, - enable_debug_reporting, desired_ad_count, ""); + TestScoreAdsRequestConfig test_score_ads_request_config = { + .test_buyer_reporting_signals = {}}; + BuildScoreAdsRequestForReporting(&request, &interest_group_to_ad, + test_score_ads_request_config); ScoreAdsResponse response; SellerCodeWrappingTestHelper( &request, &response, kSellerBaseCode, enable_seller_debug_url_generation, @@ -896,8 +942,10 @@ TEST_F(AuctionServiceIntegrationTest, TestBuyerReportingSignals test_buyer_reporting_signals; ScoreAdsRequest request; absl::flat_hash_map interest_group_to_ad; + TestScoreAdsRequestConfig test_score_ads_request_config = { + .test_buyer_reporting_signals = test_buyer_reporting_signals}; BuildScoreAdsRequestForReporting(&request, &interest_group_to_ad, - test_buyer_reporting_signals); + test_score_ads_request_config); ScoreAdsResponse response; SellerCodeWrappingTestHelper( &request, &response, kSellerBaseCode, enable_seller_debug_url_generation, @@ -938,9 +986,11 @@ TEST_F(AuctionServiceIntegrationTest, ScoreAdsRequest request; absl::flat_hash_map interest_group_to_ad; + TestScoreAdsRequestConfig test_score_ads_request_config = { + .test_buyer_reporting_signals = test_buyer_reporting_signals, + .top_level_seller = kTestTopLevelSeller}; BuildScoreAdsRequestForReporting(&request, &interest_group_to_ad, - test_buyer_reporting_signals, false, 90, - kTestTopLevelSeller); + test_score_ads_request_config); ScoreAdsResponse response; ScoreAdsRequest::ScoreAdsRawRequest score_ads_raw_request; score_ads_raw_request.ParseFromString(request.request_ciphertext()); @@ -981,6 +1031,57 @@ TEST_F(AuctionServiceIntegrationTest, kTestInteractionReportingUrl); } +TEST_F(AuctionServiceIntegrationTest, + ScoresAdsReturnsSuccessfullyWithReportingForTopLevelAuctions) { + bool enable_seller_debug_url_generation = false; + bool enable_debug_reporting = false; + bool enable_adtech_code_logging = true; + bool enable_report_result_url_generation = true; + bool enable_report_win_url_generation = true; + TestBuyerReportingSignals test_buyer_reporting_signals; + + ScoreAdsRequest request; + absl::flat_hash_map interest_group_to_ad; + BuildTopLevelAuctionScoreAdsRequest(&request); + ScoreAdsResponse response; + SellerCodeWrappingTestHelper( + &request, &response, kTopLevelSellerCode, + enable_seller_debug_url_generation, enable_debug_reporting, + enable_adtech_code_logging, enable_report_result_url_generation, + enable_report_win_url_generation); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + raw_response.ParseFromString(response.response_ciphertext()); + const auto& scoredAd = raw_response.ad_score(); + EXPECT_GT(scoredAd.desirability(), 0); + EXPECT_EQ(scoredAd.win_reporting_urls() + .top_level_seller_reporting_urls() + .reporting_url(), + kTestTopLevelReportResultUrl); + EXPECT_EQ(scoredAd.win_reporting_urls() + .top_level_seller_reporting_urls() + .interaction_reporting_urls() + .at(kTestInteractionEvent), + kTestInteractionReportingUrl); + EXPECT_EQ(scoredAd.win_reporting_urls() + .component_seller_reporting_urls() + .reporting_url(), + kTestComponentWinReportingUrl); + EXPECT_EQ(scoredAd.win_reporting_urls() + .component_seller_reporting_urls() + .interaction_reporting_urls() + .at(kTestComponentEvent), + kTestComponentInteractionReportingUrl); + + EXPECT_EQ( + scoredAd.win_reporting_urls().buyer_reporting_urls().reporting_url(), + kTestComponentWinReportingUrl); + EXPECT_EQ(scoredAd.win_reporting_urls() + .buyer_reporting_urls() + .interaction_reporting_urls() + .at(kTestComponentEvent), + kTestComponentInteractionReportingUrl); +} + TEST_F(AuctionServiceIntegrationTest, ModifiedBidSetToBuyerBidInComponentAuctionsReportUrlIfNotPresent) { bool enable_seller_debug_url_generation = false; @@ -992,9 +1093,11 @@ TEST_F(AuctionServiceIntegrationTest, ScoreAdsRequest request; absl::flat_hash_map interest_group_to_ad; + TestScoreAdsRequestConfig test_score_ads_request_config = { + .test_buyer_reporting_signals = test_buyer_reporting_signals, + .top_level_seller = kTestTopLevelSeller}; BuildScoreAdsRequestForReporting(&request, &interest_group_to_ad, - test_buyer_reporting_signals, false, 90, - kTestTopLevelSeller); + test_score_ads_request_config); ScoreAdsResponse response; SellerCodeWrappingTestHelper( &request, &response, kComponentAuctionCodeWithNoModifiedBid, @@ -1185,24 +1288,6 @@ TEST_F(AuctionServiceIntegrationTest, FiltersUnallowedAdsForComponentAuction) { EXPECT_FALSE(raw_response.has_ad_score()); } -void BuildTopLevelAuctionScoreAdsRequest( - ScoreAdsRequest* request, bool enable_debug_reporting = false, - int desired_ad_count = 20, absl::string_view seller = kTestSeller) { - ScoreAdsRequest::ScoreAdsRawRequest raw_request; - std::string generation_id = MakeARandomString(); - for (int i = 0; i < desired_ad_count; i++) { - auto auction_result = - MakeARandomComponentAuctionResult(generation_id, absl::StrCat(seller)); - *raw_request.mutable_component_auction_results()->Add() = auction_result; - } - if (enable_debug_reporting) { - raw_request.set_enable_debug_reporting(enable_debug_reporting); - } - raw_request.set_seller(seller); - *request->mutable_request_ciphertext() = raw_request.SerializeAsString(); - request->set_key_id(kKeyId); -} - TEST_F(AuctionServiceIntegrationTest, ProcessTopLevelAuctionInputAndReturnWinner) { ScoreAdsRequest request; diff --git a/services/auction_service/auction_service_integration_test_util.cc b/services/auction_service/auction_service_integration_test_util.cc new file mode 100644 index 00000000..d0ac9edb --- /dev/null +++ b/services/auction_service/auction_service_integration_test_util.cc @@ -0,0 +1,258 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/auction_service/auction_service_integration_test_util.h" + +#include +#include +#include +#include +#include + +#include "absl/random/random.h" +#include "absl/strings/str_format.h" +#include "google/protobuf/text_format.h" +#include "gtest/gtest.h" +#include "services/auction_service/auction_constants.h" +#include "services/auction_service/auction_service.h" +#include "services/auction_service/benchmarking/score_ads_benchmarking_logger.h" +#include "services/auction_service/benchmarking/score_ads_no_op_logger.h" +#include "services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h" +#include "services/auction_service/code_wrapper/seller_code_wrapper.h" +#include "services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h" +#include "services/auction_service/code_wrapper/seller_udf_wrapper.h" +#include "services/auction_service/code_wrapper/seller_udf_wrapper_test_constants.h" +#include "services/auction_service/udf_fetcher/adtech_code_version_util.h" +#include "services/common/clients/code_dispatcher/code_dispatch_client.h" +#include "services/common/clients/config/trusted_server_config_client.h" +#include "services/common/constants/common_service_flags.h" +#include "services/common/encryption/key_fetcher_factory.h" +#include "services/common/encryption/mock_crypto_client_wrapper.h" +#include "services/common/metric/server_definition.h" +#include "services/common/test/mocks.h" +#include "services/common/test/random.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { +using ::google::protobuf::TextFormat; +using AdWithBidMetadata = + ScoreAdsRequest::ScoreAdsRawRequest::AdWithBidMetadata; +using ::testing::AnyNumber; + +constexpr absl::string_view kKeyId = "key_id"; +constexpr absl::string_view kSecret = "secret"; +constexpr absl::string_view kTestConsentToken = "testConsentToken"; +constexpr absl::string_view kEurosIsoCode = "EUR"; +constexpr absl::string_view kPublisherHostname = "fenceStreetJournal.com"; + +AdWithBidMetadata GetTestAdWithBidMetadata( + const TestScoreAdsRequestConfig& test_score_ads_request_config) { + ScoreAdsRequest::ScoreAdsRawRequest::AdWithBidMetadata ad; + ad.mutable_ad()->mutable_struct_value()->MergeFrom( + MakeAnAd(MakeARandomString(), MakeARandomString(), 2)); + ad.set_bid(1.0); + ad.add_ad_components("adComponent.com"); + if (test_score_ads_request_config.buyer_reporting_id.has_value()) { + ad.set_buyer_reporting_id( + *test_score_ads_request_config.buyer_reporting_id); + } + ad.set_bid_currency(kEurosIsoCode); + ad.set_ad_cost( + test_score_ads_request_config.test_buyer_reporting_signals.ad_cost); + ad.set_modeling_signals(test_score_ads_request_config + .test_buyer_reporting_signals.modeling_signals); + ad.set_join_count( + test_score_ads_request_config.test_buyer_reporting_signals.join_count); + ad.set_interest_group_name( + test_score_ads_request_config.test_buyer_reporting_signals + .interest_group_name); + ad.set_interest_group_owner( + test_score_ads_request_config.interest_group_owner); + ad.set_render(absl::StrFormat( + "%s/ads", test_score_ads_request_config.interest_group_owner)); + ad.set_recency( + test_score_ads_request_config.test_buyer_reporting_signals.recency); + return ad; +} + +ScoreAdsRequest BuildScoreAdsRequest( + const TestScoreAdsRequestConfig& test_score_ads_request_config, + const std::vector& + ads) { + ScoreAdsRequest::ScoreAdsRawRequest raw_request; + std::string trusted_scoring_signals = + R"json({"renderUrls":{"placeholder_url":[123])json"; + for (const auto& ad : ads) { + raw_request.mutable_per_buyer_signals()->try_emplace( + ad.interest_group_owner(), + test_score_ads_request_config.test_buyer_reporting_signals + .buyer_signals); + std::string ad_signal = absl::StrFormat( + "\"%s\":%s", ad.render(), R"JSON(["short", "test", "signal"])JSON"); + absl::StrAppend(&trusted_scoring_signals, + absl::StrFormat(", %s", ad_signal)); + *raw_request.mutable_ad_bids()->Add() = ad; + } + absl::StrAppend(&trusted_scoring_signals, + R"json(},"adComponentRenderUrls":{}})json"); + raw_request.set_scoring_signals(trusted_scoring_signals); + if (test_score_ads_request_config.enable_debug_reporting) { + raw_request.set_enable_debug_reporting( + test_score_ads_request_config.enable_debug_reporting); + } + raw_request.set_seller("http://seller.com"); + raw_request.set_publisher_hostname(kPublisherHostname); + raw_request.set_auction_signals( + test_score_ads_request_config.test_buyer_reporting_signals + .auction_signals); + raw_request.mutable_consented_debug_config()->set_is_consented( + test_score_ads_request_config.is_consented); + raw_request.mutable_consented_debug_config()->set_token(kTestConsentToken); + if (!test_score_ads_request_config.top_level_seller.empty()) { + raw_request.set_top_level_seller( + test_score_ads_request_config.top_level_seller); + } + ScoreAdsRequest request; + *request.mutable_request_ciphertext() = raw_request.SerializeAsString(); + request.set_key_id(kKeyId); + return request; +} + +void SetupMockCryptoClientWrapper(MockCryptoClientWrapper& crypto_client) { + // Mock the HpkeDecrypt() call on the crypto_client. This is used by the + // service to decrypt the incoming request. + + EXPECT_CALL(crypto_client, HpkeDecrypt) + .Times(AnyNumber()) + .WillRepeatedly([](const server_common::PrivateKey& private_key, + absl::string_view ciphertext) { + google::cmrt::sdk::crypto_service::v1::HpkeDecryptResponse + hpke_decrypt_response; + hpke_decrypt_response.set_payload(ciphertext); + hpke_decrypt_response.set_secret(kSecret); + return hpke_decrypt_response; + }); + + // Mock the AeadEncrypt() call on the crypto_client. This is used to encrypt + // the response coming back from the service. + EXPECT_CALL(crypto_client, AeadEncrypt) + .Times(AnyNumber()) + .WillRepeatedly( + [](absl::string_view plaintext_payload, absl::string_view secret) { + google::cmrt::sdk::crypto_service::v1::AeadEncryptedData data; + data.set_ciphertext(plaintext_payload); + google::cmrt::sdk::crypto_service::v1::AeadEncryptResponse + aead_encrypt_response; + *aead_encrypt_response.mutable_encrypted_data() = std::move(data); + return aead_encrypt_response; + }); +} + +void InitV8Dispatcher(V8Dispatcher& dispatcher) { + ASSERT_TRUE(dispatcher.Init().ok()); +} + +void LoadTestSellerUdfWrapper( + absl::string_view adtech_code_blob, + const AuctionServiceRuntimeConfig& auction_service_runtime_config, + V8Dispatcher& dispatcher) { + std::string wrapper_js_blob = GetSellerWrappedCode( + adtech_code_blob, + auction_service_runtime_config.enable_report_result_url_generation); + ASSERT_TRUE(dispatcher.LoadSync(kScoreAdBlobVersion, wrapper_js_blob).ok()); +} + +void LoadTestBuyerUdfWrapper( + V8Dispatcher& dispatcher, const ScoreAdsRequest& request, + const AuctionServiceRuntimeConfig& auction_service_runtime_config, + AuctionType auction_type) { + ScoreAdsRequest::ScoreAdsRawRequest raw_request; + ASSERT_TRUE(raw_request.ParseFromString(request.request_ciphertext())); + if (!auction_service_runtime_config.enable_report_win_url_generation) { + return; + } + for (const auto& ad_bid : raw_request.ad_bids()) { + std::string wrapper_js_blob; + if (auction_service_runtime_config.enable_report_win_input_noising) { + wrapper_js_blob = GetBuyerWrappedCode(kBuyerBaseCodeWithValidation); + } else { + wrapper_js_blob = GetBuyerWrappedCode(kBuyerBaseCode); + } + absl::StatusOr version = + GetBuyerReportWinVersion(ad_bid.interest_group_owner(), auction_type); + ASSERT_TRUE(version.ok()); + ASSERT_TRUE(dispatcher.LoadSync(*version, wrapper_js_blob).ok()); + } +} + +void RunTestScoreAds( + CodeDispatchClient& client, ScoreAdsRequest& request, + const AuctionServiceRuntimeConfig& auction_service_runtime_config, + ScoreAdsResponse& response) { + grpc::CallbackServerContext context; + std::unique_ptr async_reporter = + std::make_unique( + std::make_unique()); + auto score_ads_reactor_factory = + [&client, async_reporter_local = std::move(async_reporter)]( + const ScoreAdsRequest* request, ScoreAdsResponse* response, + server_common::KeyFetcherManagerInterface* key_fetcher_manager, + CryptoClientWrapperInterface* crypto_client, + const AuctionServiceRuntimeConfig& runtime_config) { + return std::make_unique( + client, request, response, std::make_unique(), + key_fetcher_manager, crypto_client, async_reporter_local.get(), + runtime_config); + }; + + auto crypto_client = std::make_unique(); + SetupMockCryptoClientWrapper(*crypto_client); + TrustedServersConfigClient config_client({}); + config_client.SetFlagForTest(kTrue, TEST_MODE); + auto key_fetcher_manager = + CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr); + AuctionService service( + std::move(score_ads_reactor_factory), std::move(key_fetcher_manager), + std::move(crypto_client), auction_service_runtime_config); + + service.ScoreAds(&context, &request, &response); + std::this_thread::sleep_for(absl::ToChronoSeconds(absl::Seconds(2))); +} + +// Loads Seller's udf containing scoreAd() and reportResult() as well as +// buyer's udf containing reportWin() for Protected Audience into Roma. +void LoadPABuyerAndSellerCode( + const ScoreAdsRequest& request, V8Dispatcher& dispatcher, + const AuctionServiceRuntimeConfig& runtime_config) { + InitV8Dispatcher(dispatcher); + LoadTestSellerUdfWrapper(kSellerBaseCode, runtime_config, dispatcher); + LoadTestBuyerUdfWrapper(dispatcher, request, runtime_config, + AuctionType::kProtectedAudience); +} +} // namespace + +void LoadAndRunScoreAdsForPA( + const AuctionServiceRuntimeConfig& runtime_config, + const TestScoreAdsRequestConfig& test_score_ads_request_config, + ScoreAdsResponse& response) { + V8Dispatcher dispatcher; + CodeDispatchClient dispatch_client(dispatcher); + AdWithBidMetadata test_ad = + GetTestAdWithBidMetadata(test_score_ads_request_config); + ScoreAdsRequest request = + BuildScoreAdsRequest(test_score_ads_request_config, {test_ad}); + LoadPABuyerAndSellerCode(request, dispatcher, runtime_config); + RunTestScoreAds(dispatch_client, request, runtime_config, response); +} +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/auction_service_integration_test_util.h b/services/auction_service/auction_service_integration_test_util.h new file mode 100644 index 00000000..34849a55 --- /dev/null +++ b/services/auction_service/auction_service_integration_test_util.h @@ -0,0 +1,69 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_AUCTION_SERVICE_INTEGRATION_TEST_UTIL_H_ +#define SERVICES_AUCTION_SERVICE_INTEGRATION_TEST_UTIL_H_ + +#include + +#include "services/auction_service/auction_constants.h" +#include "services/auction_service/auction_service.h" + +namespace privacy_sandbox::bidding_auction_servers { + +constexpr absl::string_view kTestSeller = "http://seller.com"; +constexpr absl::string_view kTestInterestGroupName = "testInterestGroupName"; +constexpr double kTestAdCost = 2.0; +constexpr long kTestRecency = 3; +constexpr int kTestModelingSignals = 4; +constexpr int kTestJoinCount = 5; +constexpr absl::string_view kTestBuyerSignals = R"([1,"test",[2]])"; +constexpr absl::string_view kTestAuctionSignals = R"([[3,"test",[4]]])"; + +struct TestBuyerReportingSignals { + absl::string_view seller = kTestSeller; + absl::string_view interest_group_name = kTestInterestGroupName; + double ad_cost = kTestAdCost; + long recency = kTestRecency; + int modeling_signals = kTestModelingSignals; + int join_count = kTestJoinCount; + absl::string_view buyer_signals = kTestBuyerSignals; + absl::string_view auction_signals = kTestAuctionSignals; +}; + +struct TestScoreAdsRequestConfig { + const TestBuyerReportingSignals& test_buyer_reporting_signals; + bool enable_debug_reporting = false; + int desired_ad_count = 90; + std::string top_level_seller = ""; + std::string component_level_seller = ""; + bool is_consented = false; + std::optional buyer_reporting_id; + std::string interest_group_owner = ""; +}; + +// This function simulates a E2E successful call to ScoreAds in the +// Auction Service E2E for Protected Audience: +// - Loads a test protected audience udf for buyer and seller +// into Roma +// - Creates a test ScoreAdsRequest based on the TestScoreAdsConfig. +// - Calls ScoreAd using the request +// - Sets the response +void LoadAndRunScoreAdsForPA( + const AuctionServiceRuntimeConfig& runtime_config, + const TestScoreAdsRequestConfig& test_score_ads_request_config, + ScoreAdsResponse& response); +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_AUCTION_SERVICE_INTEGRATION_TEST_UTIL_H_ diff --git a/services/auction_service/auction_service_reporting_integration_test.cc b/services/auction_service/auction_service_reporting_integration_test.cc new file mode 100644 index 00000000..9d6c76e5 --- /dev/null +++ b/services/auction_service/auction_service_reporting_integration_test.cc @@ -0,0 +1,87 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "absl/random/random.h" +#include "absl/strings/str_format.h" +#include "absl/synchronization/blocking_counter.h" +#include "gtest/gtest.h" +#include "services/auction_service/auction_constants.h" +#include "services/auction_service/auction_service.h" +#include "services/auction_service/auction_service_integration_test_util.h" +#include "services/auction_service/code_wrapper/seller_code_wrapper.h" +#include "services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h" +#include "services/common/clients/code_dispatcher/code_dispatch_client.h" +#include "services/common/clients/config/trusted_server_config_client.h" +#include "services/common/constants/common_service_flags.h" +#include "services/common/encryption/key_fetcher_factory.h" +#include "services/common/encryption/mock_crypto_client_wrapper.h" +#include "services/common/metric/server_definition.h" +#include "services/common/test/mocks.h" +#include "services/common/test/random.h" +#include "services/common/util/request_response_constants.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { + +class AuctionServiceReportingIntegrationTest : public ::testing::Test { + protected: + void SetUp() override { + server_common::telemetry::TelemetryConfig config_proto; + config_proto.set_mode(server_common::telemetry::TelemetryConfig::OFF); + metric::MetricContextMap( + server_common::telemetry::BuildDependentConfig(config_proto)); + } +}; + +TEST_F(AuctionServiceReportingIntegrationTest, + ScoresAdsReturnsSuccessFullyWithSellerAndBuyerCodeIsolation) { + ScoreAdsResponse response; + AuctionServiceRuntimeConfig runtime_config = { + .enable_report_result_url_generation = true, + .enable_report_win_input_noising = false, + .enable_seller_and_buyer_udf_isolation = true}; + TestBuyerReportingSignals test_buyer_reporting_signals; + TestScoreAdsRequestConfig test_score_ads_request_config = { + .test_buyer_reporting_signals = test_buyer_reporting_signals, + .buyer_reporting_id = ""}; + LoadAndRunScoreAdsForPA(runtime_config, test_score_ads_request_config, + response); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + const auto& scoredAd = raw_response.ad_score(); + EXPECT_GT(scoredAd.desirability(), 0); + EXPECT_TRUE(scoredAd.win_reporting_urls() + .top_level_seller_reporting_urls() + .reporting_url() + .empty()); + EXPECT_EQ(scoredAd.win_reporting_urls() + .top_level_seller_reporting_urls() + .interaction_reporting_urls() + .size(), + 0); + EXPECT_TRUE(scoredAd.win_reporting_urls() + .buyer_reporting_urls() + .reporting_url() + .empty()); + EXPECT_EQ(scoredAd.win_reporting_urls() + .buyer_reporting_urls() + .interaction_reporting_urls() + .size(), + 0); +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/benchmarking/score_ads_reactor_benchmarks.cc b/services/auction_service/benchmarking/score_ads_reactor_benchmarks.cc index 2d81b83f..ea05cd38 100644 --- a/services/auction_service/benchmarking/score_ads_reactor_benchmarks.cc +++ b/services/auction_service/benchmarking/score_ads_reactor_benchmarks.cc @@ -47,11 +47,11 @@ constexpr char kTestBuyerSignals[] = "{\"test_key\":\"test_value\"}"; constexpr int kNumInterestGroups = 90; constexpr int kNumPasAds = 90; constexpr char kRenderUrlTemplate[] = - "https://googleads.g.doubleclick.net/td/adfetch/" - "gda?adg_id=142601302539&cr_id=628073386727&cv_id=0&distinguisher_id=%d"; + "https://adtech/" + "?adg_id=142601302539&cr_id=628073386727&cv_id=0&distinguisher_id=%d"; constexpr char kPasRenderUrlTemplate[] = - "https://googleads.g.doubleclick.pas.net/td/adfetch/" - "gda?adg_id=142601302539&cr_id=628073386727&cv_id=0&distinguisher_id=%d"; + "https://adtech/" + "?adg_id=142601302539&cr_id=628073386727&cv_id=0&distinguisher_id=%d"; constexpr char kInterestGroupTemplate[] = "InterestGroup-%d"; constexpr char kInterestGroupOwnerTemplate[] = "https://interest-group-owner.com/-%d"; diff --git a/services/auction_service/buyer_adtech_reporting_wrapper.h b/services/auction_service/buyer_adtech_reporting_wrapper.h deleted file mode 100644 index 6ef30004..00000000 --- a/services/auction_service/buyer_adtech_reporting_wrapper.h +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef FLEDGE_SERVICES_BUYER_ADTECH_REPORTING_WRAPPER_H_ -#define FLEDGE_SERVICES_BUYER_ADTECH_REPORTING_WRAPPER_H_ - -#include - -#include "absl/strings/str_cat.h" -#include "absl/strings/string_view.h" - -namespace privacy_sandbox::bidding_auction_servers { - -// Wrapper Javascript Wasm for the reportWin function provided by Buyer AdTech. -// This includes sendReport and registerAdBeacon functions. -inline constexpr absl::string_view kReportWinWrapperJavascriptWasm = R"JSCODE( - var ps_report_win_response = { - var reportWintUrl = undefined; - var interactionReportingUrls = new Map(); - var reportWinUrlGenerationErrorCount = 0; - var sendReportToInvoked = false; - var registerAdBeaconInvoked = false; - } - - globalThis.sendReportTo = function sendReportTo(url){ - if(!ps_report_win_response.sendReportToInvoked) { - throw new Exception("sendReportTo function invoked more than once"); - } - ps_report_win_response.reportWinUrl = url; - ps_report_win_response.sendReportToInvoked = true; - } - - globalThis.registerAdBeacon =function registerAdBeacon(eventUrlMap){ - if(!ps_report_win_response.registerAdBeaconInvoked) { - throw new Exception("registerAdBeaconInvoked function invoked more than once"); - } - for (const [key, value] of eventUrlMap) { - ps_report_win_response.interactionReportingUrls.set(key. value); - } - ps_report_win_response.sendReportToInvoked = true; - } - - // Handler method to call adTech provided reportWin method and wrap the - // response with reportWin url and interaction reporting urls. - function reportWinWrapper(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals) { - var reportWinResponse = {}; - try { - reportWin(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals); - } catch(ignored) { - ps_report_win_response.reportWinUrlGenerationErrorCount = 1; - } - return ps_report_win_response; -)JSCODE"; - -// Wraps the Ad Tech provided code for reportWin with wrapper code allowing -// Ad Techs to send URLs for event level and fenced frame reporting. -std::string GetWrappedAdtechCodeForReportWin( - absl::string_view adtech_code_blob) { - return absl::StrCat(kReportWinWrapperJavascriptWasm, adtech_code_blob); -} - -} // namespace privacy_sandbox::bidding_auction_servers - -#endif // FLEDGE_SERVICES_BUYER_ADTECH_REPORTING_WRAPPER_H_ diff --git a/services/auction_service/code_wrapper/BUILD b/services/auction_service/code_wrapper/BUILD index e6b11193..ec88d1ff 100644 --- a/services/auction_service/code_wrapper/BUILD +++ b/services/auction_service/code_wrapper/BUILD @@ -18,6 +18,32 @@ package(default_visibility = [ "//visibility:public", ]) +cc_library( + name = "buyer_reporting_udf_wrapper", + srcs = ["buyer_reporting_udf_wrapper.cc"], + hdrs = [ + "buyer_reporting_udf_wrapper.h", + ], + deps = [ + "@com_google_absl//absl/strings", + ], +) + +cc_test( + name = "buyer_reporting_udf_wrapper_test", + size = "small", + srcs = [ + "buyer_reporting_test_constants.h", + "buyer_reporting_udf_wrapper_test.cc", + ], + deps = [ + ":buyer_reporting_udf_wrapper", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + cc_library( name = "seller_code_wrapper", srcs = ["seller_code_wrapper.cc"], @@ -25,6 +51,7 @@ cc_library( "seller_code_wrapper.h", ], deps = [ + ":seller_udf_wrapper", "//services/common/util:reporting_util", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/functional:any_invocable", @@ -33,12 +60,24 @@ cc_library( ], ) +cc_library( + name = "seller_udf_wrapper", + srcs = ["seller_udf_wrapper.cc"], + hdrs = [ + "seller_udf_wrapper.h", + ], + deps = [ + "@com_google_absl//absl/strings", + ], +) + cc_test( name = "seller_code_wrapper_test", size = "small", srcs = [ "seller_code_wrapper_test.cc", "seller_code_wrapper_test_constants.h", + "seller_udf_wrapper_test_constants.h", ], deps = [ ":seller_code_wrapper", @@ -52,26 +91,17 @@ cc_test( ], ) -cc_library( - name = "buyer_reporting_fetcher", - srcs = ["buyer_reporting_fetcher.cc"], - hdrs = [ - "buyer_reporting_fetcher.h", +cc_test( + name = "seller_udf_wrapper_test", + size = "small", + srcs = [ + "seller_udf_wrapper_test.cc", + "seller_udf_wrapper_test_constants.h", ], deps = [ - "//services/auction_service:auction_code_fetch_config_cc_proto", - "//services/common/clients/http:http_fetcher_async", - "//services/common/code_fetch:code_fetcher_interface", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/functional:any_invocable", - "@com_google_absl//absl/log:absl_log", - "@com_google_absl//absl/status", + ":seller_udf_wrapper", "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - "@com_google_absl//absl/time", - "@google_privacysandbox_servers_common//src/concurrent:executor", - "@google_privacysandbox_servers_common//src/logger:request_context_logger", - "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", - "@google_privacysandbox_servers_common//src/util/status_macro:status_util", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", ], ) diff --git a/services/auction_service/code_wrapper/buyer_reporting_test_constants.h b/services/auction_service/code_wrapper/buyer_reporting_test_constants.h new file mode 100644 index 00000000..4f7dba57 --- /dev/null +++ b/services/auction_service/code_wrapper/buyer_reporting_test_constants.h @@ -0,0 +1,136 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef SERVICES_BUYER_REPORTING_TEST_CONSTANTS_H_ +#define SERVICES_BUYER_REPORTING_TEST_CONSTANTS_H_ + +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers { +constexpr absl::string_view kBuyerBaseCode = + R"JS_CODE(reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals){ + var test_render_url = buyerReportingSignals.renderUrl + var test_render_url = buyerReportingSignals.renderURL + if(buyerReportingSignals.seller==null || buyerReportingSignals.seller == undefined || buyerReportingSignals.seller == ""){ + console.error("Missing seller in input to reportWin") + return + } + if(buyerReportingSignals.adCost == 0 || buyerReportingSignals.adCost == -1 + || buyerReportingSignals.adCost == undefined + || buyerReportingSignals.adCost == null){ + console.error("Missing adCost in input to reportWin") + return + } + var reportWinUrl = "http://test.com?seller="+buyerReportingSignals.seller+ + "&interestGroupName="+buyerReportingSignals.interestGroupName+ + "&adCost="+buyerReportingSignals.adCost+"&modelingSignals="+ + buyerReportingSignals.modelingSignals+"&recency="+buyerReportingSignals.recency+ + "&madeHighestScoringOtherBid="+buyerReportingSignals.madeHighestScoringOtherBid+ + "&joinCount="+buyerReportingSignals.joinCount+"&signalsForWinner="+signalsForWinner+ + "&perBuyerSignals="+perBuyerSignals+"&auctionSignals="+auctionSignals+"&desirability="+buyerReportingSignals.desirability + if(buyerReportingSignals.hasOwnProperty("buyerReportingId")){ + reportWinUrl = reportWinUrl+"&buyerReportingId="+buyerReportingSignals.buyerReportingId + } + console.log("Logging from ReportWin"); + console.error("Logging error from ReportWin") + console.warn("Logging warning from ReportWin") + sendReportTo(reportWinUrl) + registerAdBeacon({"clickEvent":"http://click.com"}) + } +)JS_CODE"; + +constexpr absl::string_view kExpectedBuyerCodeWithReportWin = R"JS_CODE( + +function reportWinEntryFunction( + auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals, enable_logging, $extraArgs) { + const ps_report_win_response = { + reportWinUrl: '', + interactionReportingUrls: {}, + sendReportToInvoked: false, + registerAdBeaconInvoked: false, + }; + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } + globalThis.sendReportTo = function sendReportTo(url) { + if (ps_report_win_response.sendReportToInvoked) { + throw new Error('sendReportTo function invoked more than once'); + } + ps_report_win_response.reportWinUrl = url; + ps_report_win_response.sendReportToInvoked = true; + }; + globalThis.registerAdBeacon = function registerAdBeacon(eventUrlMap) { + if (ps_report_win_response.registerAdBeaconInvoked) { + throw new Error( + 'registerAdBeaconInvoked function invoked more than once'); + } + ps_report_win_response.interactionReportingUrls = eventUrlMap; + ps_report_win_response.registerAdBeaconInvoked = true; + }; + try { + reportWin( + auctionSignals, perBuyerSignals, signalsForWinner, + buyerReportingSignals, directFromSellerSignals, $extraArgs); + } catch (ex) { + console.error(ex.message); + } + return { + response: ps_report_win_response, + buyerLogs: ps_logs, + buyerErrors: ps_errors, + buyerWarnings: ps_warns + }; +} +reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals){ + var test_render_url = buyerReportingSignals.renderUrl + var test_render_url = buyerReportingSignals.renderURL + if(buyerReportingSignals.seller==null || buyerReportingSignals.seller == undefined || buyerReportingSignals.seller == ""){ + console.error("Missing seller in input to reportWin") + return + } + if(buyerReportingSignals.adCost == 0 || buyerReportingSignals.adCost == -1 + || buyerReportingSignals.adCost == undefined + || buyerReportingSignals.adCost == null){ + console.error("Missing adCost in input to reportWin") + return + } + var reportWinUrl = "http://test.com?seller="+buyerReportingSignals.seller+ + "&interestGroupName="+buyerReportingSignals.interestGroupName+ + "&adCost="+buyerReportingSignals.adCost+"&modelingSignals="+ + buyerReportingSignals.modelingSignals+"&recency="+buyerReportingSignals.recency+ + "&madeHighestScoringOtherBid="+buyerReportingSignals.madeHighestScoringOtherBid+ + "&joinCount="+buyerReportingSignals.joinCount+"&signalsForWinner="+signalsForWinner+ + "&perBuyerSignals="+perBuyerSignals+"&auctionSignals="+auctionSignals+"&desirability="+buyerReportingSignals.desirability + if(buyerReportingSignals.hasOwnProperty("buyerReportingId")){ + reportWinUrl = reportWinUrl+"&buyerReportingId="+buyerReportingSignals.buyerReportingId + } + console.log("Logging from ReportWin"); + console.error("Logging error from ReportWin") + console.warn("Logging warning from ReportWin") + sendReportTo(reportWinUrl) + registerAdBeacon({"clickEvent":"http://click.com"}) + } +)JS_CODE"; + +} // namespace privacy_sandbox::bidding_auction_servers +#endif // SERVICES_BUYER_REPORTING_TEST_CONSTANTS_H_ diff --git a/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.cc b/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.cc new file mode 100644 index 00000000..39816983 --- /dev/null +++ b/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.cc @@ -0,0 +1,27 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h" + +#include + +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers { + +std::string GetBuyerWrappedCode(absl::string_view buyer_js_code) { + return absl::StrCat(kReportWinWrapperFunction, buyer_js_code); +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h b/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h new file mode 100644 index 00000000..18b1b9bc --- /dev/null +++ b/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h @@ -0,0 +1,91 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_BUYER_REPORTING_UDF_WRAPPER_H_ +#define SERVICES_BUYER_REPORTING_UDF_WRAPPER_H_ + +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers { + +// The wrapper function which enables the below features for reportWin: +// - Exporting console.log, console.error and console.warn logs from UDF +// - sendReportTo() API to register the url to be pinged upon Auction win. +// - registerAdBeacon() API to register the interation reporting urls/ Fenced +// frame reporting urls. +constexpr absl::string_view kReportWinWrapperFunction = R"JS_CODE( + +function reportWinEntryFunction( + auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals, enable_logging, $extraArgs) { + const ps_report_win_response = { + reportWinUrl: '', + interactionReportingUrls: {}, + sendReportToInvoked: false, + registerAdBeaconInvoked: false, + }; + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } + globalThis.sendReportTo = function sendReportTo(url) { + if (ps_report_win_response.sendReportToInvoked) { + throw new Error('sendReportTo function invoked more than once'); + } + ps_report_win_response.reportWinUrl = url; + ps_report_win_response.sendReportToInvoked = true; + }; + globalThis.registerAdBeacon = function registerAdBeacon(eventUrlMap) { + if (ps_report_win_response.registerAdBeaconInvoked) { + throw new Error( + 'registerAdBeaconInvoked function invoked more than once'); + } + ps_report_win_response.interactionReportingUrls = eventUrlMap; + ps_report_win_response.registerAdBeaconInvoked = true; + }; + try { + reportWin( + auctionSignals, perBuyerSignals, signalsForWinner, + buyerReportingSignals, directFromSellerSignals, $extraArgs); + } catch (ex) { + console.error(ex.message); + } + return { + response: ps_report_win_response, + buyerLogs: ps_logs, + buyerErrors: ps_errors, + buyerWarnings: ps_warns + }; +} +)JS_CODE"; + +// Returns the complete wrapped code for buyer. +// The function adds wrapper to the buyer provided reportWin() udf +// The wrapper supports: +// - Generation of event level reporting urls for buyer +// - Exporting console.logs from the AdTech execution. +std::string GetBuyerWrappedCode(absl::string_view buyer_js_code); + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_BUYER_REPORTING_UDF_WRAPPER_H_ diff --git a/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper_test.cc b/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper_test.cc new file mode 100644 index 00000000..e522a2f6 --- /dev/null +++ b/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper_test.cc @@ -0,0 +1,30 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache-form License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h" + +#include "gtest/gtest.h" +#include "services/auction_service/code_wrapper/buyer_reporting_test_constants.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { +TEST(GetBuyerWrappedCode, GeneratesCompleteFinalCode) { + std::string observed = GetBuyerWrappedCode(kBuyerBaseCode); + std::cerr << "Code:\n" << observed << "\n"; + EXPECT_EQ(observed, kExpectedBuyerCodeWithReportWin) + << "Observed:\n" + << observed << "\n\n" + << "Expected: " << kExpectedBuyerCodeWithReportWin; +} +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/code_wrapper/seller_code_wrapper.cc b/services/auction_service/code_wrapper/seller_code_wrapper.cc index 2bd65540..dc078c3b 100644 --- a/services/auction_service/code_wrapper/seller_code_wrapper.cc +++ b/services/auction_service/code_wrapper/seller_code_wrapper.cc @@ -24,6 +24,7 @@ #include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "services/auction_service/code_wrapper/seller_udf_wrapper.h" #include "services/common/util/reporting_util.h" namespace privacy_sandbox::bidding_auction_servers { @@ -142,14 +143,4 @@ std::string GetSellerWrappedCode( return wrap_code; } -std::string GetSellerWrappedCode(absl::string_view seller_js_code, - bool enable_report_result_url_generation) { - std::string wrap_code{kEntryFunction}; - if (enable_report_result_url_generation) { - wrap_code.append(kReportResultWrapperFunction); - } - wrap_code.append(seller_js_code); - return wrap_code; -} - } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/code_wrapper/seller_code_wrapper.h b/services/auction_service/code_wrapper/seller_code_wrapper.h index 70b88723..103c961c 100644 --- a/services/auction_service/code_wrapper/seller_code_wrapper.h +++ b/services/auction_service/code_wrapper/seller_code_wrapper.h @@ -33,111 +33,6 @@ inline constexpr char kEgressPayloadTag[] = inline constexpr char kReportWinCodePlaceholder[] = "$reportWinCode"; inline constexpr char kReportWinWrapperFunctionName[] = "reportWinWrapper"; -// The function that will be called first by Roma. -// The dispatch function name will be scoreAdEntryFunction. -// This wrapper supports the features below: -//- Exporting logs to Auction Service using console.log -constexpr absl::string_view kEntryFunction = R"JS_CODE( - var forDebuggingOnly_auction_loss_url = undefined; - var forDebuggingOnly_auction_win_url = undefined; - const forDebuggingOnly = {}; - forDebuggingOnly.reportAdAuctionLoss = function(url){ - forDebuggingOnly_auction_loss_url = url; - } - forDebuggingOnly.reportAdAuctionWin = function(url){ - forDebuggingOnly_auction_win_url = url; - } - globalThis.forDebuggingOnly = forDebuggingOnly; - - function scoreAdEntryFunction(adMetadata, bid, auctionConfig, trustedScoringSignals, - browserSignals, directFromSellerSignals, featureFlags){ - const ps_logs = []; - const ps_errors = []; - const ps_warns = []; - if (featureFlags.enable_logging) { - console.log = (...args) => ps_logs.push(JSON.stringify(args)); - console.warn = (...args) => ps_warns.push(JSON.stringify(args)); - console.error = (...args) => ps_errors.push(JSON.stringify(args)); - } else { - console.log = console.warn = console.error = function() {}; - } - var scoreAdResponse = {}; - try { - scoreAdResponse = scoreAd(adMetadata, bid, auctionConfig, - trustedScoringSignals, browserSignals, directFromSellerSignals); - } catch({error, message}) { - console.error("[Error: " + error + "; Message: " + message + "]"); - } finally { - if( featureFlags.enable_debug_url_generation && - (forDebuggingOnly_auction_loss_url - || forDebuggingOnly_auction_win_url)) { - scoreAdResponse.debugReportUrls = { - auctionDebugLossUrl: forDebuggingOnly_auction_loss_url, - auctionDebugWinUrl: forDebuggingOnly_auction_win_url - } - } - } - return { - response: scoreAdResponse, - logs: ps_logs, - errors: ps_errors, - warnings: ps_warns - } - } -)JS_CODE"; - -inline constexpr absl::string_view kReportResultWrapperFunction = - R"JSCODE( - //Handler method to call adTech provided reportResult method and wrap the - // response with reportResult url and interaction reporting urls. - function reportingResultEntryFunction(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging) { - ps_signalsForWinner = "" - const ps_report_result_response = { - reportResultUrl : "", - interactionReportingUrls : {}, - sendReportToInvoked : false, - registerAdBeaconInvoked : false, - } - const ps_logs = []; - const ps_errors = []; - const ps_warns = []; - if (enable_logging) { - console.log = (...args) => ps_logs.push(JSON.stringify(args)); - console.warn = (...args) => ps_warns.push(JSON.stringify(args)); - console.error = (...args) => ps_errors.push(JSON.stringify(args)); - } else { - console.log = console.warn = console.error = function() {}; - } - globalThis.sendReportTo = function sendReportTo(url){ - if(ps_report_result_response.sendReportToInvoked) { - throw new Error("sendReportTo function invoked more than once"); - } - ps_report_result_response.reportResultUrl = url; - ps_report_result_response.sendReportToInvoked = true; - } - globalThis.registerAdBeacon = function registerAdBeacon(eventUrlMap){ - if(ps_report_result_response.registerAdBeaconInvoked) { - throw new Error("registerAdBeaconInvoked function invoked more than once"); - } - ps_report_result_response.interactionReportingUrls=eventUrlMap; - ps_report_result_response.registerAdBeaconInvoked = true; - } - try{ - ps_signalsForWinner = reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals); - } catch(ex){ - console.error(ex.message) - } - return { - signalsForWinner: ps_signalsForWinner, - interactionReportingUrls: ps_report_result_response.interactionReportingUrls, - reportResultUrl: ps_report_result_response.reportResultUrl - logs: ps_logs, - errors: ps_errors, - warnings: ps_warns - } - } -)JSCODE"; - // The function that will be called by Roma to generate reporting urls. // The dispatch function name will be reportingEntryFunction. // This wrapper supports the features below: @@ -306,14 +201,6 @@ std::string GetSellerWrappedCode( const absl::flat_hash_map& buyer_origin_code_map, const absl::flat_hash_map& protected_app_signals_buyer_origin_code_map); -// Returns the complete wrapped code for Seller. -// The function adds wrappers to the Seller provided scoreAd and reportResult -// UDF. The wrapper supports: -// - Generation of event level reporting urls for Seller -// - Generation of event level debug reporting -// - Exporting console.logs from the AdTech execution. -std::string GetSellerWrappedCode(absl::string_view seller_js_code, - bool enable_report_result_url_generation); } // namespace privacy_sandbox::bidding_auction_servers #endif // FLEDGE_SERVICES_SELLER_CODE_WRAPPER_H_ diff --git a/services/auction_service/code_wrapper/seller_code_wrapper_test.cc b/services/auction_service/code_wrapper/seller_code_wrapper_test.cc index 28a0b167..4fc2cf90 100644 --- a/services/auction_service/code_wrapper/seller_code_wrapper_test.cc +++ b/services/auction_service/code_wrapper/seller_code_wrapper_test.cc @@ -19,6 +19,7 @@ #include "gtest/gtest.h" #include "rapidjson/document.h" #include "services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h" +#include "services/auction_service/code_wrapper/seller_udf_wrapper_test_constants.h" #include "services/common/util/json_util.h" #include "services/common/util/reporting_util.h" @@ -38,16 +39,6 @@ TEST(GetSellerWrappedCode, GeneratesCompleteFinalCode) { << "Expected: " << kExpectedFinalCode; } -TEST(GetSellerWrappedCode, GeneratesWrappedCodeWithScoreAdAndReportResult) { - bool enable_report_result_url_generation = true; - std::string observed = GetSellerWrappedCode( - kSellerBaseCode, enable_report_result_url_generation); - EXPECT_EQ(observed, kExpectedSellerCodeWithScoreAdAndReportResult) - << "Observed:\n" - << observed << "\n\n" - << "Expected: " << kExpectedSellerCodeWithScoreAdAndReportResult; -} - TEST(GetSellerWrappedCode, GeneratesCompleteFinalCodeWithProtectedAppSignals) { bool enable_report_result_url_generation = true; bool enable_report_win_url_generation = true; diff --git a/services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h b/services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h index 247f020a..ced5dd8a 100644 --- a/services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h +++ b/services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h @@ -24,14 +24,17 @@ constexpr char kTestReportResultUrl[] = "topWindowHostname=fenceStreetJournal.com&interestGroupOwner=" "barStandardAds.com"; constexpr char kTestComponentReportResultUrlWithEverything[] = - "http://test.com&topLevelSeller=topLevelSeller&componentSeller=http://" - "seller.com&bid=1&bidCurrency=EUR&modifiedBid=2&modifiedBidCurrency=USD&" + "http://" + "test.com&topLevelSeller=topLevelSeller&bid=1&bidCurrency=EUR&modifiedBid=" + "2&modifiedBidCurrency=USD&" "highestScoringOtherBid=1&highestScoringOtherBidCurrency=???&" "topWindowHostname=fenceStreetJournal.com&interestGroupOwner=" "barStandardAds.com"; constexpr char kTestComponentReportResultUrlWithNoModifiedBid[] = - "http://test.com&topLevelSeller=topLevelSeller&componentSeller=http://" - "seller.com&bid=1&modifiedBid=1"; + "http://test.com&topLevelSeller=topLevelSeller&bid=1&modifiedBid=1"; +constexpr char kTestTopLevelReportResultUrl[] = + "http://test.com&componentSeller=http://" + "componentSeller.com&bid=1&desirability=1"; constexpr char kTestInteractionEvent[] = "clickEvent"; constexpr char kTestInteractionReportingUrl[] = "http://click.com"; constexpr char kTestReportWinUrl[] = @@ -138,21 +141,15 @@ constexpr absl::string_view kProtectedAppSignalsBuyerBaseCode = } )JS_CODE"; -constexpr absl::string_view kSellerBaseCode = R"JS_CODE( - function fibonacci(num) { - if (num <= 1) return 1; - return fibonacci(num - 1) + fibonacci(num - 2); - } - +constexpr absl::string_view kComponentAuctionCode = R"JS_CODE( function scoreAd(ad_metadata, bid, auction_config, scoring_signals, bid_metadata, directFromSellerSignals){ - // Do a random amount of work to generate the score: - const score = fibonacci(Math.floor(Math.random() * 10 + 1)); - console.log("Logging from ScoreAd") - console.error("Logging error from ScoreAd") - console.warn("Logging warn from ScoreAd") return { - desirability: score, - allow_component_auction: false + ad: bid_metadata["topLevelSeller"], + desirability: 1, + bid: 2, + incomingBidInSellerCurrency: 1.868, + bidCurrency: "USD", + allowComponentAuction: true } } function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ @@ -160,30 +157,29 @@ constexpr absl::string_view kSellerBaseCode = R"JS_CODE( if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) } else { - sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller) + sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&modifiedBid="+sellerReportingSignals.modifiedBid+"&modifiedBidCurrency="+sellerReportingSignals.modifiedBidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) } registerAdBeacon({"clickEvent":"http://click.com"}) return "testSignalsForWinner" } + )JS_CODE"; -constexpr absl::string_view kComponentAuctionCode = R"JS_CODE( - function scoreAd(ad_metadata, bid, auction_config, scoring_signals, bid_metadata, directFromSellerSignals){ +constexpr absl::string_view kTopLevelSellerCode = R"JS_CODE( + function scoreAd(ad_metadata, bid, auction_config, scoring_signals, device_signals, directFromSellerSignals){ return { - ad: bid_metadata["topLevelSeller"], + ad: device_signals["topLevelSeller"], desirability: 1, bid: 2, - incomingBidInSellerCurrency: 1.868, - bidCurrency: "USD", allowComponentAuction: true } } function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ console.log("Logging from ReportResult"); - if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ - sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) - } else { - sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&modifiedBid="+sellerReportingSignals.modifiedBid+"&modifiedBidCurrency="+sellerReportingSignals.modifiedBidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) + if(sellerReportingSignals.componentSeller === undefined || sellerReportingSignals.componentSeller.length === 0){ + sendReportTo("http://test.com") + } else{ + sendReportTo("http://test.com&componentSeller="+sellerReportingSignals.componentSeller+"&bid="+sellerReportingSignals.bid+"&desirability="+sellerReportingSignals.desirability) } registerAdBeacon({"clickEvent":"http://click.com"}) return "testSignalsForWinner" @@ -204,7 +200,7 @@ constexpr absl::string_view kComponentAuctionCodeWithNoModifiedBid = R"JS_CODE( if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ sendReportTo("http://test.com") } else { - sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller+"&bid="+sellerReportingSignals.bid+"&modifiedBid="+sellerReportingSignals.modifiedBid) + sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&bid="+sellerReportingSignals.bid+"&modifiedBid="+sellerReportingSignals.modifiedBid) } registerAdBeacon({"clickEvent":"http://click.com"}) return "testSignalsForWinner" @@ -486,132 +482,6 @@ constexpr absl::string_view kExpectedFinalCode = R"JS_CODE( } )JS_CODE"; -constexpr absl::string_view kExpectedSellerCodeWithScoreAdAndReportResult = - R"JS_CODE( - var forDebuggingOnly_auction_loss_url = undefined; - var forDebuggingOnly_auction_win_url = undefined; - const forDebuggingOnly = {}; - forDebuggingOnly.reportAdAuctionLoss = function(url){ - forDebuggingOnly_auction_loss_url = url; - } - forDebuggingOnly.reportAdAuctionWin = function(url){ - forDebuggingOnly_auction_win_url = url; - } - globalThis.forDebuggingOnly = forDebuggingOnly; - - function scoreAdEntryFunction(adMetadata, bid, auctionConfig, trustedScoringSignals, - browserSignals, directFromSellerSignals, featureFlags){ - const ps_logs = []; - const ps_errors = []; - const ps_warns = []; - if (featureFlags.enable_logging) { - console.log = (...args) => ps_logs.push(JSON.stringify(args)); - console.warn = (...args) => ps_warns.push(JSON.stringify(args)); - console.error = (...args) => ps_errors.push(JSON.stringify(args)); - } else { - console.log = console.warn = console.error = function() {}; - } - var scoreAdResponse = {}; - try { - scoreAdResponse = scoreAd(adMetadata, bid, auctionConfig, - trustedScoringSignals, browserSignals, directFromSellerSignals); - } catch({error, message}) { - console.error("[Error: " + error + "; Message: " + message + "]"); - } finally { - if( featureFlags.enable_debug_url_generation && - (forDebuggingOnly_auction_loss_url - || forDebuggingOnly_auction_win_url)) { - scoreAdResponse.debugReportUrls = { - auctionDebugLossUrl: forDebuggingOnly_auction_loss_url, - auctionDebugWinUrl: forDebuggingOnly_auction_win_url - } - } - } - return { - response: scoreAdResponse, - logs: ps_logs, - errors: ps_errors, - warnings: ps_warns - } - } - - //Handler method to call adTech provided reportResult method and wrap the - // response with reportResult url and interaction reporting urls. - function reportingResultEntryFunction(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging) { - ps_signalsForWinner = "" - const ps_report_result_response = { - reportResultUrl : "", - interactionReportingUrls : {}, - sendReportToInvoked : false, - registerAdBeaconInvoked : false, - } - const ps_logs = []; - const ps_errors = []; - const ps_warns = []; - if (enable_logging) { - console.log = (...args) => ps_logs.push(JSON.stringify(args)); - console.warn = (...args) => ps_warns.push(JSON.stringify(args)); - console.error = (...args) => ps_errors.push(JSON.stringify(args)); - } else { - console.log = console.warn = console.error = function() {}; - } - globalThis.sendReportTo = function sendReportTo(url){ - if(ps_report_result_response.sendReportToInvoked) { - throw new Error("sendReportTo function invoked more than once"); - } - ps_report_result_response.reportResultUrl = url; - ps_report_result_response.sendReportToInvoked = true; - } - globalThis.registerAdBeacon = function registerAdBeacon(eventUrlMap){ - if(ps_report_result_response.registerAdBeaconInvoked) { - throw new Error("registerAdBeaconInvoked function invoked more than once"); - } - ps_report_result_response.interactionReportingUrls=eventUrlMap; - ps_report_result_response.registerAdBeaconInvoked = true; - } - try{ - ps_signalsForWinner = reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals); - } catch(ex){ - console.error(ex.message) - } - return { - signalsForWinner: ps_signalsForWinner, - interactionReportingUrls: ps_report_result_response.interactionReportingUrls, - reportResultUrl: ps_report_result_response.reportResultUrl - logs: ps_logs, - errors: ps_errors, - warnings: ps_warns - } - } - - function fibonacci(num) { - if (num <= 1) return 1; - return fibonacci(num - 1) + fibonacci(num - 2); - } - - function scoreAd(ad_metadata, bid, auction_config, scoring_signals, bid_metadata, directFromSellerSignals){ - // Do a random amount of work to generate the score: - const score = fibonacci(Math.floor(Math.random() * 10 + 1)); - console.log("Logging from ScoreAd") - console.error("Logging error from ScoreAd") - console.warn("Logging warn from ScoreAd") - return { - desirability: score, - allow_component_auction: false - } - } - function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ - console.log("Logging from ReportResult"); - if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ - sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) - } else { - sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller) - } - registerAdBeacon({"clickEvent":"http://click.com"}) - return "testSignalsForWinner" - } -)JS_CODE"; - constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( var forDebuggingOnly_auction_loss_url = undefined; var forDebuggingOnly_auction_win_url = undefined; @@ -1136,80 +1006,5 @@ constexpr absl::string_view kExpectedCodeWithReportWinDisabled = R"JS_CODE( } )JS_CODE"; -constexpr absl::string_view kExpectedCodeWithReportingDisabled = R"JS_CODE( - var forDebuggingOnly_auction_loss_url = undefined; - var forDebuggingOnly_auction_win_url = undefined; - const forDebuggingOnly = {}; - forDebuggingOnly.reportAdAuctionLoss = function(url){ - forDebuggingOnly_auction_loss_url = url; - } - forDebuggingOnly.reportAdAuctionWin = function(url){ - forDebuggingOnly_auction_win_url = url; - } - globalThis.forDebuggingOnly = forDebuggingOnly; - - function scoreAdEntryFunction(adMetadata, bid, auctionConfig, trustedScoringSignals, - browserSignals, directFromSellerSignals, featureFlags){ - const ps_logs = []; - const ps_errors = []; - const ps_warns = []; - if (featureFlags.enable_logging) { - console.log = (...args) => ps_logs.push(JSON.stringify(args)); - console.warn = (...args) => ps_warns.push(JSON.stringify(args)); - console.error = (...args) => ps_errors.push(JSON.stringify(args)); - } else { - console.log = console.warn = console.error = function() {}; - } - var scoreAdResponse = {}; - try { - scoreAdResponse = scoreAd(adMetadata, bid, auctionConfig, - trustedScoringSignals, browserSignals, directFromSellerSignals); - } catch({error, message}) { - console.error("[Error: " + error + "; Message: " + message + "]"); - } finally { - if( featureFlags.enable_debug_url_generation && - (forDebuggingOnly_auction_loss_url - || forDebuggingOnly_auction_win_url)) { - scoreAdResponse.debugReportUrls = { - auctionDebugLossUrl: forDebuggingOnly_auction_loss_url, - auctionDebugWinUrl: forDebuggingOnly_auction_win_url - } - } - } - return { - response: scoreAdResponse, - logs: ps_logs, - errors: ps_errors, - warnings: ps_warns - } - } - - function fibonacci(num) { - if (num <= 1) return 1; - return fibonacci(num - 1) + fibonacci(num - 2); - } - - function scoreAd(ad_metadata, bid, auction_config, scoring_signals, bid_metadata, directFromSellerSignals){ - // Do a random amount of work to generate the score: - const score = fibonacci(Math.floor(Math.random() * 10 + 1)); - console.log("Logging from ScoreAd") - console.error("Logging error from ScoreAd") - console.warn("Logging warn from ScoreAd") - return { - desirability: score, - allow_component_auction: false - } - } - function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ - console.log("Logging from ReportResult"); - if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ - sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) - } else { - sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller) - } - registerAdBeacon({"clickEvent":"http://click.com"}) - return "testSignalsForWinner" - } -)JS_CODE"; } // namespace privacy_sandbox::bidding_auction_servers #endif // FLEDGE_SERVICES_SELLER_CODE_WRAPPER_TEST_CONSTANTS_H_ diff --git a/services/auction_service/code_wrapper/seller_udf_wrapper.cc b/services/auction_service/code_wrapper/seller_udf_wrapper.cc new file mode 100644 index 00000000..f74e2b01 --- /dev/null +++ b/services/auction_service/code_wrapper/seller_udf_wrapper.cc @@ -0,0 +1,33 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/auction_service/code_wrapper/seller_udf_wrapper.h" + +#include + +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers { + +std::string GetSellerWrappedCode(absl::string_view seller_js_code, + bool enable_report_result_url_generation) { + std::string wrap_code{kEntryFunction}; + if (enable_report_result_url_generation) { + wrap_code.append(kReportResultWrapperFunction); + } + wrap_code.append(seller_js_code); + return wrap_code; +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/code_wrapper/seller_udf_wrapper.h b/services/auction_service/code_wrapper/seller_udf_wrapper.h new file mode 100644 index 00000000..2180fe89 --- /dev/null +++ b/services/auction_service/code_wrapper/seller_udf_wrapper.h @@ -0,0 +1,142 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_AUCTION_SERVICE_SELLER_UDF_WRAPPER_H_ +#define SERVICES_AUCTION_SERVICE_SELLER_UDF_WRAPPER_H_ + +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers { + +constexpr char kReportResultEntryFunction[] = "reportResultEntryFunction"; + +// The function that will be called first by Roma. +// The dispatch function name will be scoreAdEntryFunction. +// This wrapper supports the features below: +//- Exporting logs to Auction Service using console.log +constexpr absl::string_view kEntryFunction = R"JS_CODE( + var forDebuggingOnly_auction_loss_url = undefined; + var forDebuggingOnly_auction_win_url = undefined; + const forDebuggingOnly = {}; + forDebuggingOnly.reportAdAuctionLoss = function(url){ + forDebuggingOnly_auction_loss_url = url; + } + forDebuggingOnly.reportAdAuctionWin = function(url){ + forDebuggingOnly_auction_win_url = url; + } + globalThis.forDebuggingOnly = forDebuggingOnly; + + function scoreAdEntryFunction(adMetadata, bid, auctionConfig, trustedScoringSignals, + browserSignals, directFromSellerSignals, featureFlags){ + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (featureFlags.enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } + var scoreAdResponse = {}; + try { + scoreAdResponse = scoreAd(adMetadata, bid, auctionConfig, + trustedScoringSignals, browserSignals, directFromSellerSignals); + } catch({error, message}) { + console.error("[Error: " + error + "; Message: " + message + "]"); + } finally { + if( featureFlags.enable_debug_url_generation && + (forDebuggingOnly_auction_loss_url + || forDebuggingOnly_auction_win_url)) { + scoreAdResponse.debugReportUrls = { + auctionDebugLossUrl: forDebuggingOnly_auction_loss_url, + auctionDebugWinUrl: forDebuggingOnly_auction_win_url + } + } + } + return { + response: scoreAdResponse, + logs: ps_logs, + errors: ps_errors, + warnings: ps_warns + } + } +)JS_CODE"; + +inline constexpr absl::string_view kReportResultWrapperFunction = + R"JSCODE( + //Handler method to call adTech provided reportResult method and wrap the + // response with reportResult url and interaction reporting urls. + function reportResultEntryFunction(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging) { + ps_signalsForWinner = "" + const ps_report_result_response = { + reportResultUrl : "", + interactionReportingUrls : {}, + sendReportToInvoked : false, + registerAdBeaconInvoked : false, + } + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } + globalThis.sendReportTo = function sendReportTo(url){ + if(ps_report_result_response.sendReportToInvoked) { + throw new Error("sendReportTo function invoked more than once"); + } + ps_report_result_response.reportResultUrl = url; + ps_report_result_response.sendReportToInvoked = true; + } + globalThis.registerAdBeacon = function registerAdBeacon(eventUrlMap){ + if(ps_report_result_response.registerAdBeaconInvoked) { + throw new Error("registerAdBeaconInvoked function invoked more than once"); + } + ps_report_result_response.interactionReportingUrls=eventUrlMap; + ps_report_result_response.registerAdBeaconInvoked = true; + } + try{ + ps_signalsForWinner = reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals); + } catch(ex){ + console.error(ex.message) + } + return { + signalsForWinner: ps_signalsForWinner, + interactionReportingUrls: ps_report_result_response.interactionReportingUrls, + reportResultUrl: ps_report_result_response.reportResultUrl, + logs: ps_logs, + errors: ps_errors, + warnings: ps_warns + } + } +)JSCODE"; + +// Returns the complete wrapped code for Seller. +// The function adds wrappers to the Seller provided scoreAd and reportResult +// UDF. The wrapper supports: +// - Generation of event level reporting urls for Seller +// - Generation of event level debug reporting +// - Exporting console.logs from the AdTech execution. +std::string GetSellerWrappedCode(absl::string_view seller_js_code, + bool enable_report_result_url_generation); +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_AUCTION_SERVICE_SELLER_UDF_WRAPPER_H_ diff --git a/services/auction_service/code_wrapper/seller_udf_wrapper_test.cc b/services/auction_service/code_wrapper/seller_udf_wrapper_test.cc new file mode 100644 index 00000000..1a4a3734 --- /dev/null +++ b/services/auction_service/code_wrapper/seller_udf_wrapper_test.cc @@ -0,0 +1,40 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache-form License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "services/auction_service/code_wrapper/seller_udf_wrapper.h" + +#include "gtest/gtest.h" +#include "services/auction_service/code_wrapper/seller_udf_wrapper_test_constants.h" + +namespace privacy_sandbox::bidding_auction_servers { + +namespace { + +TEST(GetSellerWrappedCode, GeneratesWrappedCodeWithScoreAdAndReportResult) { + bool enable_report_result_url_generation = true; + std::string observed = GetSellerWrappedCode( + kSellerBaseCode, enable_report_result_url_generation); + EXPECT_EQ(observed, kExpectedSellerCodeWithScoreAdAndReportResult) + << "Observed:\n" + << observed << "\n\n" + << "Expected: " << kExpectedSellerCodeWithScoreAdAndReportResult; +} + +TEST(GetSellerWrappedCode, LoadsOnlySellerCodeWithReportResultDisabled) { + bool enable_report_result_url_generation = false; + EXPECT_EQ(GetSellerWrappedCode(kSellerBaseCode, + enable_report_result_url_generation), + kExpectedCodeWithReportingDisabled); +} +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/code_wrapper/seller_udf_wrapper_test_constants.h b/services/auction_service/code_wrapper/seller_udf_wrapper_test_constants.h new file mode 100644 index 00000000..e1f5523a --- /dev/null +++ b/services/auction_service/code_wrapper/seller_udf_wrapper_test_constants.h @@ -0,0 +1,252 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef SERVICES_AUCTION_SERVICE_SELLER_UDF_WRAPPER_TEST_CONSTANTS_H_ +#define SERVICES_AUCTION_SERVICE_SELLER_UDF_WRAPPER_TEST_CONSTANTS_H_ + +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers { + +constexpr absl::string_view kSellerBaseCode = R"JS_CODE( + function fibonacci(num) { + if (num <= 1) return 1; + return fibonacci(num - 1) + fibonacci(num - 2); + } + + function scoreAd(ad_metadata, bid, auction_config, scoring_signals, bid_metadata, directFromSellerSignals){ + // Do a random amount of work to generate the score: + const score = fibonacci(Math.floor(Math.random() * 10 + 1)); + console.log("Logging from ScoreAd") + console.error("Logging error from ScoreAd") + console.warn("Logging warn from ScoreAd") + return { + desirability: score, + allow_component_auction: false + } + } + function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ + console.log("Logging from ReportResult"); + if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ + sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) + } else { + sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller) + } + registerAdBeacon({"clickEvent":"http://click.com"}) + return "testSignalsForWinner" + } +)JS_CODE"; + +constexpr absl::string_view kExpectedSellerCodeWithScoreAdAndReportResult = + R"JS_CODE( + var forDebuggingOnly_auction_loss_url = undefined; + var forDebuggingOnly_auction_win_url = undefined; + const forDebuggingOnly = {}; + forDebuggingOnly.reportAdAuctionLoss = function(url){ + forDebuggingOnly_auction_loss_url = url; + } + forDebuggingOnly.reportAdAuctionWin = function(url){ + forDebuggingOnly_auction_win_url = url; + } + globalThis.forDebuggingOnly = forDebuggingOnly; + + function scoreAdEntryFunction(adMetadata, bid, auctionConfig, trustedScoringSignals, + browserSignals, directFromSellerSignals, featureFlags){ + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (featureFlags.enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } + var scoreAdResponse = {}; + try { + scoreAdResponse = scoreAd(adMetadata, bid, auctionConfig, + trustedScoringSignals, browserSignals, directFromSellerSignals); + } catch({error, message}) { + console.error("[Error: " + error + "; Message: " + message + "]"); + } finally { + if( featureFlags.enable_debug_url_generation && + (forDebuggingOnly_auction_loss_url + || forDebuggingOnly_auction_win_url)) { + scoreAdResponse.debugReportUrls = { + auctionDebugLossUrl: forDebuggingOnly_auction_loss_url, + auctionDebugWinUrl: forDebuggingOnly_auction_win_url + } + } + } + return { + response: scoreAdResponse, + logs: ps_logs, + errors: ps_errors, + warnings: ps_warns + } + } + + //Handler method to call adTech provided reportResult method and wrap the + // response with reportResult url and interaction reporting urls. + function reportResultEntryFunction(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging) { + ps_signalsForWinner = "" + const ps_report_result_response = { + reportResultUrl : "", + interactionReportingUrls : {}, + sendReportToInvoked : false, + registerAdBeaconInvoked : false, + } + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } + globalThis.sendReportTo = function sendReportTo(url){ + if(ps_report_result_response.sendReportToInvoked) { + throw new Error("sendReportTo function invoked more than once"); + } + ps_report_result_response.reportResultUrl = url; + ps_report_result_response.sendReportToInvoked = true; + } + globalThis.registerAdBeacon = function registerAdBeacon(eventUrlMap){ + if(ps_report_result_response.registerAdBeaconInvoked) { + throw new Error("registerAdBeaconInvoked function invoked more than once"); + } + ps_report_result_response.interactionReportingUrls=eventUrlMap; + ps_report_result_response.registerAdBeaconInvoked = true; + } + try{ + ps_signalsForWinner = reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals); + } catch(ex){ + console.error(ex.message) + } + return { + signalsForWinner: ps_signalsForWinner, + interactionReportingUrls: ps_report_result_response.interactionReportingUrls, + reportResultUrl: ps_report_result_response.reportResultUrl, + logs: ps_logs, + errors: ps_errors, + warnings: ps_warns + } + } + + function fibonacci(num) { + if (num <= 1) return 1; + return fibonacci(num - 1) + fibonacci(num - 2); + } + + function scoreAd(ad_metadata, bid, auction_config, scoring_signals, bid_metadata, directFromSellerSignals){ + // Do a random amount of work to generate the score: + const score = fibonacci(Math.floor(Math.random() * 10 + 1)); + console.log("Logging from ScoreAd") + console.error("Logging error from ScoreAd") + console.warn("Logging warn from ScoreAd") + return { + desirability: score, + allow_component_auction: false + } + } + function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ + console.log("Logging from ReportResult"); + if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ + sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) + } else { + sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller) + } + registerAdBeacon({"clickEvent":"http://click.com"}) + return "testSignalsForWinner" + } +)JS_CODE"; + +constexpr absl::string_view kExpectedCodeWithReportingDisabled = R"JS_CODE( + var forDebuggingOnly_auction_loss_url = undefined; + var forDebuggingOnly_auction_win_url = undefined; + const forDebuggingOnly = {}; + forDebuggingOnly.reportAdAuctionLoss = function(url){ + forDebuggingOnly_auction_loss_url = url; + } + forDebuggingOnly.reportAdAuctionWin = function(url){ + forDebuggingOnly_auction_win_url = url; + } + globalThis.forDebuggingOnly = forDebuggingOnly; + + function scoreAdEntryFunction(adMetadata, bid, auctionConfig, trustedScoringSignals, + browserSignals, directFromSellerSignals, featureFlags){ + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (featureFlags.enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } + var scoreAdResponse = {}; + try { + scoreAdResponse = scoreAd(adMetadata, bid, auctionConfig, + trustedScoringSignals, browserSignals, directFromSellerSignals); + } catch({error, message}) { + console.error("[Error: " + error + "; Message: " + message + "]"); + } finally { + if( featureFlags.enable_debug_url_generation && + (forDebuggingOnly_auction_loss_url + || forDebuggingOnly_auction_win_url)) { + scoreAdResponse.debugReportUrls = { + auctionDebugLossUrl: forDebuggingOnly_auction_loss_url, + auctionDebugWinUrl: forDebuggingOnly_auction_win_url + } + } + } + return { + response: scoreAdResponse, + logs: ps_logs, + errors: ps_errors, + warnings: ps_warns + } + } + + function fibonacci(num) { + if (num <= 1) return 1; + return fibonacci(num - 1) + fibonacci(num - 2); + } + + function scoreAd(ad_metadata, bid, auction_config, scoring_signals, bid_metadata, directFromSellerSignals){ + // Do a random amount of work to generate the score: + const score = fibonacci(Math.floor(Math.random() * 10 + 1)); + console.log("Logging from ScoreAd") + console.error("Logging error from ScoreAd") + console.warn("Logging warn from ScoreAd") + return { + desirability: score, + allow_component_auction: false + } + } + function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ + console.log("Logging from ReportResult"); + if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ + sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) + } else { + sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller) + } + registerAdBeacon({"clickEvent":"http://click.com"}) + return "testSignalsForWinner" + } +)JS_CODE"; +} // namespace privacy_sandbox::bidding_auction_servers +#endif // SERVICES_AUCTION_SERVICE_SELLER_UDF_WRAPPER_TEST_CONSTANTS_H_ diff --git a/services/auction_service/data/runtime_config.h b/services/auction_service/data/runtime_config.h index 9cf7108a..f9dc299e 100644 --- a/services/auction_service/data/runtime_config.h +++ b/services/auction_service/data/runtime_config.h @@ -55,6 +55,8 @@ struct AuctionServiceRuntimeConfig { // Default code version to pass to Roma. std::string default_code_version = kScoreAdBlobVersion; + // Temporary flag to enable seller and buyer udf isolation. + bool enable_seller_and_buyer_udf_isolation = false; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/reporting/BUILD b/services/auction_service/reporting/BUILD index 7e009d28..cf0253e5 100644 --- a/services/auction_service/reporting/BUILD +++ b/services/auction_service/reporting/BUILD @@ -38,6 +38,7 @@ cc_library( "//services/common:feature_flags", "//services/common/clients/code_dispatcher:code_dispatch_client", "//services/common/code_dispatch:code_dispatch_reactor", + "//services/common/loggers:request_log_context", "//services/common/util:json_util", "//services/common/util:post_auction_signals", "//services/common/util:reporting_util", @@ -45,7 +46,6 @@ cc_library( "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", "@google_privacysandbox_servers_common//src/util/status_macro:status_util", "@rapidjson", diff --git a/services/auction_service/reporting/buyer/BUILD b/services/auction_service/reporting/buyer/BUILD new file mode 100644 index 00000000..a2f91b7c --- /dev/null +++ b/services/auction_service/reporting/buyer/BUILD @@ -0,0 +1,45 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "pas_buyer_reporting_manager", + hdrs = [ + "pas_buyer_reporting_manager.h", + ], + visibility = ["//visibility:public"], + deps = [ + "//services/auction_service/reporting:reporting_helper", + "//services/auction_service/reporting:reporting_response", + "//services/common/clients/code_dispatcher:code_dispatch_client", + "@com_google_absl//absl/status:statusor", + "@rapidjson", + ], +) + +cc_library( + name = "pa_buyer_reporting_manager", + hdrs = [ + "pa_buyer_reporting_manager.h", + ], + visibility = ["//visibility:public"], + deps = [ + "//services/auction_service/reporting:reporting_helper", + "//services/auction_service/reporting:reporting_response", + "//services/common/clients/code_dispatcher:code_dispatch_client", + "@com_google_absl//absl/status:statusor", + "@rapidjson", + ], +) diff --git a/services/auction_service/reporting/buyer/pa_buyer_reporting_manager.h b/services/auction_service/reporting/buyer/pa_buyer_reporting_manager.h new file mode 100644 index 00000000..f3289a68 --- /dev/null +++ b/services/auction_service/reporting/buyer/pa_buyer_reporting_manager.h @@ -0,0 +1,43 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_AUCTION_SERVICE_REPORTING_BUYER_REPORTING_MANAGER_H_ +#define SERVICES_AUCTION_SERVICE_REPORTING_BUYER_REPORTING_MANAGER_H_ + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "rapidjson/document.h" +#include "services/auction_service/reporting/reporting_helper.h" +#include "services/auction_service/reporting/reporting_response.h" +#include "services/common/clients/code_dispatcher/code_dispatch_client.h" + +namespace privacy_sandbox::bidding_auction_servers { +// Generates the DispatchRequest and invokes reportWin() udf with +// report_win_callback function for Protected Audience auctions +absl::Status PerformPAReportWin( + const ReportingDispatchRequestConfig& dispatch_request_config, + const BuyerReportingMetadata& buyer_reporting_metadata, + const std::string& seller_signals, + const rapidjson::Document& seller_device_signals, + absl::AnyInvocable< + void(const std::vector>&)> + report_win_callback, + CodeDispatchClient& dispatcher); + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_AUCTION_SERVICE_REPORTING_BUYER_REPORTING_MANAGER_H_ diff --git a/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.h b/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.h new file mode 100644 index 00000000..ccdcbbc8 --- /dev/null +++ b/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.h @@ -0,0 +1,44 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_AUCTION_SERVICE_REPORTING_BUYER_REPORTING_MANAGER_H_ +#define SERVICES_AUCTION_SERVICE_REPORTING_BUYER_REPORTING_MANAGER_H_ + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "rapidjson/document.h" +#include "services/auction_service/reporting/reporting_helper.h" +#include "services/auction_service/reporting/reporting_response.h" +#include "services/common/clients/code_dispatcher/code_dispatch_client.h" + +namespace privacy_sandbox::bidding_auction_servers { + +// Generates the DispatchRequest and invokes reportWin() udf with +// report_win_callback for Protected App Signal auctions +absl::Status PerformPASReportWin( + const ReportingDispatchRequestConfig& dispatch_request_config, + const BuyerReportingMetadata& buyer_reporting_metadata, + const std::string& seller_signals, + const rapidjson::Document& seller_device_signals, + absl::AnyInvocable< + void(const std::vector>&)> + report_win_callback, + CodeDispatchClient& dispatcher); + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_AUCTION_SERVICE_REPORTING_BUYER_REPORTING_MANAGER_H_ diff --git a/services/auction_service/reporting/reporting_helper.cc b/services/auction_service/reporting/reporting_helper.cc index 7619587a..e049412b 100644 --- a/services/auction_service/reporting/reporting_helper.cc +++ b/services/auction_service/reporting/reporting_helper.cc @@ -110,13 +110,7 @@ absl::StatusOr ParseAndGetReportingResponse( return reporting_response; } -// Stochastically rounds floating point value to 8 bit mantissa and 8 -// bit exponent. If there is an error while generating the rounded -// number, -1 will be returned. -double GetEightBitRoundedValue(bool enable_noising, double value) { - if (!enable_noising) { - return value; - } +double GetEightBitRoundedValue(double value) { absl::StatusOr rounded_value = RoundStochasticallyToKBits(value, kStochasticalRoundingBits); if (rounded_value.ok()) { @@ -125,6 +119,17 @@ double GetEightBitRoundedValue(bool enable_noising, double value) { return -1; } +// Stochastically rounds floating point value to 8 bit mantissa and 8 +// bit exponent. If there is an error while generating the rounded +// number, -1 will be returned. Returns the value as it is if enable_noising +// is false +double GetEightBitRoundedValue(bool enable_noising, double value) { + if (!enable_noising) { + return value; + } + return GetEightBitRoundedValue(value); +} + absl::StatusOr GetSellerReportingSignals( const ReportingDispatchRequestData& dispatch_request_data, const ReportingDispatchRequestConfig& dispatch_request_config) { @@ -148,12 +153,12 @@ absl::StatusOr GetSellerReportingSignals( dispatch_request_data.post_auction_signals.winning_ig_owner.c_str(), document.GetAllocator()); - document.AddMember(kTopWindowHostname, hostname_value, + document.AddMember(kTopWindowHostnameTag, hostname_value, document.GetAllocator()); document.AddMember(kInterestGroupOwner, interest_group_owner, document.GetAllocator()); - document.AddMember(kRenderURL, render_URL_value, document.GetAllocator()); - document.AddMember(kRenderUrl, render_url_value, document.GetAllocator()); + document.AddMember(kRenderURLTag, render_URL_value, document.GetAllocator()); + document.AddMember(kRenderUrlTag, render_url_value, document.GetAllocator()); double bid = GetEightBitRoundedValue( dispatch_request_config.enable_report_win_input_noising, dispatch_request_data.post_auction_signals.winning_bid); @@ -184,10 +189,10 @@ absl::StatusOr GetSellerReportingSignals( dispatch_request_config.enable_report_win_input_noising, dispatch_request_data.post_auction_signals.winning_score); if (desirability > -1) { - document.AddMember(kDesirability, desirability, document.GetAllocator()); + document.AddMember(kDesirabilityTag, desirability, document.GetAllocator()); } document.AddMember( - kHighestScoringOtherBid, + kHighestScoringOtherBidTag, dispatch_request_data.post_auction_signals.highest_scoring_other_bid, document.GetAllocator()); @@ -198,11 +203,6 @@ absl::StatusOr GetSellerReportingSignals( dispatch_request_data.component_reporting_metadata.top_level_seller .c_str(), document.GetAllocator()); - rapidjson::Value component_seller; - component_seller.SetString( - dispatch_request_data.component_reporting_metadata.component_seller - .c_str(), - document.GetAllocator()); document.AddMember(kTopLevelSellerTag, top_level_seller, document.GetAllocator()); double modified_bid = GetEightBitRoundedValue( @@ -221,9 +221,18 @@ absl::StatusOr GetSellerReportingSignals( document.AddMember(kModifiedBidCurrencyTag, modified_bid_currency, document.GetAllocator()); } + } + if (!dispatch_request_data.component_reporting_metadata.component_seller + .empty()) { + rapidjson::Value component_seller; + component_seller.SetString( + dispatch_request_data.component_reporting_metadata.component_seller + .c_str(), + document.GetAllocator()); document.AddMember(kComponentSeller, component_seller, document.GetAllocator()); } + return SerializeJsonDoc(document); } @@ -415,6 +424,7 @@ std::vector> GetReportingInput( << *(input[ReportingArgIndex(ReportingArgs::kDirectFromSellerSignals)]) << "\nBuyer Reporting Metadata:\n" << *(input[ReportingArgIndex(ReportingArgs::kBuyerReportingMetadata)]); + return input; } @@ -430,5 +440,4 @@ DispatchRequest GetReportingDispatchRequest( GetReportingInput(dispatch_request_config, dispatch_request_data), }; } - } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/reporting/reporting_helper.h b/services/auction_service/reporting/reporting_helper.h index b27ccc1d..e031db07 100644 --- a/services/auction_service/reporting/reporting_helper.h +++ b/services/auction_service/reporting/reporting_helper.h @@ -25,8 +25,8 @@ #include "api/bidding_auction_servers.pb.h" #include "services/auction_service/reporting/reporting_response.h" #include "services/common/clients/code_dispatcher/v8_dispatcher.h" +#include "services/common/loggers/request_log_context.h" #include "services/common/util/post_auction_signals.h" -#include "src/logger/request_context_impl.h" #include "src/util/status_macro/status_util.h" namespace privacy_sandbox::bidding_auction_servers { @@ -35,18 +35,22 @@ inline constexpr char kReportResultResponse[] = "reportResultResponse"; inline constexpr char kReportResultUrl[] = "reportResultUrl"; inline constexpr char kSendReportToInvoked[] = "sendReportToInvoked"; inline constexpr char kRegisterAdBeaconInvoked[] = "registerAdBeaconInvoked"; -inline constexpr char kTopWindowHostname[] = "topWindowHostname"; -inline constexpr char kRenderURL[] = "renderURL"; -inline constexpr char kRenderUrl[] = "renderUrl"; -inline constexpr char kDesirability[] = "desirability"; -inline constexpr char kHighestScoringOtherBid[] = "highestScoringOtherBid"; +inline constexpr char kTopWindowHostnameTag[] = "topWindowHostname"; +inline constexpr char kRenderURLTag[] = "renderURL"; +inline constexpr char kRenderUrlTag[] = "renderUrl"; +inline constexpr char kDesirabilityTag[] = "desirability"; +inline constexpr char kHighestScoringOtherBidTag[] = "highestScoringOtherBid"; inline constexpr char kSellerLogs[] = "sellerLogs"; inline constexpr char kSellerErrors[] = "sellerErrors"; inline constexpr char kSellerWarnings[] = "sellerWarnings"; +inline constexpr char kReportingUdfLogs[] = "logs"; +inline constexpr char kReportingUdfErrors[] = "errors"; +inline constexpr char kReportingUdfWarnings[] = "warnings"; inline constexpr char kBuyerLogs[] = "buyerLogs"; inline constexpr char kBuyerErrors[] = "buyerErrors"; inline constexpr char kBuyerWarnings[] = "buyerWarnings"; inline constexpr int kReportingArgSize = 5; +static constexpr char kRomaTimeoutMs[] = "TimeoutMs"; inline constexpr int kReportingArgSizeWithProtectedAppSignals = 7; inline constexpr char kReportingDispatchHandlerFunctionName[] = "reportingEntryFunction"; @@ -75,6 +79,8 @@ inline constexpr char kMadeHighestScoringOtherBid[] = "madeHighestScoringOtherBid"; inline constexpr char kInteractionReportingUrlsWrapperResponse[] = "interactionReportingUrls"; +inline constexpr char kBidTag[] = "bid"; +inline constexpr char kInterestGroupOwnerTag[] = "interestGroupOwner"; enum class ReportingArgs : int { kAuctionConfig = 0, @@ -115,6 +121,7 @@ struct ReportingDispatchRequestConfig { bool enable_protected_app_signals = false; bool enable_report_win_input_noising = false; bool enable_adtech_code_logging = false; + std::string roma_timeout_ms; }; // Data required to build the inputs required to dispatch @@ -124,7 +131,7 @@ struct ReportingDispatchRequestData { std::shared_ptr auction_config; PostAuctionSignals post_auction_signals; std::string_view publisher_hostname; - server_common::log::ContextImpl& log_context; + RequestLogContext& log_context; BuyerReportingMetadata buyer_reporting_metadata; ComponentReportingMetadata component_reporting_metadata; absl::string_view egress_payload; @@ -147,6 +154,16 @@ struct SellerReportingMetadata { std::string highest_scoring_other_bid_currency; }; +// Data required to build the inputs for +// reportResult() execution. +struct SellerReportingDispatchRequestData { + std::shared_ptr auction_config; + PostAuctionSignals& post_auction_signals; + std::string_view publisher_hostname; + ComponentReportingMetadata component_reporting_metadata; + RequestLogContext& log_context; +}; + inline const std::string kDefaultBuyerReportingMetadata = absl::StrFormat( R"JSON( { @@ -179,6 +196,11 @@ DispatchRequest GetReportingDispatchRequest( const ReportingDispatchRequestConfig& dispatch_request_config, const ReportingDispatchRequestData& dispatch_request_data); +// Stochastically rounds floating point value to 8 bit mantissa and 8 +// bit exponent. If there is an error while generating the rounded +// number, -1 will be returned. +double GetEightBitRoundedValue(double value); + } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_AUCTION_SERVICE_REPORTING_REPORTING_HELPER_H_ diff --git a/services/auction_service/reporting/reporting_helper_test.cc b/services/auction_service/reporting/reporting_helper_test.cc index 0810f002..2f462eba 100644 --- a/services/auction_service/reporting/reporting_helper_test.cc +++ b/services/auction_service/reporting/reporting_helper_test.cc @@ -14,6 +14,7 @@ #include "services/auction_service/reporting/reporting_helper.h" #include +#include #include #include "absl/status/status.h" @@ -183,16 +184,16 @@ BuyerReportingMetadata GetTestBuyerReportingMetadata() { ReportingDispatchRequestData GetTestDispatchRequestData( const ScoreAdsResponse::AdScore& winning_ad_score, const ReportingDispatchRequestConfig& dispatch_request_config, - const std::string& handler_name, absl::string_view seller_currency) { - server_common::log::ContextImpl log_context( - {}, server_common::ConsentedDebugConfiguration()); + const std::string& handler_name, std::string seller_currency) { + RequestLogContext log_context(/*context_map=*/{}, + server_common::ConsentedDebugConfiguration()); std::shared_ptr auction_config = std::make_shared(kTestAuctionConfig); ReportingDispatchRequestData reporting_dispatch_request_data = { .handler_name = handler_name, .auction_config = auction_config, - .post_auction_signals = - GeneratePostAuctionSignals(winning_ad_score, seller_currency), + .post_auction_signals = GeneratePostAuctionSignals( + winning_ad_score, std::move(seller_currency)), .publisher_hostname = kTestPublisherHostName, .log_context = log_context}; if (dispatch_request_config.enable_report_win_url_generation) { @@ -211,10 +212,10 @@ ReportingDispatchRequestData GetTestDispatchRequestData( ReportingDispatchRequestData GetTestComponentDispatchRequestData( const ScoreAdsResponse::AdScore& winning_ad_score, const ReportingDispatchRequestConfig& dispatch_request_config, - const std::string& handler_name, absl::string_view seller_currency) { + const std::string& handler_name, std::string seller_currency) { ReportingDispatchRequestData reporting_dispatch_request_data = GetTestDispatchRequestData(winning_ad_score, dispatch_request_config, - handler_name, seller_currency); + handler_name, std::move(seller_currency)); reporting_dispatch_request_data.component_reporting_metadata = { .top_level_seller = kTestTopLevelSeller, .component_seller = kTestSeller, @@ -226,11 +227,11 @@ ReportingDispatchRequestData GetTestComponentDispatchRequestData( ReportingDispatchRequestData GetTestComponentDispatchRequestDataForPAS( const ScoreAdsResponse::AdScore& winning_ad_score, const ReportingDispatchRequestConfig& dispatch_request_config, - absl::string_view seller_currency) { + std::string seller_currency) { ReportingDispatchRequestData reporting_dispatch_request_data = GetTestDispatchRequestData(winning_ad_score, dispatch_request_config, kReportingProtectedAppSignalsFunctionName, - seller_currency); + std::move(seller_currency)); reporting_dispatch_request_data.egress_payload = kTestEgressPayload; return reporting_dispatch_request_data; } @@ -528,8 +529,8 @@ TEST(GetReportingDispatchRequest, winning_ad_score.set_render(kTestRender); std::shared_ptr auction_config = std::make_shared(kTestAuctionConfig); - server_common::log::ContextImpl log_context( - {}, server_common::ConsentedDebugConfiguration()); + RequestLogContext log_context(/*context_map=*/{}, + server_common::ConsentedDebugConfiguration()); ReportingDispatchRequestConfig dispatch_request_config = { .enable_report_win_url_generation = true, .enable_report_win_input_noising = true, diff --git a/services/auction_service/reporting/seller/BUILD b/services/auction_service/reporting/seller/BUILD new file mode 100644 index 00000000..f886d430 --- /dev/null +++ b/services/auction_service/reporting/seller/BUILD @@ -0,0 +1,73 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") + +cc_library( + name = "seller_reporting_manager", + srcs = [ + "seller_reporting_manager.cc", + ], + hdrs = ["seller_reporting_manager.h"], + visibility = ["//visibility:public"], + deps = [ + "//services/auction_service/code_wrapper:seller_udf_wrapper", + "//services/auction_service/reporting:reporting_helper", + "//services/auction_service/reporting:reporting_response", + "//services/auction_service/udf_fetcher:adtech_code_version_util", + "//services/common/clients/code_dispatcher:code_dispatch_client", + "//services/common/util:json_util", + "@com_google_absl//absl/status:statusor", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + "@rapidjson", + ], +) + +cc_library( + name = "component_seller_reporting_manager", + hdrs = ["component_seller_reporting_manager.h"], + visibility = ["//visibility:public"], + deps = [ + ":seller_reporting_manager", + "//services/auction_service/reporting:reporting_helper", + "//services/auction_service/reporting:reporting_response", + "//services/common/clients/code_dispatcher:code_dispatch_client", + "@com_google_absl//absl/status:statusor", + "@rapidjson", + ], +) + +cc_test( + name = "seller_reporting_manager_test", + size = "small", + srcs = [ + "seller_reporting_manager_test.cc", + ], + visibility = ["//visibility:public"], + deps = [ + ":seller_reporting_manager", + "//services/auction_service/reporting:reporting_helper", + "//services/auction_service/reporting:reporting_helper_test", + "//services/auction_service/reporting:reporting_response", + "//services/auction_service/udf_fetcher:adtech_code_version_util", + "//services/common/clients/code_dispatcher:code_dispatch_client", + "//services/common/code_dispatch:code_dispatch_reactor", + "//services/common/test:mocks", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + "@rapidjson", + ], +) diff --git a/services/auction_service/reporting/seller/component_seller_reporting_manager.h b/services/auction_service/reporting/seller/component_seller_reporting_manager.h new file mode 100644 index 00000000..e965020b --- /dev/null +++ b/services/auction_service/reporting/seller/component_seller_reporting_manager.h @@ -0,0 +1,48 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_AUCTION_SERVICE_COMPONENT_SELLER_REPORTING_MANAGER_H_ +#define SERVICES_AUCTION_SERVICE_COMPONENT_SELLER_REPORTING_MANAGER_H_ + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "rapidjson/document.h" +#include "services/auction_service/reporting/reporting_helper.h" +#include "services/auction_service/reporting/reporting_response.h" +#include "services/auction_service/reporting/seller/seller_reporting_manager.h" +#include "services/common/clients/code_dispatcher/code_dispatch_client.h" + +namespace privacy_sandbox::bidding_auction_servers { + +// Generates device signals for reportResult input for Component auctions +rapidjson::Document GenerateSellerDeviceSignalsForComponentAuction( + const SellerReportingDispatchRequestData& request_data); + +// Generates the DispatchRequest, invokes reportResult() with the +// report_result_callback function for component seller. +absl::Status PerformReportResultForComponentAuction( + const ReportingDispatchRequestConfig& dispatch_request_config, + const rapidjson::Document& seller_device_signals, + const SellerReportingDispatchRequestData& request_data, + absl::AnyInvocable< + void(const std::vector>&)> + report_result_callback, + CodeDispatchClient& dispatcher); +} // namespace privacy_sandbox::bidding_auction_servers + +#endif +// SERVICES_AUCTION_SERVICE_COMPONENT_SELLER_REPORTING_MANAGER_H_ diff --git a/services/auction_service/reporting/seller/seller_reporting_manager.cc b/services/auction_service/reporting/seller/seller_reporting_manager.cc new file mode 100644 index 00000000..5d2f9ecf --- /dev/null +++ b/services/auction_service/reporting/seller/seller_reporting_manager.cc @@ -0,0 +1,160 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +#include "services/auction_service/reporting/seller/seller_reporting_manager.h" + +#include + +#include "absl/status/statusor.h" +#include "services/auction_service/code_wrapper/seller_udf_wrapper.h" +#include "services/auction_service/reporting/reporting_helper.h" +#include "services/auction_service/reporting/reporting_response.h" +#include "services/auction_service/udf_fetcher/adtech_code_version_util.h" +#include "services/common/util/json_util.h" +#include "services/common/util/request_response_constants.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { + +inline std::vector> GetReportResultInput( + const std::string& seller_device_signals, + const ReportingDispatchRequestConfig& dispatch_request_config, + const SellerReportingDispatchRequestData& dispatch_request_data) { + std::vector> input( + kReportResultArgSize); // ReportResultArgs size + + input[ReportResultArgIndex(ReportResultArgs::kAuctionConfig)] = + dispatch_request_data.auction_config; + input[ReportResultArgIndex(ReportResultArgs::kSellerReportingSignals)] = + std::make_shared(seller_device_signals); + // This is only added to prevent errors in the ReportResult ad script, and + // will always be an empty object. + input[ReportResultArgIndex(ReportResultArgs::kDirectFromSellerSignals)] = + std::make_shared("{}"); + input[ReportResultArgIndex(ReportResultArgs::kEnableAdTechCodeLogging)] = + std::make_shared( + dispatch_request_config.enable_adtech_code_logging ? "true" + : "false"); + PS_VLOG(kDispatch, dispatch_request_data.log_context) + << "\n\nReportResult Input Args:" << "\nAuction Config:\n" + << *(input[ReportResultArgIndex(ReportResultArgs::kAuctionConfig)]) + << "\nSeller Device Signals:\n" + << *(input[ReportResultArgIndex( + ReportResultArgs::kSellerReportingSignals)]) + << "\nEnable AdTech Code Logging:\n" + << *(input[ReportResultArgIndex( + ReportResultArgs::kEnableAdTechCodeLogging)]) + << "\nDirect from Seller Signals:\n" + << *(input[ReportResultArgIndex( + ReportResultArgs::kDirectFromSellerSignals)]); + + return input; +} + +inline DispatchRequest GetReportResultDispatchRequest( + const ReportingDispatchRequestConfig& dispatch_request_config, + const SellerReportingDispatchRequestData& request_data, + const std::string& seller_device_signals_json) { + // Construct the wrapper struct for our V8 Dispatch Request. + return {.id = request_data.post_auction_signals.winning_ad_render_url, + .version_string = GetDefaultSellerUdfVersion(), + .handler_name = kReportResultEntryFunction, + .input = GetReportResultInput(seller_device_signals_json, + dispatch_request_config, request_data)}; +} +} // namespace +rapidjson::Document GenerateSellerDeviceSignals( + const SellerReportingDispatchRequestData& dispatch_request_data) { + rapidjson::Document document(rapidjson::kObjectType); + // Convert the std::string to a rapidjson::Value object. + rapidjson::Value hostname_value( + dispatch_request_data.publisher_hostname.data(), document.GetAllocator()); + rapidjson::Value render_URL_value( + dispatch_request_data.post_auction_signals.winning_ad_render_url.c_str(), + document.GetAllocator()); + rapidjson::Value render_url_value( + dispatch_request_data.post_auction_signals.winning_ad_render_url.c_str(), + document.GetAllocator()); + + rapidjson::Value interest_group_owner( + dispatch_request_data.post_auction_signals.winning_ig_owner.c_str(), + document.GetAllocator()); + + document.AddMember(kTopWindowHostnameTag, hostname_value.Move(), + document.GetAllocator()); + document.AddMember(kInterestGroupOwnerTag, interest_group_owner.Move(), + document.GetAllocator()); + document.AddMember(kRenderURLTag, render_URL_value.Move(), + document.GetAllocator()); + document.AddMember(kRenderUrlTag, render_url_value.Move(), + document.GetAllocator()); + double bid = GetEightBitRoundedValue( + dispatch_request_data.post_auction_signals.winning_bid); + if (bid > -1) { + document.AddMember(kBidTag, bid, document.GetAllocator()); + } + if (!dispatch_request_data.post_auction_signals.winning_bid_currency + .empty()) { + rapidjson::Value winning_bid_currency_value( + dispatch_request_data.post_auction_signals.winning_bid_currency.c_str(), + document.GetAllocator()); + document.AddMember(kWinningBidCurrencyTag, + winning_bid_currency_value.Move(), + document.GetAllocator()); + } + if (!dispatch_request_data.post_auction_signals + .highest_scoring_other_bid_currency.empty()) { + rapidjson::Value highest_scoring_other_bid_currency_value( + dispatch_request_data.post_auction_signals + .highest_scoring_other_bid_currency.c_str(), + document.GetAllocator()); + document.AddMember(kHighestScoringOtherBidCurrencyTag, + highest_scoring_other_bid_currency_value.Move(), + document.GetAllocator()); + } + double desirability = GetEightBitRoundedValue( + dispatch_request_data.post_auction_signals.winning_score); + if (desirability > -1) { + document.AddMember(kDesirabilityTag, desirability, document.GetAllocator()); + } + document.AddMember( + kHighestScoringOtherBidTag, + dispatch_request_data.post_auction_signals.highest_scoring_other_bid, + document.GetAllocator()); + + return document; +} + +absl::Status PerformReportResult( + const ReportingDispatchRequestConfig& dispatch_request_config, + const rapidjson::Document& seller_device_signals, + const SellerReportingDispatchRequestData& request_data, + absl::AnyInvocable< + void(const std::vector>&)> + report_result_callback, + CodeDispatchClient& dispatcher) { + std::string seller_device_signals_json; + PS_ASSIGN_OR_RETURN(seller_device_signals_json, + SerializeJsonDoc(seller_device_signals)); + DispatchRequest dispatch_request = GetReportResultDispatchRequest( + dispatch_request_config, request_data, seller_device_signals_json); + dispatch_request.tags[kRomaTimeoutMs] = + dispatch_request_config.roma_timeout_ms; + std::vector dispatch_requests = { + std::move(dispatch_request)}; + return dispatcher.BatchExecute(dispatch_requests, + std::move(report_result_callback)); +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/reporting/seller/seller_reporting_manager.h b/services/auction_service/reporting/seller/seller_reporting_manager.h new file mode 100644 index 00000000..b1a7c288 --- /dev/null +++ b/services/auction_service/reporting/seller/seller_reporting_manager.h @@ -0,0 +1,60 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_AUCTION_SERVICE_REPORTING_SELLER_AND_BUYER_REPORTING_MANAGER_H_ +#define SERVICES_AUCTION_SERVICE_REPORTING_SELLER_AND_BUYER_REPORTING_MANAGER_H_ + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "rapidjson/document.h" +#include "services/auction_service/reporting/reporting_helper.h" +#include "services/auction_service/reporting/reporting_response.h" +#include "services/common/clients/code_dispatcher/code_dispatch_client.h" + +namespace privacy_sandbox::bidding_auction_servers { + +inline constexpr int kReportResultArgSize = 4; + +enum class ReportResultArgs : int { + kAuctionConfig, + kSellerReportingSignals, + kDirectFromSellerSignals, + kEnableAdTechCodeLogging +}; + +inline constexpr int ReportResultArgIndex(ReportResultArgs arg) { + return static_cast>(arg); +} + +// Generates device signals for reportResult input +rapidjson::Document GenerateSellerDeviceSignals( + const SellerReportingDispatchRequestData& request_data); + +// Generates the DispatchRequest, invokes reportResult() with the +// report_result_callback function for single seller auctions. +absl::Status PerformReportResult( + const ReportingDispatchRequestConfig& dispatch_request_config, + const rapidjson::Document& seller_device_signals, + const SellerReportingDispatchRequestData& request_data, + absl::AnyInvocable< + void(const std::vector>&)> + report_result_callback, + CodeDispatchClient& dispatcher); +} // namespace privacy_sandbox::bidding_auction_servers + +#endif +// SERVICES_AUCTION_SERVICE_REPORTING_SELLER_AND_BUYER_REPORTING_MANAGER_H_ diff --git a/services/auction_service/reporting/seller/seller_reporting_manager_test.cc b/services/auction_service/reporting/seller/seller_reporting_manager_test.cc new file mode 100644 index 00000000..9ef639d7 --- /dev/null +++ b/services/auction_service/reporting/seller/seller_reporting_manager_test.cc @@ -0,0 +1,199 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache-form License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "services/auction_service/reporting/seller/seller_reporting_manager.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "services/auction_service/auction_constants.h" +#include "services/auction_service/code_wrapper/seller_udf_wrapper.h" +#include "services/auction_service/reporting/reporting_helper.h" +#include "services/auction_service/reporting/reporting_helper_test_constants.h" +#include "services/auction_service/reporting/reporting_response.h" +#include "services/auction_service/udf_fetcher/adtech_code_version_util.h" +#include "services/common/clients/code_dispatcher/v8_dispatcher.h" +#include "services/common/test/mocks.h" +#include "services/common/util/json_util.h" + +namespace privacy_sandbox::bidding_auction_servers { + +namespace { + +constexpr absl::string_view kExpectedSellerDeviceSignals = + R"JSON({"topWindowHostname":"publisherName","interestGroupOwner":"testOwner","renderURL":"http://testurl.com","renderUrl":"http://testurl.com","bid":1.0,"bidCurrency":"EUR","highestScoringOtherBidCurrency":"USD","desirability":2.0,"highestScoringOtherBid":0.5})JSON"; + +SellerReportingDispatchRequestData GetTestDispatchRequestData( + PostAuctionSignals& post_auction_signals, RequestLogContext& log_context) { + std::shared_ptr auction_config = + std::make_shared(kTestAuctionConfig); + PS_LOG(INFO, log_context) << "test log"; + SellerReportingDispatchRequestData reporting_dispatch_request_data = { + .auction_config = auction_config, + .post_auction_signals = post_auction_signals, + .publisher_hostname = kTestPublisherHostName, + .component_reporting_metadata = {}, + .log_context = log_context}; + return reporting_dispatch_request_data; +} + +ScoreAdsResponse::AdScore GetTestWinningScoreAdsResponse() { + ScoreAdsResponse::AdScore winning_ad_score; + winning_ad_score.set_buyer_bid(kTestBuyerBid); + winning_ad_score.set_buyer_bid_currency(kEurosIsoCode); + winning_ad_score.set_interest_group_owner(kTestInterestGroupOwner); + winning_ad_score.set_interest_group_name(kTestInterestGroupName); + winning_ad_score.mutable_ig_owner_highest_scoring_other_bids_map() + ->try_emplace(kTestInterestGroupOwner, google::protobuf::ListValue()); + winning_ad_score.mutable_ig_owner_highest_scoring_other_bids_map() + ->at(kTestInterestGroupOwner) + .add_values() + ->set_number_value(kTestHighestScoringOtherBid); + winning_ad_score.set_desirability(kTestDesirability); + winning_ad_score.set_render(kTestRender); + return winning_ad_score; +} + +struct ReportResultArgIndices { + int kAuctionConfigArgIdx = + ReportResultArgIndex(ReportResultArgs::kAuctionConfig); + int kSellerDeviceSignalsIdx = + ReportResultArgIndex(ReportResultArgs::kSellerReportingSignals); + int kDirectFromSignalsIdx = + ReportResultArgIndex(ReportResultArgs::kDirectFromSellerSignals); + int kAdTechCodeLoggingIdx = + ReportResultArgIndex(ReportResultArgs::kEnableAdTechCodeLogging); +}; + +struct TestData { + absl::string_view expected_auction_config = kTestAuctionConfig; + absl::string_view expected_seller_device_signals = + kExpectedSellerDeviceSignals; + absl::string_view expected_direct_from_seller_signals = "{}"; + bool enable_adtech_code_logging = false; + absl::string_view id = kTestRender; + std::string expected_version = GetDefaultSellerUdfVersion(); + absl::string_view expected_handler_name = kReportResultEntryFunction; + ReportResultArgIndices indexes; +}; + +void VerifyDispatchRequest(const TestData& test_data, + std::vector& batch) { + EXPECT_EQ(batch.size(), 1); + std::string expected_auction_config = kTestAuctionConfig; + EXPECT_EQ(batch[0].id, test_data.id); + EXPECT_EQ(batch[0].version_string, test_data.expected_version); + EXPECT_EQ(batch[0].handler_name, test_data.expected_handler_name); + std::vector> response_vector = batch[0].input; + EXPECT_EQ(*(response_vector[test_data.indexes.kAuctionConfigArgIdx]), + test_data.expected_auction_config); + EXPECT_EQ(*(response_vector[test_data.indexes.kSellerDeviceSignalsIdx]), + test_data.expected_seller_device_signals); + EXPECT_EQ(*(response_vector[test_data.indexes.kDirectFromSignalsIdx]), + test_data.expected_direct_from_seller_signals); + EXPECT_EQ(*(response_vector[test_data.indexes.kAdTechCodeLoggingIdx]), + test_data.enable_adtech_code_logging ? "true" : "false"); +} + +TEST(TestSellerReportingManager, ReturnsRapidJsonDocOfSellerDeviceSignals) { + ScoreAdsResponse::AdScore winning_ad_score = GetTestWinningScoreAdsResponse(); + PostAuctionSignals post_auction_signals = + GeneratePostAuctionSignals(winning_ad_score, kUsdIsoCode); + RequestLogContext log_context(/*context_map=*/{}, + server_common::ConsentedDebugConfiguration()); + SellerReportingDispatchRequestData dispatch_request_data = + GetTestDispatchRequestData(post_auction_signals, log_context); + PS_LOG(INFO, dispatch_request_data.log_context) << "test log"; + rapidjson::Document json_doc = + GenerateSellerDeviceSignals(dispatch_request_data); + absl::StatusOr generatedjson = SerializeJsonDoc(json_doc); + ASSERT_TRUE(generatedjson.ok()); + EXPECT_EQ(*generatedjson, kExpectedSellerDeviceSignals); +} + +TEST(PerformReportResult, DispatchesRequestToReportResult) { + auto report_result_callback = + [](const std::vector>& result) {}; + absl::StatusOr document = + ParseJsonString(kExpectedSellerDeviceSignals); + MockCodeDispatchClient mock_dispatch_client; + ScoreAdsResponse::AdScore winning_ad_score = GetTestWinningScoreAdsResponse(); + PostAuctionSignals post_auction_signals = + GeneratePostAuctionSignals(winning_ad_score, kUsdIsoCode); + RequestLogContext log_context(/*context_map=*/{}, + server_common::ConsentedDebugConfiguration()); + + SellerReportingDispatchRequestData dispatch_request_data = + GetTestDispatchRequestData(post_auction_signals, log_context); + + ASSERT_TRUE(document.ok()) << document.status(); + ReportingDispatchRequestConfig config; + absl::Notification notification; + + EXPECT_CALL(mock_dispatch_client, BatchExecute) + .WillOnce([¬ification](std::vector& batch, + BatchDispatchDoneCallback done_callback) { + TestData test_data; + VerifyDispatchRequest(test_data, batch); + notification.Notify(); + return absl::OkStatus(); + }); + absl::Status status = PerformReportResult( + config, *document, dispatch_request_data, + std::move(report_result_callback), mock_dispatch_client); + + notification.WaitForNotification(); + ASSERT_TRUE(status.ok()); +} + +TEST(PerformReportResult, DispatchRequestFailsAndStatusNotOkReturned) { + auto report_result_callback = + [](const std::vector>& result) {}; + absl::StatusOr document = + ParseJsonString(kExpectedSellerDeviceSignals); + MockCodeDispatchClient mock_dispatch_client; + ScoreAdsResponse::AdScore winning_ad_score = GetTestWinningScoreAdsResponse(); + PostAuctionSignals post_auction_signals = + GeneratePostAuctionSignals(winning_ad_score, kUsdIsoCode); + RequestLogContext log_context(/*context_map=*/{}, + server_common::ConsentedDebugConfiguration()); + + SellerReportingDispatchRequestData dispatch_request_data = + GetTestDispatchRequestData(post_auction_signals, log_context); + + ASSERT_TRUE(document.ok()) << document.status(); + ReportingDispatchRequestConfig config; + absl::Notification notification; + EXPECT_CALL(mock_dispatch_client, BatchExecute) + .WillOnce([¬ification](std::vector& batch, + BatchDispatchDoneCallback done_callback) { + TestData test_data; + VerifyDispatchRequest(test_data, batch); + notification.Notify(); + return absl::InternalError("Something went wrong"); + }); + absl::Status status = PerformReportResult( + config, *document, dispatch_request_data, + std::move(report_result_callback), mock_dispatch_client); + + notification.WaitForNotification(); + ASSERT_FALSE(status.ok()); +} +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/score_ads_reactor.cc b/services/auction_service/score_ads_reactor.cc index 8671fc45..4057d119 100644 --- a/services/auction_service/score_ads_reactor.cc +++ b/services/auction_service/score_ads_reactor.cc @@ -48,14 +48,13 @@ using ProtectedAppSignalsAdWithBidMetadata = using ::google::protobuf::RepeatedPtrField; using HighestScoringOtherBidsMap = ::google::protobuf::Map; -using server_common::log::ContextImpl; using server_common::log::PS_VLOG_IS_ON; constexpr int kBytesMultiplyer = 1024; inline void MayVlogRomaResponses( const std::vector>& responses, - ContextImpl& log_context) { + RequestLogContext& log_context) { if (PS_VLOG_IS_ON(2)) { for (const auto& dispatch_response : responses) { PS_VLOG(kDispatch, log_context) @@ -69,7 +68,8 @@ inline void MayVlogRomaResponses( inline void LogWarningForBadResponse( const absl::Status& status, const DispatchResponse& response, - const AdWithBidMetadata* ad_with_bid_metadata, ContextImpl& log_context) { + const AdWithBidMetadata* ad_with_bid_metadata, + RequestLogContext& log_context) { PS_LOG(ERROR, log_context) << "Failed to parse response from Roma ", status.ToString(absl::StatusToStringMode::kWithEverything); if (ad_with_bid_metadata) { @@ -116,6 +116,62 @@ bool IsIncomingBidInSellerCurrencyIllegallyModified( kCurrencyFloatComparisonEpsilon); } +// Generates GetComponentReportingDataInAuctionResult based on auction_result. +ComponentReportingDataInAuctionResult GetComponentReportingDataInAuctionResult( + AuctionResult& auction_result) { + ComponentReportingDataInAuctionResult + component_reporting_data_in_auction_result; + component_reporting_data_in_auction_result.component_seller = + auction_result.auction_params().component_seller(); + component_reporting_data_in_auction_result.win_reporting_urls = + std::move(*auction_result.mutable_win_reporting_urls()); + return component_reporting_data_in_auction_result; +} + +// Populates reporting urls of component level seller and buyer in top level +// seller's ScoreAdResponse. +void PopulateComponentReportingUrlsInTopLevelResponse( + absl::string_view dispatch_id, + ScoreAdsResponse::ScoreAdsRawResponse& raw_response_, + absl::flat_hash_map& + component_level_reporting_data_) { + if (auto ad_it = component_level_reporting_data_.find(dispatch_id); + ad_it != component_level_reporting_data_.end()) { + auto* seller_reporting_urls = + ad_it->second.win_reporting_urls + .mutable_component_seller_reporting_urls(); + raw_response_.mutable_ad_score() + ->mutable_win_reporting_urls() + ->mutable_component_seller_reporting_urls() + ->set_reporting_url( + std::move(*seller_reporting_urls->mutable_reporting_url())); + for (auto& [event, url] : + *seller_reporting_urls->mutable_interaction_reporting_urls()) { + raw_response_.mutable_ad_score() + ->mutable_win_reporting_urls() + ->mutable_component_seller_reporting_urls() + ->mutable_interaction_reporting_urls() + ->try_emplace(event, std::move(url)); + } + + auto* component_buyer_reporting_urls = + ad_it->second.win_reporting_urls.mutable_buyer_reporting_urls(); + raw_response_.mutable_ad_score() + ->mutable_win_reporting_urls() + ->mutable_buyer_reporting_urls() + ->set_reporting_url(std::move( + *component_buyer_reporting_urls->mutable_reporting_url())); + for (auto& [event, url] : *component_buyer_reporting_urls + ->mutable_interaction_reporting_urls()) { + raw_response_.mutable_ad_score() + ->mutable_win_reporting_urls() + ->mutable_buyer_reporting_urls() + ->mutable_interaction_reporting_urls() + ->try_emplace(event, std::move(url)); + } + } +} + // If this is a component auction, and a seller currency is set, // and a modified bid is set, and a currency for that modofied bid is set, // it must match the seller currency. If not, reject the bid. @@ -187,8 +243,6 @@ ScoreAdsReactor::ScoreAdsReactor( enable_adtech_code_logging_(log_context_.is_consented()), enable_report_result_url_generation_( runtime_config.enable_report_result_url_generation), - enable_report_win_url_generation_( - runtime_config.enable_report_win_url_generation), enable_protected_app_signals_( runtime_config.enable_protected_app_signals), enable_report_win_input_noising_( @@ -198,7 +252,11 @@ ScoreAdsReactor::ScoreAdsReactor( max_allowed_size_all_debug_urls_chars_( kBytesMultiplyer * runtime_config.max_allowed_size_all_debug_urls_kb), auction_scope_(GetAuctionScope(raw_request_)), - code_version_(runtime_config.default_code_version) { + code_version_(runtime_config.default_code_version), + enable_report_win_url_generation_( + runtime_config.enable_report_win_url_generation && + auction_scope_ != + AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { CHECK_OK([this]() { PS_ASSIGN_OR_RETURN(metric_context_, metric::AuctionContextMap()->Remove(request_)); @@ -257,7 +315,22 @@ absl::Status ScoreAdsReactor::PopulateTopLevelAuctionDispatchRequests( << dispatch_request.status(); continue; } - + auto [unused_it_reporting, reporting_inserted] = + component_level_reporting_data_.emplace( + dispatch_request->id, + GetComponentReportingDataInAuctionResult(auction_result)); + if (!reporting_inserted) { + PS_VLOG(kNoisyWarn, log_context_) + << "Could not insert reporting data for Protected Audience ScoreAd " + "Request id " + "conflict detected: " + << dispatch_request->id; + LogIfError( + metric_context_ + ->AccumulateMetric( + 1, metric::kAuctionScoreAdsFailedToInsertDispatchRequest)); + continue; + } // Map all fields from a component auction result to a // AdWithBidMetadata used in this reactor. This way all the parsing // and result handling logic for single seller auctions can be @@ -270,6 +343,10 @@ absl::Status ScoreAdsReactor::PopulateTopLevelAuctionDispatchRequests( << "Protected Audience ScoreAd Request id " "conflict detected: " << dispatch_request->id; + LogIfError( + metric_context_ + ->AccumulateMetric( + 1, metric::kAuctionScoreAdsFailedToInsertDispatchRequest)); continue; } @@ -369,8 +446,10 @@ void ScoreAdsReactor::MayPopulateProtectedAppSignalsDispatchRequests( void ScoreAdsReactor::Execute() { PS_VLOG(kEncrypted, log_context_) << "Encrypted ScoreAdsRequest:\n" << request_->ShortDebugString(); + log_context_.SetEventMessageField(*request_); PS_VLOG(kPlain, log_context_) << "ScoreAdsRawRequest:\n" << raw_request_.ShortDebugString(); + log_context_.SetEventMessageField(raw_request_); DCHECK(raw_request_.protected_app_signals_ad_bids().empty() || enable_protected_app_signals_) @@ -467,18 +546,6 @@ void ScoreAdsReactor::Execute() { void ScoreAdsReactor::PerformReporting( const ScoreAdsResponse::AdScore& winning_ad_score, absl::string_view id) { - if (auction_scope_ == AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { - // TODO: Implement reporting for top level auction - // The following properties will not be available: - // raw_request_.per_buyer_signals() - // ad->join_count() - // ad->recency() - // ad->modeling_signals(), - // ad->ad_cost() - benchmarking_logger_->HandleResponseEnd(); - EncryptAndFinishOK(); - return; - } if (auto ad_it = ad_data_.find(id); ad_it != ad_data_.end()) { const auto& ad = ad_it->second; BuyerReportingMetadata buyer_reporting_metadata; @@ -498,7 +565,7 @@ void ScoreAdsReactor::PerformReporting( buyer_reporting_metadata.buyer_reporting_id = ad->buyer_reporting_id(); } } - DispatchReportingRequestForPA(winning_ad_score, + DispatchReportingRequestForPA(id, winning_ad_score, BuildAuctionConfig(raw_request_), buyer_reporting_metadata); @@ -879,12 +946,17 @@ void ScoreAdsReactor::ScoreAdsCallback( PerformDebugReporting(winning_ad); } *raw_response_.mutable_ad_score() = *winning_ad; - if (!enable_report_result_url_generation_) { + if (enable_report_result_url_generation_) { + PerformReporting(*winning_ad, id); + } else { + if (auction_scope_ == AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { + PopulateComponentReportingUrlsInTopLevelResponse( + id, raw_response_, component_level_reporting_data_); + } benchmarking_logger_->HandleResponseEnd(); EncryptAndFinishOK(); return; } - PerformReporting(*winning_ad, id); } void ScoreAdsReactor::ReportingCallback( @@ -909,47 +981,48 @@ void ScoreAdsReactor::ReportingCallback( absl::StatusToStringMode::kWithEverything); continue; } - if (PS_VLOG_IS_ON(1) && enable_adtech_code_logging_) { - for (std::string& log : reporting_response.value().seller_logs) { - PS_VLOG(kUdfLog, log_context_) - << "Log from Seller's execution script:" << log; - } - for (std::string& log : reporting_response.value().seller_error_logs) { - PS_LOG(ERROR, log_context_) - << "Error Log from Seller's execution script:" << log; - } - for (std::string& log : - reporting_response.value().seller_warning_logs) { - PS_LOG(WARNING, log_context_) - << "Warning Log from Seller's execution script:" << log; - } - for (std::string& log : reporting_response.value().buyer_logs) { - PS_VLOG(kUdfLog, log_context_) - << "Log from Buyer's execution script:" << log; + + auto LogUdf = [this](const std::vector& adtech_logs, + absl::string_view prefix) { + if (!enable_adtech_code_logging_) { + return; } - for (std::string& log : reporting_response.value().buyer_error_logs) { - PS_LOG(ERROR, log_context_) - << "Error Log from Buyer's execution script:" << log; + if (!PS_VLOG_IS_ON(kUdfLog) && !log_context_.is_debug_response()) { + return; } - for (std::string& log : reporting_response.value().buyer_warning_logs) { - PS_LOG(WARNING, log_context_) - << "Warning Log from Buyer's execution script:" << log; + for (const std::string& log : adtech_logs) { + std::string log_str = + absl::StrCat(prefix, " execution script: ", log); + PS_VLOG(kUdfLog, log_context_) << log_str; + log_context_.SetEventMessageField(log_str); } - } + }; + LogUdf(reporting_response->seller_logs, "Log from Seller's"); + LogUdf(reporting_response->seller_error_logs, "Error Log from Seller's"); + LogUdf(reporting_response->seller_warning_logs, + "Warning Log from Seller's"); + LogUdf(reporting_response->buyer_logs, "Log from Buyer's"); + LogUdf(reporting_response->buyer_error_logs, "Error Log from Buyer's"); + LogUdf(reporting_response->buyer_warning_logs, + "Warning Log from Buyer's"); + // For component auctions, the reporting urls for seller are set in // the component_seller_reporting_urls field. For single seller - // auctions and top level auctions, the reporting urls are set in the - // top_level_seller_reporting_urls field. + // auctions and top level auctions, the reporting urls for the top level + // seller are set in the top_level_seller_reporting_urls field. For top + // level auctions, the component_seller_reporting_urls and + // buyer_reporting_urls are set based on the urls obtained from the + // component auction whose buyer won the top level auction. if (auction_scope_ == AuctionScope::AUCTION_SCOPE_DEVICE_COMPONENT_MULTI_SELLER) { raw_response_.mutable_ad_score() ->mutable_win_reporting_urls() ->mutable_component_seller_reporting_urls() - ->set_reporting_url(reporting_response.value() - .report_result_response.report_result_url); + ->set_reporting_url( + reporting_response->report_result_response.report_result_url); for (const auto& [event, interactionReportingUrl] : - reporting_response.value() - .report_result_response.interaction_reporting_urls) { + reporting_response->report_result_response + .interaction_reporting_urls) { raw_response_.mutable_ad_score() ->mutable_win_reporting_urls() ->mutable_component_seller_reporting_urls() @@ -957,15 +1030,35 @@ void ScoreAdsReactor::ReportingCallback( ->try_emplace(event, interactionReportingUrl); } + } else if (auction_scope_ == + AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { + raw_response_.mutable_ad_score() + ->mutable_win_reporting_urls() + ->mutable_top_level_seller_reporting_urls() + ->set_reporting_url( + reporting_response->report_result_response.report_result_url); + for (const auto& [event, interactionReportingUrl] : + reporting_response->report_result_response + .interaction_reporting_urls) { + raw_response_.mutable_ad_score() + ->mutable_win_reporting_urls() + ->mutable_top_level_seller_reporting_urls() + ->mutable_interaction_reporting_urls() + ->try_emplace(event, interactionReportingUrl); + } + + PopulateComponentReportingUrlsInTopLevelResponse( + response->id, raw_response_, component_level_reporting_data_); + } else { raw_response_.mutable_ad_score() ->mutable_win_reporting_urls() ->mutable_top_level_seller_reporting_urls() - ->set_reporting_url(reporting_response.value() - .report_result_response.report_result_url); + ->set_reporting_url( + reporting_response->report_result_response.report_result_url); for (const auto& [event, interactionReportingUrl] : - reporting_response.value() - .report_result_response.interaction_reporting_urls) { + reporting_response->report_result_response + .interaction_reporting_urls) { raw_response_.mutable_ad_score() ->mutable_win_reporting_urls() ->mutable_top_level_seller_reporting_urls() @@ -973,20 +1066,22 @@ void ScoreAdsReactor::ReportingCallback( ->try_emplace(event, interactionReportingUrl); } } - raw_response_.mutable_ad_score() - ->mutable_win_reporting_urls() - ->mutable_buyer_reporting_urls() - ->set_reporting_url( - reporting_response.value().report_win_response.report_win_url); - - for (const auto& [event, interactionReportingUrl] : - reporting_response.value() - .report_win_response.interaction_reporting_urls) { + if (enable_report_win_url_generation_) { raw_response_.mutable_ad_score() ->mutable_win_reporting_urls() ->mutable_buyer_reporting_urls() - ->mutable_interaction_reporting_urls() - ->try_emplace(event, interactionReportingUrl); + ->set_reporting_url( + reporting_response->report_win_response.report_win_url); + + for (const auto& [event, interactionReportingUrl] : + reporting_response->report_win_response + .interaction_reporting_urls) { + raw_response_.mutable_ad_score() + ->mutable_win_reporting_urls() + ->mutable_buyer_reporting_urls() + ->mutable_interaction_reporting_urls() + ->try_emplace(event, interactionReportingUrl); + } } } else { LogIfError(metric_context_ @@ -1057,6 +1152,8 @@ void ScoreAdsReactor::PerformDebugReporting( void ScoreAdsReactor::EncryptAndFinishOK() { PS_VLOG(kPlain, log_context_) << "ScoreAdsRawResponse:\n" << raw_response_.ShortDebugString(); + log_context_.SetEventMessageField(raw_response_); + log_context_.ExportEventMessage(); EncryptResponse(); PS_VLOG(kEncrypted, log_context_) << "Encrypted ScoreAdsResponse\n" << response_->ShortDebugString(); @@ -1072,14 +1169,22 @@ void ScoreAdsReactor::FinishWithStatus(const grpc::Status& status) { } void ScoreAdsReactor::DispatchReportingRequestForPA( + absl::string_view dispatch_id, const ScoreAdsResponse::AdScore& winning_ad_score, const std::shared_ptr& auction_config, const BuyerReportingMetadata& buyer_reporting_metadata) { + PostAuctionSignals post_auction_signals; + if (auction_scope_ == AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { + post_auction_signals = + GeneratePostAuctionSignalsForTopLevelSeller(winning_ad_score); + } else { + post_auction_signals = GeneratePostAuctionSignals( + winning_ad_score, raw_request_.seller_currency()); + } ReportingDispatchRequestData dispatch_request_data = { .handler_name = kReportingDispatchHandlerFunctionName, .auction_config = auction_config, - .post_auction_signals = GeneratePostAuctionSignals( - winning_ad_score, raw_request_.seller_currency()), + .post_auction_signals = post_auction_signals, .publisher_hostname = raw_request_.publisher_hostname(), .log_context = log_context_, .buyer_reporting_metadata = buyer_reporting_metadata}; @@ -1090,6 +1195,16 @@ void ScoreAdsReactor::DispatchReportingRequestForPA( dispatch_request_data.component_reporting_metadata = { .top_level_seller = raw_request_.top_level_seller(), .component_seller = raw_request_.seller()}; + } else if (auction_scope_ == AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { + if (auto ad_it = component_level_reporting_data_.find(dispatch_id); + ad_it != component_level_reporting_data_.end()) { + dispatch_request_data.component_reporting_metadata = { + .component_seller = ad_it->second.component_seller}; + } else { + PS_LOG(ERROR, log_context_) << "Could not find component reporting data " + "for winner ad with dispatch id: " + << dispatch_id; + } } if (winning_ad_score.bid() > 0) { dispatch_request_data.component_reporting_metadata.modified_bid = diff --git a/services/auction_service/score_ads_reactor.h b/services/auction_service/score_ads_reactor.h index c0738b4f..de642f92 100644 --- a/services/auction_service/score_ads_reactor.h +++ b/services/auction_service/score_ads_reactor.h @@ -37,10 +37,10 @@ #include "services/common/clients/code_dispatcher/code_dispatch_client.h" #include "services/common/code_dispatch/code_dispatch_reactor.h" #include "services/common/encryption/crypto_client_wrapper_interface.h" +#include "services/common/loggers/request_log_context.h" #include "services/common/metric/server_definition.h" #include "services/common/reporters/async_reporter.h" #include "src/encryption/key_fetcher/interface/key_fetcher_manager_interface.h" -#include "src/logger/request_context_impl.h" namespace privacy_sandbox::bidding_auction_servers { @@ -73,6 +73,13 @@ struct ScoringData { void UpdateWinner(int index, const ScoreAdsResponse::AdScore& ad_score); }; +// Fields needed from component level AuctionResult for +// reporting in the Top level Seller +struct ComponentReportingDataInAuctionResult { + std::string component_seller; + WinReportingUrls win_reporting_urls; +}; + // This is a gRPC reactor that serves a single ScoreAdsRequest. // It stores state relevant to the request and after the // response is finished being served, ScoreAdsReactor cleans up all @@ -144,6 +151,7 @@ class ScoreAdsReactor static constexpr char kRomaTimeoutMs[] = "TimeoutMs"; void DispatchReportingRequestForPA( + absl::string_view dispatch_id, const ScoreAdsResponse::AdScore& winning_ad_score, const std::shared_ptr& auction_config, const BuyerReportingMetadata& buyer_reporting_metadata); @@ -220,6 +228,12 @@ class ScoreAdsReactor std::string, std::unique_ptr> ad_data_; + + // Map of dispatch id to component level reporting data in a component auction + // result. + absl::flat_hash_map + component_level_reporting_data_; + absl::flat_hash_map> protected_app_signals_ad_data_; @@ -227,7 +241,7 @@ class ScoreAdsReactor const AsyncReporter& async_reporter_; bool enable_seller_debug_url_generation_; std::string roma_timeout_ms_; - server_common::log::ContextImpl log_context_; + RequestLogContext log_context_; // Used to log metric, same life time as reactor. std::unique_ptr metric_context_; @@ -240,7 +254,6 @@ class ScoreAdsReactor // code. bool enable_adtech_code_logging_; bool enable_report_result_url_generation_; - bool enable_report_win_url_generation_; const bool enable_protected_app_signals_; bool enable_report_win_input_noising_; std::string seller_origin_; @@ -251,10 +264,9 @@ class ScoreAdsReactor // Impacts the creation of scoreAd input params and // parsing of scoreAd output. AuctionScope auction_scope_; - // Specifies which verison of scoreAd to use for this request. absl::string_view code_version_; - + bool enable_report_win_url_generation_; google::protobuf::RepeatedPtrField GetEmptyAdComponentRenderUrls() { static google::protobuf::RepeatedPtrField diff --git a/services/auction_service/score_ads_reactor_test.cc b/services/auction_service/score_ads_reactor_test.cc index f50d30f5..738c828e 100644 --- a/services/auction_service/score_ads_reactor_test.cc +++ b/services/auction_service/score_ads_reactor_test.cc @@ -170,14 +170,12 @@ void GetTestAdWithBidFoo(AdWithBidMetadata& foo, absl::string_view buyer_reporting_id = "") { int number_of_component_ads = 3; foo.mutable_ad()->mutable_struct_value()->MergeFrom( - MakeAnAd("https://googleads.g.doubleclick.net/td/adfetch/" - "gda?adg_id=142601302539&cr_id=628073386727&cv_id=0", + MakeAnAd("https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0", "arbitraryMetadataKey", 2)); // This must have an entry in kTestScoringSignals. foo.set_render( - "https://googleads.g.doubleclick.net/td/adfetch/" - "gda?adg_id=142601302539&cr_id=628073386727&cv_id=0"); + "https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0"); foo.set_bid(2.10); foo.set_interest_group_name("foo"); foo.set_interest_group_owner("https://fooAds.com"); @@ -311,7 +309,7 @@ constexpr char kTestAuctionSignals[] = constexpr char kTestScoringSignals[] = R"json( { "renderUrls": { - "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0": [ + "https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0": [ 123 ], "barStandardAds.com/render_ad?id=bar": [ @@ -423,10 +421,10 @@ void CheckInputCorrectForFoo(std::vector> input, R"JSON({"auctionSignals": {"auction_signal": "test 2"}, "sellerSignals": {"seller_signal": "test 1"}})JSON"); EXPECT_EQ( *input[3], - R"JSON({"adComponentRenderUrls":{"adComponent.com/foo_components/id=0":["foo0"],"adComponent.com/foo_components/id=1":["foo1"],"adComponent.com/foo_components/id=2":["foo2"]},"renderUrl":{"https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0":[123]}})JSON"); + R"JSON({"adComponentRenderUrls":{"adComponent.com/foo_components/id=0":["foo0"],"adComponent.com/foo_components/id=1":["foo1"],"adComponent.com/foo_components/id=2":["foo2"]},"renderUrl":{"https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0":[123]}})JSON"); EXPECT_EQ( *input[4], - R"JSON({"interestGroupOwner":"https://fooAds.com","topWindowHostname":"publisher_hostname","adComponents":["adComponent.com/foo_components/id=0","adComponent.com/foo_components/id=1","adComponent.com/foo_components/id=2"],"bidCurrency":"???","renderUrl":"https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0"})JSON"); + R"JSON({"interestGroupOwner":"https://fooAds.com","topWindowHostname":"publisher_hostname","adComponents":["adComponent.com/foo_components/id=0","adComponent.com/foo_components/id=1","adComponent.com/foo_components/id=2"],"bidCurrency":"???","renderUrl":"https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0"})JSON"); EXPECT_EQ(*input[5], "{}"); } @@ -548,13 +546,13 @@ TEST_F( R"JSON( { "metadata": {"arbitraryMetadataKey": 2}, - "renderUrl": "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0" + "renderUrl": "https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0" })JSON"); CHECK_OK(expected_foo_ad) << "Malformed ad JSON"; auto expected_foo_comp_ad_1 = JsonStringToValue( R"JSON( { - "renderUrl": "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0", + "renderUrl": "https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0", "metadata": {"arbitraryMetadataKey": 2} })JSON"); CHECK_OK(expected_foo_comp_ad_1) << "Malformed ad JSON"; @@ -614,7 +612,7 @@ TEST_F(ScoreAdsReactorTest, "metadata": { "arbitraryMetadataKey": 2 }, - "renderUrl": "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0" + "renderUrl": "https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0" })JSON"); CHECK_OK(expected_foo_ad_2) << "Malformed ad JSON"; auto expected_bar_ad_1 = JsonStringToValue( @@ -623,7 +621,7 @@ TEST_F(ScoreAdsReactorTest, "metadata": { "arbitraryMetadataKey":2 }, - "renderUrl": "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0" + "renderUrl": "https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0" })JSON"); CHECK_OK(expected_bar_ad_1) << "Malformed ad JSON"; auto expected_bar_ad_2 = JsonStringToValue( @@ -677,7 +675,7 @@ TEST_F(ScoreAdsReactorTest, "metadata": { "arbitraryMetadataKey": 2 }, - "renderUrl": "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0" + "renderUrl": "https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0" })JSON"); CHECK_OK(expected_foo_ad_2) << "Malformed ad JSON"; @@ -711,7 +709,7 @@ TEST_F(ScoreAdsReactorTest, "metadata": { "arbitraryMetadataKey":2 }, - "renderUrl": "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0" + "renderUrl": "https://adtech?adg_id=142601302539&cr_id=628073386727&cv_id=0" })JSON"); CHECK_OK(expected_bar_ad_1) << "Malformed ad JSON"; auto expected_bar_ad_2 = JsonStringToValue( diff --git a/services/auction_service/score_ads_reactor_test_util.h b/services/auction_service/score_ads_reactor_test_util.h index ab1a5ac0..30dce1cd 100644 --- a/services/auction_service/score_ads_reactor_test_util.h +++ b/services/auction_service/score_ads_reactor_test_util.h @@ -39,6 +39,13 @@ constexpr char kKeyId[] = "keyid"; constexpr char kSecret[] = "secret"; constexpr float kTestBid = 1.25; constexpr char kTestProtectedAppSignalsAdOwner[] = "https://PAS-Ad-Owner.com"; +constexpr char kTestReportingWinResponseJson[] = + R"({"reportResultResponse":{"reportResultUrl":"http://reportResultUrl.com","signalsForWinner":"{testKey:testValue}","sendReportToInvoked":true,"registerAdBeaconInvoked":true,"interactionReportingUrls":{"click":"http://event.com"}},"sellerLogs":["testLog"], "sellerErrors":["testLog"], "sellerWarnings":["testLog"], +"reportWinResponse":{"reportWinUrl":"http://reportWinUrl.com","sendReportToInvoked":true,"registerAdBeaconInvoked":true,"interactionReportingUrls":{"click":"http://event.com"}},"buyerLogs":["testLog"], "buyerErrors":["testLog"], "buyerWarnings":["testLog"]})"; +constexpr char kTestReportingResponseJson[] = + R"({"reportResultResponse":{"reportResultUrl":"http://reportResultUrl.com","signalsForWinner":"{testKey:testValue}","sendReportToInvoked":true,"registerAdBeaconInvoked":true,"interactionReportingUrls":{"click":"http://event.com"}},"sellerLogs":["testLog"]})"; +constexpr char kEmptyTestReportingResponseJson[] = + R"({"reportResultResponse":{"reportResultUrl":"","sendReportToInvoked":false,"registerAdBeaconInvoked":false,"interactionReportingUrls":{}},"sellerLogs":[], "sellerErrors":[], "sellerWarnings":[])"; ProtectedAppSignalsAdWithBidMetadata GetProtectedAppSignalsAdWithBidMetadata( absl::string_view render_url, float bid = kTestBid); diff --git a/services/auction_service/score_ads_reactor_top_level_auction_test.cc b/services/auction_service/score_ads_reactor_top_level_auction_test.cc index 6a662faa..f2e629ee 100644 --- a/services/auction_service/score_ads_reactor_top_level_auction_test.cc +++ b/services/auction_service/score_ads_reactor_top_level_auction_test.cc @@ -15,6 +15,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "services/auction_service/auction_constants.h" +#include "services/auction_service/reporting/reporting_helper_test_constants.h" #include "services/auction_service/score_ads_reactor.h" #include "services/auction_service/score_ads_reactor_test_util.h" #include "services/common/test/mocks.h" @@ -29,6 +30,12 @@ constexpr char kTestAuctionSignals[] = constexpr char kTestPublisherHostname[] = "publisher_hostname"; constexpr char kTestTopLevelSeller[] = "top_level_seller"; constexpr char kTestGenerationId[] = "test_generation_id"; +constexpr char kTestComponentWinReportingUrl[] = + "http://componentReportingUrl.com"; +constexpr char kTestComponentEvent[] = "click"; +constexpr char kTestComponentInteractionReportingUrl[] = + "http://componentInteraction.com"; +constexpr char kTestComponentSeller[] = "http://componentSeller.com"; using RawRequest = ScoreAdsRequest::ScoreAdsRawRequest; @@ -92,18 +99,37 @@ TEST(ScoreAdsReactorTopLevelAuctionTest, EXPECT_TRUE(response.response_ciphertext().empty()); } -TEST(ScoreAdsReactorTopLevelAuctionTest, ReturnsWinningAdFromDispatcher) { +TEST(ScoreAdsReactorTopLevelAuctionTest, + ReturnsWinnerWithComponentUrlsWhenReportingDisabled) { MockCodeDispatchClient dispatcher; - AuctionResult car_1 = MakeARandomComponentAuctionResult(MakeARandomString(), - kTestTopLevelSeller); + AuctionResult car_1 = MakeARandomComponentAuctionResultWithReportingUrls( + MakeARandomString(), kTestTopLevelSeller, kTestComponentEvent, + kTestComponentWinReportingUrl, kTestComponentInteractionReportingUrl, + kTestComponentSeller); RawRequest raw_request = BuildTopLevelAuctionRawRequest( {car_1}, kTestSellerSignals, kTestAuctionSignals, kTestPublisherHostname); + bool enable_report_win_url_generation = false; + bool enable_report_result_url_generation = true; EXPECT_CALL(dispatcher, BatchExecute) - .WillOnce([](std::vector& batch, - BatchDispatchDoneCallback done_callback) { - std::vector score_logic(batch.size(), - R"( - { + .WillRepeatedly([enable_report_win_url_generation, + enable_report_result_url_generation]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + std::vector response; + for (const auto& request : batch) { + if (std::strcmp(request.handler_name.c_str(), + kReportingDispatchHandlerFunctionName) == 0) { + if (enable_report_win_url_generation) { + response.emplace_back(kTestReportingWinResponseJson); + } else if (enable_report_result_url_generation) { + response.emplace_back(kTestReportingResponseJson); + } else { + response.emplace_back(kEmptyTestReportingResponseJson); + } + } else { + response.push_back( + R"JSON( + { "response" : { "ad": {"key1":"adMetadata"}, "desirability" : 1, @@ -111,13 +137,22 @@ TEST(ScoreAdsReactorTopLevelAuctionTest, ReturnsWinningAdFromDispatcher) { "allowComponentAuction" : true }, "logs":[] - } - )"); - return FakeExecute(batch, std::move(done_callback), - std::move(score_logic), false); + } + )JSON"); + } + } + return FakeExecute(batch, std::move(done_callback), std::move(response), + false); }); + AuctionServiceRuntimeConfig runtime_config = { + .enable_seller_debug_url_generation = false, + .enable_adtech_code_logging = false, + .enable_report_result_url_generation = + enable_report_result_url_generation, + .enable_report_win_url_generation = enable_report_win_url_generation}; ScoreAdsReactorTestHelper test_helper; - auto response = test_helper.ExecuteScoreAds(raw_request, dispatcher); + auto response = + test_helper.ExecuteScoreAds(raw_request, dispatcher, runtime_config); ScoreAdsResponse::ScoreAdsRawResponse raw_response; ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); const auto& scored_ad = raw_response.ad_score(); @@ -128,7 +163,37 @@ TEST(ScoreAdsReactorTopLevelAuctionTest, ReturnsWinningAdFromDispatcher) { EXPECT_EQ(scored_ad.interest_group_name(), car_1.interest_group_name()); EXPECT_EQ(scored_ad.interest_group_owner(), car_1.interest_group_owner()); EXPECT_EQ(scored_ad.buyer_bid(), car_1.bid()); - + EXPECT_EQ(scored_ad.win_reporting_urls() + .top_level_seller_reporting_urls() + .reporting_url(), + kTestReportResultUrl); + EXPECT_EQ(scored_ad.win_reporting_urls() + .top_level_seller_reporting_urls() + .interaction_reporting_urls() + .size(), + 1); + EXPECT_EQ(scored_ad.win_reporting_urls() + .top_level_seller_reporting_urls() + .interaction_reporting_urls() + .at(kTestInteractionEvent), + kTestInteractionUrl); + EXPECT_EQ(scored_ad.win_reporting_urls() + .component_seller_reporting_urls() + .reporting_url(), + kTestComponentWinReportingUrl); + EXPECT_EQ(scored_ad.win_reporting_urls() + .component_seller_reporting_urls() + .interaction_reporting_urls() + .at(kTestComponentEvent), + kTestComponentInteractionReportingUrl); + EXPECT_EQ( + scored_ad.win_reporting_urls().buyer_reporting_urls().reporting_url(), + kTestComponentWinReportingUrl); + EXPECT_EQ(scored_ad.win_reporting_urls() + .buyer_reporting_urls() + .interaction_reporting_urls() + .at(kTestComponentEvent), + kTestComponentInteractionReportingUrl); // Since in the above test we are assuming non-component auctions, check that // the required fields for component auctions are not set. EXPECT_FALSE(scored_ad.allow_component_auction()); @@ -136,54 +201,111 @@ TEST(ScoreAdsReactorTopLevelAuctionTest, ReturnsWinningAdFromDispatcher) { EXPECT_EQ(scored_ad.bid(), 0); } -TEST(ScoreAdsReactorTopLevelAuctionTest, DoesNotPopulateHighestOtherBid) { +TEST(ScoreAdsReactorTopLevelAuctionTest, + ReturnsWinningAdFromDispatcherWithReportingEnabled) { MockCodeDispatchClient dispatcher; - AuctionResult car_1 = - MakeARandomComponentAuctionResult(kTestGenerationId, kTestTopLevelSeller); - AuctionResult car_2 = - MakeARandomComponentAuctionResult(kTestGenerationId, kTestTopLevelSeller); + AuctionResult car_1 = MakeARandomComponentAuctionResultWithReportingUrls( + MakeARandomString(), kTestTopLevelSeller, kTestComponentEvent, + kTestComponentWinReportingUrl, kTestComponentInteractionReportingUrl, + kTestComponentSeller); RawRequest raw_request = BuildTopLevelAuctionRawRequest( - {car_1, car_2}, kTestSellerSignals, kTestAuctionSignals, - kTestPublisherHostname); - EXPECT_EQ(raw_request.component_auction_results_size(), 2); + {car_1}, kTestSellerSignals, kTestAuctionSignals, kTestPublisherHostname); + bool enable_report_win_url_generation = true; EXPECT_CALL(dispatcher, BatchExecute) - .WillOnce([](std::vector& batch, - BatchDispatchDoneCallback done_callback) { - std::vector score_logic(batch.size(), - R"( - { + .WillRepeatedly([enable_report_win_url_generation]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + std::vector response; + for (const auto& request : batch) { + if (std::strcmp(request.handler_name.c_str(), + kReportingDispatchHandlerFunctionName) == 0) { + if (enable_report_win_url_generation) { + response.emplace_back(kTestReportingWinResponseJson); + } else { + response.emplace_back(kTestReportingResponseJson); + } + } else { + response.push_back( + R"JSON( + { "response" : { "ad": {"key1":"adMetadata"}, "desirability" : 1, "bid" : 0.1, + "allowComponentAuction" : true }, "logs":[] - } - )"); - return FakeExecute(batch, std::move(done_callback), - std::move(score_logic), false); + } + )JSON"); + } + } + return FakeExecute(batch, std::move(done_callback), std::move(response), + false); }); + AuctionServiceRuntimeConfig runtime_config = { + .enable_seller_debug_url_generation = false, + .enable_adtech_code_logging = false, + .enable_report_result_url_generation = false, + .enable_report_win_url_generation = true}; ScoreAdsReactorTestHelper test_helper; - auto response = test_helper.ExecuteScoreAds(raw_request, dispatcher); + auto response = + test_helper.ExecuteScoreAds(raw_request, dispatcher, runtime_config); ScoreAdsResponse::ScoreAdsRawResponse raw_response; ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); const auto& scored_ad = raw_response.ad_score(); - // Do not populate highest scoring other bids. - EXPECT_TRUE(scored_ad.ig_owner_highest_scoring_other_bids_map().empty()); + EXPECT_TRUE(scored_ad.win_reporting_urls() + .top_level_seller_reporting_urls() + .reporting_url() + .empty()); + EXPECT_EQ(scored_ad.win_reporting_urls() + .top_level_seller_reporting_urls() + .interaction_reporting_urls() + .size(), + 0); + EXPECT_EQ(scored_ad.win_reporting_urls() + .component_seller_reporting_urls() + .reporting_url(), + kTestComponentWinReportingUrl); + EXPECT_EQ(scored_ad.win_reporting_urls() + .component_seller_reporting_urls() + .interaction_reporting_urls() + .at(kTestComponentEvent), + kTestComponentInteractionReportingUrl); + EXPECT_EQ( + scored_ad.win_reporting_urls().buyer_reporting_urls().reporting_url(), + kTestComponentWinReportingUrl); + EXPECT_EQ(scored_ad.win_reporting_urls() + .buyer_reporting_urls() + .interaction_reporting_urls() + .at(kTestComponentEvent), + kTestComponentInteractionReportingUrl); } -TEST(ScoreAdsReactorTopLevelAuctionTest, DoesNotSupportWinReporting) { +TEST(ScoreAdsReactorTopLevelAuctionTest, + NoUrlsReturnedWhenNoComponentUrlsPresent) { MockCodeDispatchClient dispatcher; AuctionResult car_1 = MakeARandomComponentAuctionResult(MakeARandomString(), kTestTopLevelSeller); RawRequest raw_request = BuildTopLevelAuctionRawRequest( {car_1}, kTestSellerSignals, kTestAuctionSignals, kTestPublisherHostname); + bool enable_report_win_url_generation = true; EXPECT_CALL(dispatcher, BatchExecute) - .WillOnce([](std::vector& batch, - BatchDispatchDoneCallback done_callback) { - std::vector score_logic(batch.size(), - R"( - { + .WillRepeatedly([enable_report_win_url_generation]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + std::vector response; + for (const auto& request : batch) { + if (std::strcmp(request.handler_name.c_str(), + kReportingDispatchHandlerFunctionName) == 0) { + if (enable_report_win_url_generation) { + response.emplace_back(kTestReportingWinResponseJson); + } else { + response.emplace_back(kTestReportingResponseJson); + } + } else { + response.push_back( + R"JSON( + { "response" : { "ad": {"key1":"adMetadata"}, "desirability" : 1, @@ -191,15 +313,17 @@ TEST(ScoreAdsReactorTopLevelAuctionTest, DoesNotSupportWinReporting) { "allowComponentAuction" : true }, "logs":[] - } - )"); - return FakeExecute(batch, std::move(done_callback), - std::move(score_logic), false); + } + )JSON"); + } + } + return FakeExecute(batch, std::move(done_callback), std::move(response), + false); }); AuctionServiceRuntimeConfig runtime_config = { .enable_seller_debug_url_generation = false, .enable_adtech_code_logging = false, - .enable_report_result_url_generation = true, + .enable_report_result_url_generation = false, .enable_report_win_url_generation = true}; ScoreAdsReactorTestHelper test_helper; auto response = @@ -236,6 +360,42 @@ TEST(ScoreAdsReactorTopLevelAuctionTest, DoesNotSupportWinReporting) { 0); } +TEST(ScoreAdsReactorTopLevelAuctionTest, DoesNotPopulateHighestOtherBid) { + MockCodeDispatchClient dispatcher; + AuctionResult car_1 = + MakeARandomComponentAuctionResult(kTestGenerationId, kTestTopLevelSeller); + AuctionResult car_2 = + MakeARandomComponentAuctionResult(kTestGenerationId, kTestTopLevelSeller); + RawRequest raw_request = BuildTopLevelAuctionRawRequest( + {car_1, car_2}, kTestSellerSignals, kTestAuctionSignals, + kTestPublisherHostname); + EXPECT_EQ(raw_request.component_auction_results_size(), 2); + EXPECT_CALL(dispatcher, BatchExecute) + .WillOnce([](std::vector& batch, + BatchDispatchDoneCallback done_callback) { + std::vector score_logic(batch.size(), + R"( + { + "response" : { + "ad": {"key1":"adMetadata"}, + "desirability" : 1, + "bid" : 0.1, + }, + "logs":[] + } + )"); + return FakeExecute(batch, std::move(done_callback), + std::move(score_logic), false); + }); + ScoreAdsReactorTestHelper test_helper; + auto response = test_helper.ExecuteScoreAds(raw_request, dispatcher); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + const auto& scored_ad = raw_response.ad_score(); + // Do not populate highest scoring other bids. + EXPECT_TRUE(scored_ad.ig_owner_highest_scoring_other_bids_map().empty()); +} + TEST(ScoreAdsReactorTopLevelAuctionTest, DoesNotPerformDebugReporting) { MockCodeDispatchClient dispatcher; AuctionResult car_1 = MakeARandomComponentAuctionResult(MakeARandomString(), @@ -243,8 +403,8 @@ TEST(ScoreAdsReactorTopLevelAuctionTest, DoesNotPerformDebugReporting) { RawRequest raw_request = BuildTopLevelAuctionRawRequest( {car_1}, kTestSellerSignals, kTestAuctionSignals, kTestPublisherHostname); EXPECT_CALL(dispatcher, BatchExecute) - .WillOnce([](std::vector& batch, - BatchDispatchDoneCallback done_callback) { + .WillRepeatedly([](std::vector& batch, + BatchDispatchDoneCallback done_callback) { std::vector score_logic(batch.size(), R"( { diff --git a/services/auction_service/seller_code_fetch_manager_test.cc b/services/auction_service/seller_code_fetch_manager_test.cc deleted file mode 100644 index 416618b1..00000000 --- a/services/auction_service/seller_code_fetch_manager_test.cc +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "services/auction_service/seller_code_fetch_manager.h" - -#include "absl/strings/str_cat.h" -#include "absl/synchronization/blocking_counter.h" -#include "absl/synchronization/notification.h" -#include "absl/time/time.h" -#include "gtest/gtest.h" -#include "services/auction_service/auction_constants.h" -#include "services/auction_service/code_wrapper/seller_code_wrapper.h" -#include "services/common/test/mocks.h" -#include "services/common/util/file_util.h" -#include "src/core/interface/async_context.h" -#include "src/public/cpio/interface/blob_storage_client/blob_storage_client_interface.h" -#include "src/public/cpio/interface/cpio.h" -#include "src/public/cpio/interface/error_codes.h" -#include "src/public/cpio/mock/blob_storage_client/mock_blob_storage_client.h" - -namespace privacy_sandbox::bidding_auction_servers { -namespace { - -using auction_service::FetchMode; -using auction_service::SellerCodeFetchConfig; - -using ::google::cmrt::sdk::blob_storage_service::v1::BlobMetadata; -using ::google::cmrt::sdk::blob_storage_service::v1::GetBlobRequest; -using ::google::cmrt::sdk::blob_storage_service::v1::GetBlobResponse; -using ::google::cmrt::sdk::blob_storage_service::v1::ListBlobsMetadataRequest; -using ::google::cmrt::sdk::blob_storage_service::v1::ListBlobsMetadataResponse; -using ::google::scp::core::AsyncContext; -using ::google::scp::core::ExecutionResult; -using ::google::scp::core::FailureExecutionResult; -using ::google::scp::core::SuccessExecutionResult; -using ::google::scp::cpio::MockBlobStorageClient; -using ::testing::Return; - -class SellerCodeFetchManagerTest : public testing::Test { - protected: - void SetUp() override { - executor_ = std::make_unique(); - http_fetcher_ = std::make_unique(); - dispatcher_ = std::make_unique(); - blob_storage_client_ = std::make_unique(); - } - std::unique_ptr executor_; - std::unique_ptr http_fetcher_; - std::unique_ptr dispatcher_; - std::unique_ptr blob_storage_client_; -}; - -TEST_F(SellerCodeFetchManagerTest, FetchModeLocalTriesFileLoad) { - EXPECT_CALL(*blob_storage_client_, Init).Times(0); - EXPECT_CALL(*blob_storage_client_, Run).Times(0); - - SellerCodeFetchConfig udf_config; - udf_config.set_fetch_mode(FetchMode::FETCH_MODE_LOCAL); - const std::string bad_path = "error"; - const bool enable_protected_app_signals = true; - - udf_config.set_auction_js_path(bad_path); - SellerCodeFetchManager udf_fetcher(std::move(blob_storage_client_), - executor_.get(), http_fetcher_.get(), - http_fetcher_.get(), dispatcher_.get(), - udf_config, enable_protected_app_signals); - absl::Status load_status = udf_fetcher.Init(); - EXPECT_FALSE(load_status.ok()); - EXPECT_EQ(load_status.message(), absl::StrCat(kPathFailed, bad_path)); -} - -TEST_F(SellerCodeFetchManagerTest, - FetchModeBucketSucceedsLoadingWrappedBucketBlobs) { - std::string fake_udf = "udf_data"; - std::string pa_buyer_origin = "pa_origin"; - std::string pas_buyer_origin = "pas_origin"; - std::string pa_reporting_udf_data = "pa_reporting_udf_data"; - std::string pas_reporting_udf_data = "pas_reporting_udf_data"; - const bool enable_protected_app_signals = true; - - SellerCodeFetchConfig udf_config; - udf_config.set_fetch_mode(FetchMode::FETCH_MODE_BUCKET); - udf_config.set_enable_report_win_url_generation(true); - udf_config.set_enable_report_result_url_generation(true); - udf_config.mutable_buyer_report_win_js_urls()->try_emplace(pa_buyer_origin, - "foo.com"); - udf_config.mutable_protected_app_signals_buyer_report_win_js_urls() - ->try_emplace(pas_buyer_origin, "bar.com"); - udf_config.set_auction_js_bucket("js"); - udf_config.set_auction_js_bucket_default_blob("default"); - - EXPECT_CALL(*blob_storage_client_, Init).WillOnce(Return(absl::OkStatus())); - EXPECT_CALL(*blob_storage_client_, Run).WillOnce(Return(absl::OkStatus())); - EXPECT_CALL(*executor_, RunAfter).Times(2); - EXPECT_CALL(*http_fetcher_, FetchUrls) - .Times(1) - .WillOnce([&udf_config, &pa_buyer_origin, &pas_buyer_origin, - &pa_reporting_udf_data, &pas_reporting_udf_data]( - const std::vector& requests, - absl::Duration timeout, OnDoneFetchUrls done_callback) { - EXPECT_EQ(requests[0].url, - udf_config.buyer_report_win_js_urls().at(pa_buyer_origin)); - EXPECT_EQ( - requests[1].url, - udf_config.protected_app_signals_buyer_report_win_js_urls().at( - pas_buyer_origin)); - - std::move(done_callback)( - {pa_reporting_udf_data, pas_reporting_udf_data}); - }); - - EXPECT_CALL(*blob_storage_client_, ListBlobsMetadata) - .WillOnce( - [&udf_config]( - AsyncContext - context) { - EXPECT_EQ(context.request->blob_metadata().bucket_name(), - udf_config.auction_js_bucket()); - BlobMetadata md; - md.set_bucket_name(udf_config.auction_js_bucket()); - md.set_blob_name(udf_config.auction_js_bucket_default_blob()); - - context.response = std::make_shared(); - context.response->mutable_blob_metadatas()->Add(std::move(md)); - context.Finish(SuccessExecutionResult()); - return absl::OkStatus(); - }); - - EXPECT_CALL(*blob_storage_client_, GetBlob) - .WillOnce( - [&udf_config, &fake_udf]( - AsyncContext async_context) { - auto async_bucket_name = - async_context.request->blob_metadata().bucket_name(); - auto async_blob_name = - async_context.request->blob_metadata().blob_name(); - EXPECT_EQ(async_bucket_name, udf_config.auction_js_bucket()); - EXPECT_EQ(async_blob_name, - udf_config.auction_js_bucket_default_blob()); - - async_context.response = std::make_shared(); - async_context.response->mutable_blob()->set_data(fake_udf); - async_context.result = SuccessExecutionResult(); - async_context.Finish(); - - return absl::OkStatus(); - }); - - EXPECT_CALL(*dispatcher_, LoadSync) - .WillOnce([&udf_config, &fake_udf, &pa_buyer_origin, &pas_buyer_origin, - &pa_reporting_udf_data, &pas_reporting_udf_data]( - std::string_view version, absl::string_view blob_data) { - EXPECT_EQ(version, udf_config.auction_js_bucket_default_blob()); - absl::flat_hash_map pa_reporting_udfs; - pa_reporting_udfs.try_emplace(pa_buyer_origin, pa_reporting_udf_data); - absl::flat_hash_map pas_reporting_udfs; - pas_reporting_udfs.try_emplace(pas_buyer_origin, - pas_reporting_udf_data); - - EXPECT_EQ( - blob_data, - GetSellerWrappedCode( - fake_udf, udf_config.enable_report_result_url_generation(), - enable_protected_app_signals, - udf_config.enable_report_win_url_generation(), - pa_reporting_udfs, pas_reporting_udfs)); - return absl::OkStatus(); - }); - - SellerCodeFetchManager udf_fetcher(std::move(blob_storage_client_), - executor_.get(), http_fetcher_.get(), - http_fetcher_.get(), dispatcher_.get(), - udf_config, enable_protected_app_signals); - absl::Status load_status = udf_fetcher.Init(); - EXPECT_TRUE(load_status.ok()); -} - -TEST_F(SellerCodeFetchManagerTest, FetchModeUrlSucceedsLoadingWrappedUrlBlobs) { - std::string fake_udf = "udf_data"; - std::string pa_buyer_origin = "pa_origin"; - std::string pas_buyer_origin = "pas_origin"; - std::string pa_reporting_udf_data = "pa_reporting_udf_data"; - std::string pas_reporting_udf_data = "pas_reporting_udf_data"; - const bool enable_protected_app_signals = true; - - SellerCodeFetchConfig udf_config; - udf_config.set_fetch_mode(FetchMode::FETCH_MODE_URL); - udf_config.set_enable_report_win_url_generation(true); - udf_config.set_enable_report_result_url_generation(true); - udf_config.mutable_buyer_report_win_js_urls()->try_emplace(pa_buyer_origin, - "foo.com"); - udf_config.mutable_protected_app_signals_buyer_report_win_js_urls() - ->try_emplace(pas_buyer_origin, "bar.com"); - udf_config.set_auction_js_url("auction.com"); - udf_config.set_url_fetch_timeout_ms(70000000); - udf_config.set_url_fetch_period_ms(70000001); - EXPECT_CALL(*blob_storage_client_, Init).Times(0); - EXPECT_CALL(*blob_storage_client_, Run).Times(0); - EXPECT_CALL(*executor_, RunAfter).Times(2); - EXPECT_CALL(*http_fetcher_, FetchUrls) - .Times(2) - .WillOnce([&udf_config, &pa_buyer_origin, &pas_buyer_origin, - &pa_reporting_udf_data, &pas_reporting_udf_data]( - const std::vector& requests, - absl::Duration timeout, OnDoneFetchUrls done_callback) { - EXPECT_EQ(requests[0].url, - udf_config.buyer_report_win_js_urls().at(pa_buyer_origin)); - EXPECT_EQ( - requests[1].url, - udf_config.protected_app_signals_buyer_report_win_js_urls().at( - pas_buyer_origin)); - - std::move(done_callback)( - {pa_reporting_udf_data, pas_reporting_udf_data}); - }) - .WillOnce([&udf_config, &fake_udf]( - const std::vector& requests, - absl::Duration timeout, OnDoneFetchUrls done_callback) { - EXPECT_EQ(requests[0].url, udf_config.auction_js_url()); - std::move(done_callback)({fake_udf}); - }); - - EXPECT_CALL(*dispatcher_, LoadSync) - .WillOnce([&udf_config, &fake_udf, &pa_buyer_origin, &pas_buyer_origin, - &pa_reporting_udf_data, &pas_reporting_udf_data]( - std::string_view version, absl::string_view blob_data) { - EXPECT_EQ(version, kScoreAdBlobVersion); - absl::flat_hash_map pa_reporting_udfs; - pa_reporting_udfs.try_emplace(pa_buyer_origin, pa_reporting_udf_data); - absl::flat_hash_map pas_reporting_udfs; - pas_reporting_udfs.try_emplace(pas_buyer_origin, - pas_reporting_udf_data); - - EXPECT_EQ( - blob_data, - GetSellerWrappedCode( - fake_udf, udf_config.enable_report_result_url_generation(), - enable_protected_app_signals, - udf_config.enable_report_win_url_generation(), - pa_reporting_udfs, pas_reporting_udfs)); - return absl::OkStatus(); - }); - - SellerCodeFetchManager udf_fetcher(std::move(blob_storage_client_), - executor_.get(), http_fetcher_.get(), - http_fetcher_.get(), dispatcher_.get(), - udf_config, enable_protected_app_signals); - absl::Status load_status = udf_fetcher.Init(); - EXPECT_TRUE(load_status.ok()); -} - -} // namespace -} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/udf_fetcher/BUILD b/services/auction_service/udf_fetcher/BUILD new file mode 100644 index 00000000..743dc365 --- /dev/null +++ b/services/auction_service/udf_fetcher/BUILD @@ -0,0 +1,196 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_proto_library", "cc_test") +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = [ + "//visibility:public", +]) + +cc_library( + name = "buyer_reporting_fetcher", + srcs = ["buyer_reporting_fetcher.cc"], + hdrs = [ + "buyer_reporting_fetcher.h", + ], + deps = [ + ":auction_code_fetch_config_cc_proto", + "//services/common/clients/http:http_fetcher_async", + "//services/common/code_fetch:code_fetcher_interface", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/functional:any_invocable", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", + "@google_privacysandbox_servers_common//src/concurrent:executor", + "@google_privacysandbox_servers_common//src/logger:request_context_logger", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + "@google_privacysandbox_servers_common//src/util/status_macro:status_util", + ], +) + +cc_library( + name = "buyer_reporting_udf_fetch_manager", + srcs = ["buyer_reporting_udf_fetch_manager.cc"], + hdrs = [ + "buyer_reporting_udf_fetch_manager.h", + ], + deps = [ + ":adtech_code_version_util", + ":auction_code_fetch_config_cc_proto", + "//services/common/clients/code_dispatcher:v8_dispatcher", + "//services/common/clients/http:http_fetcher_async", + "//services/common/code_fetch:code_fetcher_interface", + "//services/common/util:request_response_constants", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/functional:any_invocable", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", + "@google_privacysandbox_servers_common//src/concurrent:executor", + "@google_privacysandbox_servers_common//src/logger:request_context_logger", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + "@google_privacysandbox_servers_common//src/util/status_macro:status_util", + ], +) + +proto_library( + name = "auction_code_fetch_config_proto", + srcs = ["auction_code_fetch_config.proto"], + deps = [ + "//services/common/blob_fetch:fetch_mode_proto", + "@com_google_googleapis//google/api:annotations_proto", + "@com_google_protobuf//:struct_proto", + ], +) + +cc_proto_library( + name = "auction_code_fetch_config_cc_proto", + visibility = ["//services/auction_service:__subpackages__"], + deps = [":auction_code_fetch_config_proto"], +) + +cc_library( + name = "adtech_code_version_util", + srcs = [ + "adtech_code_version_util.cc", + ], + hdrs = [ + "adtech_code_version_util.h", + ], + deps = [ + "//services/auction_service:auction_constants", + "//services/common/util:request_response_constants", + "@com_google_absl//absl/strings", + "@google_privacysandbox_servers_common//src/logger:request_context_impl", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + "@google_privacysandbox_servers_common//src/util/status_macro:status_util", + ], +) + +cc_library( + name = "seller_udf_fetch_manager", + srcs = [ + "seller_udf_fetch_manager.cc", + ], + hdrs = [ + "seller_udf_fetch_manager.h", + ], + deps = [ + ":auction_code_fetch_config_cc_proto", + ":buyer_reporting_fetcher", + ":buyer_reporting_udf_fetch_manager", + "//services/auction_service:auction_constants", + "//services/auction_service/code_wrapper:buyer_reporting_udf_wrapper", + "//services/auction_service/code_wrapper:seller_code_wrapper", + "//services/auction_service/code_wrapper:seller_udf_wrapper", + "//services/common/clients/code_dispatcher:v8_dispatcher", + "//services/common/code_fetch:periodic_bucket_fetcher", + "//services/common/code_fetch:periodic_code_fetcher", + "//services/common/util:file_util", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/time", + "@google_privacysandbox_servers_common//src/concurrent:executor", + "@google_privacysandbox_servers_common//src/logger:request_context_logger", + "@google_privacysandbox_servers_common//src/public/core/interface:errors", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + ], +) + +cc_test( + name = "seller_udf_fetch_manager_test", + size = "small", + srcs = ["seller_udf_fetch_manager_test.cc"], + deps = [ + ":seller_udf_fetch_manager", + "//services/auction_service/code_wrapper:buyer_reporting_udf_wrapper", + "//services/auction_service/code_wrapper:seller_code_wrapper", + "//services/auction_service/code_wrapper:seller_udf_wrapper", + "//services/common/clients/code_dispatcher:v8_dispatcher", + "//services/common/clients/http:http_fetcher_async", + "//services/common/code_fetch:periodic_bucket_fetcher", + "//services/common/code_fetch:periodic_code_fetcher", + "//services/common/test:mocks", + "@com_google_absl//absl/functional:any_invocable", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + "@google_privacysandbox_servers_common//src/concurrent:executor", + "@google_privacysandbox_servers_common//src/public/core/interface:errors", + "@google_privacysandbox_servers_common//src/public/core/interface:execution_result", + "@google_privacysandbox_servers_common//src/public/cpio/interface:cpio", + "@google_privacysandbox_servers_common//src/public/cpio/interface/blob_storage_client", + "@google_privacysandbox_servers_common//src/public/cpio/mock/blob_storage_client:blob_storage_client_mock", + ], +) + +cc_test( + name = "buyer_reporting_udf_fetch_manager_test", + size = "small", + srcs = ["buyer_reporting_udf_fetch_manager_test.cc"], + deps = [ + ":buyer_reporting_udf_fetch_manager", + ":seller_udf_fetch_manager", + "//services/auction_service/code_wrapper:seller_code_wrapper", + "//services/common/clients/code_dispatcher:v8_dispatcher", + "//services/common/clients/http:http_fetcher_async", + "//services/common/code_fetch:periodic_bucket_fetcher", + "//services/common/code_fetch:periodic_code_fetcher", + "//services/common/test:mocks", + "//services/common/util:request_response_constants", + "@com_google_absl//absl/functional:any_invocable", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + "@google_privacysandbox_servers_common//src/concurrent:executor", + "@google_privacysandbox_servers_common//src/core/interface:async_context", + "@google_privacysandbox_servers_common//src/public/core/interface:errors", + "@google_privacysandbox_servers_common//src/public/core/interface:execution_result", + "@google_privacysandbox_servers_common//src/public/cpio/interface:cpio", + "@google_privacysandbox_servers_common//src/public/cpio/interface/blob_storage_client", + "@google_privacysandbox_servers_common//src/public/cpio/mock/blob_storage_client:blob_storage_client_mock", + ], +) diff --git a/services/auction_service/udf_fetcher/adtech_code_version_util.cc b/services/auction_service/udf_fetcher/adtech_code_version_util.cc new file mode 100644 index 00000000..77ddc1ce --- /dev/null +++ b/services/auction_service/udf_fetcher/adtech_code_version_util.cc @@ -0,0 +1,46 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "services/auction_service/udf_fetcher/adtech_code_version_util.h" + +#include + +#include "absl/strings/str_cat.h" +#include "services/auction_service/auction_constants.h" +#include "services/common/util/request_response_constants.h" +#include "src/logger/request_context_impl.h" +#include "src/util/status_macro/status_macros.h" +#include "src/util/status_macro/status_util.h" + +namespace privacy_sandbox::bidding_auction_servers { + +absl::StatusOr GetBuyerReportWinVersion( + absl::string_view buyer_origin, AuctionType auction_type) { + switch (auction_type) { + case AuctionType::kProtectedAppSignals: + return absl::StrCat(kProtectedAppSignalWrapperPrefix, buyer_origin); + case AuctionType::kProtectedAudience: + return absl::StrCat(kProtectedAuctionWrapperPrefix, buyer_origin); + default: + return absl::Status( + absl::StatusCode::kInvalidArgument, + "Error generating buyer reportWin version. Unknown auction type."); + } +} + +std::string GetDefaultSellerUdfVersion() { return kScoreAdBlobVersion; } + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/udf_fetcher/adtech_code_version_util.h b/services/auction_service/udf_fetcher/adtech_code_version_util.h new file mode 100644 index 00000000..99f0ac7f --- /dev/null +++ b/services/auction_service/udf_fetcher/adtech_code_version_util.h @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_ADTECH_CODE_VERSION_UTIL_H_ +#define SERVICES_ADTECH_CODE_VERSION_UTIL_H_ + +#include + +#include "absl/strings/str_cat.h" +#include "services/common/util/request_response_constants.h" +#include "src/logger/request_context_impl.h" +#include "src/util/status_macro/status_macros.h" +#include "src/util/status_macro/status_util.h" + +namespace privacy_sandbox::bidding_auction_servers { + +inline constexpr absl::string_view kProtectedAppSignalWrapperPrefix = "pas_"; +inline constexpr absl::string_view kProtectedAuctionWrapperPrefix = "pa_"; + +// Returns the code version for reportWin() udf loaded for given +// buyer and platform +absl::StatusOr GetBuyerReportWinVersion( + absl::string_view buyer_origin, AuctionType auction_type); +// Returns Seller's default udf version +std::string GetDefaultSellerUdfVersion(); +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_ADTECH_CODE_VERSION_UTIL_H_ diff --git a/services/auction_service/auction_code_fetch_config.proto b/services/auction_service/udf_fetcher/auction_code_fetch_config.proto similarity index 87% rename from services/auction_service/auction_code_fetch_config.proto rename to services/auction_service/udf_fetcher/auction_code_fetch_config.proto index 9343446a..f120aec4 100644 --- a/services/auction_service/auction_code_fetch_config.proto +++ b/services/auction_service/udf_fetcher/auction_code_fetch_config.proto @@ -18,16 +18,7 @@ syntax = "proto3"; package privacy_sandbox.bidding_auction_servers.auction_service; -// Specify how to fetch the code blob. -enum FetchMode { - // Fetch a single blob from an arbitrary url. - FETCH_MODE_URL = 0; - // Fetch all blobs from a specified bucket - // and rely on a default blob specification. - FETCH_MODE_BUCKET = 1; - // Fetch a blob from a local path. Dev-only. - FETCH_MODE_LOCAL = 2; -} +import "services/common/blob_fetch/fetch_mode.proto"; message SellerCodeFetchConfig { @@ -57,7 +48,7 @@ message SellerCodeFetchConfig { // Required. // Specifies which mode to use when fetching UDF blobs. - FetchMode fetch_mode = 13; + blob_fetch.FetchMode fetch_mode = 13; // Allow seller debug URL generation. bool enable_seller_debug_url_generation = 5; @@ -77,4 +68,8 @@ message SellerCodeFetchConfig { // Map of buyer origin to URL endpoint for reportWin js file for protected // app signals. map protected_app_signals_buyer_report_win_js_urls = 10; + + // Temporary flag to enable seller's udf and buyer's reportWin() + // udf isolation. + bool enable_seller_and_buyer_udf_isolation = 14; } diff --git a/services/auction_service/code_wrapper/buyer_reporting_fetcher.cc b/services/auction_service/udf_fetcher/buyer_reporting_fetcher.cc similarity index 98% rename from services/auction_service/code_wrapper/buyer_reporting_fetcher.cc rename to services/auction_service/udf_fetcher/buyer_reporting_fetcher.cc index 761033d9..44cc0a44 100644 --- a/services/auction_service/code_wrapper/buyer_reporting_fetcher.cc +++ b/services/auction_service/udf_fetcher/buyer_reporting_fetcher.cc @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "services/auction_service/code_wrapper/buyer_reporting_fetcher.h" +#include "services/auction_service/udf_fetcher/buyer_reporting_fetcher.h" #include #include diff --git a/services/auction_service/code_wrapper/buyer_reporting_fetcher.h b/services/auction_service/udf_fetcher/buyer_reporting_fetcher.h similarity index 97% rename from services/auction_service/code_wrapper/buyer_reporting_fetcher.h rename to services/auction_service/udf_fetcher/buyer_reporting_fetcher.h index 86ea8e04..3006b2e4 100644 --- a/services/auction_service/code_wrapper/buyer_reporting_fetcher.h +++ b/services/auction_service/udf_fetcher/buyer_reporting_fetcher.h @@ -24,7 +24,7 @@ #include "absl/container/flat_hash_map.h" #include "absl/synchronization/mutex.h" -#include "services/auction_service/auction_code_fetch_config.pb.h" +#include "services/auction_service/udf_fetcher/auction_code_fetch_config.pb.h" #include "services/common/clients/http/http_fetcher_async.h" #include "services/common/code_fetch/code_fetcher_interface.h" #include "src/concurrent/executor.h" diff --git a/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.cc b/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.cc new file mode 100644 index 00000000..417eb3e0 --- /dev/null +++ b/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.cc @@ -0,0 +1,180 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.h" + +#include +#include +#include +#include + +#include "absl/status/statusor.h" +#include "absl/synchronization/notification.h" +#include "absl/time/time.h" +#include "services/auction_service/udf_fetcher/adtech_code_version_util.h" +#include "services/common/clients/code_dispatcher/v8_dispatcher.h" +#include "services/common/clients/http/http_fetcher_async.h" +#include "services/common/util/request_response_constants.h" +#include "src/logger/request_context_logger.h" +#include "src/util/status_macro/status_macros.h" + +namespace privacy_sandbox::bidding_auction_servers { + +absl::Status BuyerReportingUdfFetchManager::Start() { + if (config_.enable_report_win_url_generation()) { + return PeriodicBuyerReportingFetchAndLoadSync(); + } + return absl::OkStatus(); +} + +void BuyerReportingUdfFetchManager::End() { + if (task_id_.has_value()) { + executor_.Cancel(*task_id_); + task_id_ = absl::nullopt; + } +} + +absl::flat_hash_map +BuyerReportingUdfFetchManager::GetProtectedAudienceReportingByOriginForTesting() + ABSL_LOCKS_EXCLUDED(code_blob_per_origin_mu_) { + absl::MutexLock lock(&code_blob_per_origin_mu_); + return protected_auction_code_blob_per_origin_; +} + +absl::flat_hash_map BuyerReportingUdfFetchManager:: + GetProtectedAppSignalsReportingByOriginForTesting() + ABSL_LOCKS_EXCLUDED(code_blob_per_origin_mu_) { + absl::MutexLock lock(&code_blob_per_origin_mu_); + return protected_app_signals_code_blob_per_origin_; +} + +void BuyerReportingUdfFetchManager::LoadBuyerCode( + const std::string& version, const std::string& fetched_blob) { + std::string wrapped_code = buyer_code_wrapper_(fetched_blob); + absl::Status syncResult = dispatcher_.LoadSync(version, wrapped_code); + if (syncResult.ok()) { + PS_VLOG(kSuccess) << "Roma loaded buyer reporting udf version: " << version + << " with contents:\n" + << wrapped_code; + } else { + PS_LOG(ERROR) << "Roma LoadSync fail for buyer: " << syncResult; + } +} + +void BuyerReportingUdfFetchManager::OnProtectedAudienceUdfFetch( + const std::string& buyer_origin, std::string& udf) { + absl::MutexLock lock(&code_blob_per_origin_mu_); + if (protected_auction_code_blob_per_origin_[buyer_origin] != udf) { + absl::StatusOr code_version = + GetBuyerReportWinVersion(buyer_origin, AuctionType::kProtectedAudience); + if (!code_version.ok()) { + PS_LOG(ERROR) << "Error getting code version for buyer:" << buyer_origin + << ";Error:" << code_version.status().message(); + return; + } + LoadBuyerCode(*code_version, udf); + protected_auction_code_blob_per_origin_[buyer_origin] = std::move(udf); + } +} + +void BuyerReportingUdfFetchManager::OnProtectedAppSignalUdfFetch( + const std::string& buyer_origin, std::string& udf) { + absl::MutexLock lock(&code_blob_per_origin_mu_); + if (protected_app_signals_code_blob_per_origin_[buyer_origin] != udf) { + absl::StatusOr code_version = GetBuyerReportWinVersion( + buyer_origin, AuctionType::kProtectedAppSignals); + if (!code_version.ok()) { + PS_LOG(ERROR) << "Error getting code version for buyer:" << buyer_origin + << ";Error:" << code_version.status().message(); + } + LoadBuyerCode(code_version.value(), udf); + protected_app_signals_code_blob_per_origin_[buyer_origin] = std::move(udf); + } +} + +absl::Status +BuyerReportingUdfFetchManager::PeriodicBuyerReportingFetchAndLoadSync() { + std::vector requests; + requests.reserve( + config_.buyer_report_win_js_urls().size() + + config_.protected_app_signals_buyer_report_win_js_urls().size()); + std::vector buyer_origins; + for (const auto& [buyer_origin, report_win_endpoint] : + config_.buyer_report_win_js_urls()) { + requests.push_back({.url = report_win_endpoint}); + buyer_origins.push_back(buyer_origin); + } + + for (const auto& [buyer_origin, report_win_endpoint] : + config_.protected_app_signals_buyer_report_win_js_urls()) { + requests.push_back({.url = report_win_endpoint}); + buyer_origins.push_back(buyer_origin); + } + + if (requests.empty()) { + PS_LOG(WARNING) << "No buyer reporting UDFs to fetch in config."; + return absl::OkStatus(); + } + + absl::Notification fetched_and_loaded; + auto done_callback = + [&fetched_and_loaded, &requests, &buyer_origins, + this](std::vector> results) mutable { + for (int i = 0; i < results.size(); i++) { + auto& result = results[i]; + if (!result.ok()) { + PS_LOG(ERROR) << "Failed origin " << buyer_origins[i] + << " fetch at " << requests[i].url + << " with status: " << result.status(); + continue; + } + if (i < config_.buyer_report_win_js_urls().size()) { + OnProtectedAudienceUdfFetch(buyer_origins[i], *result); + } else { + OnProtectedAppSignalUdfFetch(buyer_origins[i], *result); + } + } + fetched_and_loaded.Notify(); + }; + + http_fetcher_.FetchUrls(requests, + absl::Milliseconds(config_.url_fetch_timeout_ms()), + std::move(done_callback)); + PS_VLOG(kPlain) + << "Waiting for reporting udf fetch and load done notification."; + fetched_and_loaded.WaitForNotification(); + // Verify if all the udfs were fetched and loaded successfully. + if (config_.buyer_report_win_js_urls().size() + + config_.protected_app_signals_buyer_report_win_js_urls().size() != + GetProtectedAudienceReportingByOriginForTesting().size() + + GetProtectedAppSignalsReportingByOriginForTesting().size()) { + PS_LOG(ERROR) + << "Error fetching and loading one or more buyer's reportWin() udf."; + } else { + PS_VLOG(kPlain) << "Reporting udf fetch and load done."; + } + // Schedules the next code blob fetch and saves that task into task_id_. + task_id_ = executor_.RunAfter( + absl::Milliseconds(config_.url_fetch_period_ms()), [this]() { + if (!PeriodicBuyerReportingFetchAndLoadSync().ok()) { + PS_LOG(ERROR) << "Error fetching and loading reportWin udf for one " + "or more buyers"; + } + }); + return absl::OkStatus(); +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.h b/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.h new file mode 100644 index 00000000..92c81b65 --- /dev/null +++ b/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.h @@ -0,0 +1,110 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_BUYER_REPORTING_UDF_FETCH_MANAGER_H_ +#define SERVICES_BUYER_REPORTING_UDF_FETCH_MANAGER_H_ + +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/synchronization/mutex.h" +#include "services/auction_service/udf_fetcher/auction_code_fetch_config.pb.h" +#include "services/common/clients/code_dispatcher/v8_dispatcher.h" +#include "services/common/clients/http/http_fetcher_async.h" +#include "services/common/code_fetch/code_fetcher_interface.h" +#include "src/concurrent/executor.h" + +namespace privacy_sandbox::bidding_auction_servers { + +// This class manages: +// -Periodic fetch of the reportWin() udf for all buyers +// -Loading of the fetched blobs into Roma. +class BuyerReportingUdfFetchManager : public CodeFetcherInterface { + public: + explicit BuyerReportingUdfFetchManager( + const auction_service::SellerCodeFetchConfig* config, + HttpFetcherAsync* http_fetcher, server_common::Executor* executor, + WrapSingleCodeBlobForDispatch buyer_code_wrapper, + V8Dispatcher* dispatcher) + : config_(*config), + http_fetcher_(*http_fetcher), + executor_(*executor), + buyer_code_wrapper_(std::move(buyer_code_wrapper)), + dispatcher_(*dispatcher) {} + + ~BuyerReportingUdfFetchManager() { End(); } + + // Not copyable or movable. + BuyerReportingUdfFetchManager(const BuyerReportingUdfFetchManager&) = delete; + BuyerReportingUdfFetchManager& operator=( + const BuyerReportingUdfFetchManager&) = delete; + + // Starts a periodic code blob fetching process by fetching url with + // MultiCurlHttpFetcherAsync and loading the updated content into a publicly + // accessible map. + absl::Status Start() override; + + // Ends the periodic fetching process by canceling the last scheduled task + // using task_id_. + void End() override; + + // Return a map of buyer origin to reporting blob + // WARNING: do not use in critical paths, this returns a copy of a map. + absl::flat_hash_map + GetProtectedAudienceReportingByOriginForTesting() + ABSL_LOCKS_EXCLUDED(code_blob_per_origin_mu_); + + // WARNING: do not use in critical paths, this returns a copy of a map. + // return a map of buyer origin to reporting blob + absl::flat_hash_map + GetProtectedAppSignalsReportingByOriginForTesting() + ABSL_LOCKS_EXCLUDED(code_blob_per_origin_mu_); + + private: + absl::Status PeriodicBuyerReportingFetchAndLoadSync(); + // Acts as a lock on the code_blob_per_origin_ maps. + absl::Mutex code_blob_per_origin_mu_; + // Keeps track of the next task to be performed on the executor. + std::optional task_id_; + + const auction_service::SellerCodeFetchConfig& config_; + + // Configured as a constructor parameter to aid testing. + HttpFetcherAsync& http_fetcher_; + server_common::Executor& executor_; + WrapSingleCodeBlobForDispatch buyer_code_wrapper_; + V8Dispatcher& dispatcher_; + + absl::flat_hash_map + protected_auction_code_blob_per_origin_ + ABSL_GUARDED_BY(code_blob_per_origin_mu_); + absl::flat_hash_map + protected_app_signals_code_blob_per_origin_ + ABSL_GUARDED_BY(code_blob_per_origin_mu_); + void LoadBuyerCode(const std::string& version, + const std::string& fetched_blob); + void OnProtectedAudienceUdfFetch(const std::string& buyer_origin, + std::string& udf); + void OnProtectedAppSignalUdfFetch(const std::string& buyer_origin, + std::string& udf); +}; + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_BUYER_REPORTING_UDF_FETCH_MANAGER_H_ diff --git a/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager_test.cc b/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager_test.cc new file mode 100644 index 00000000..8ffcdada --- /dev/null +++ b/services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager_test.cc @@ -0,0 +1,288 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.h" + +#include +#include + +#include "absl/status/status.h" +#include "absl/synchronization/blocking_counter.h" +#include "gtest/gtest.h" +#include "services/auction_service/udf_fetcher/auction_code_fetch_config.pb.h" +#include "services/common/code_fetch/periodic_code_fetcher.h" +#include "services/common/test/mocks.h" +#include "src/concurrent/event_engine_executor.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { + +struct BuyerConfig { + std::string pa_buyer_origin = "http://PABuyerOrigin.com"; + std::string pas_buyer_origin = "http://PASBuyerOrigin.com"; + std::string pa_buyer_udf_url = "foo.com"; + std::string pas_buyer_udf_url = "bar.com"; + bool enable_report_win_url_generation = true; +}; + +auction_service::SellerCodeFetchConfig GetTestSellerUdfConfig( + const BuyerConfig& buyer_config) { + auction_service::SellerCodeFetchConfig udf_config; + udf_config.set_enable_report_win_url_generation( + buyer_config.enable_report_win_url_generation); + udf_config.mutable_buyer_report_win_js_urls()->try_emplace( + buyer_config.pa_buyer_origin, buyer_config.pa_buyer_udf_url); + udf_config.mutable_protected_app_signals_buyer_report_win_js_urls() + ->try_emplace(buyer_config.pas_buyer_origin, + buyer_config.pas_buyer_udf_url); + udf_config.set_url_fetch_timeout_ms(70000000); + udf_config.set_url_fetch_period_ms(70000001); + return udf_config; +} + +auction_service::SellerCodeFetchConfig +GetTestSellerUdfConfigWithNoUrlsConfigured(const BuyerConfig& buyer_config) { + auction_service::SellerCodeFetchConfig udf_config; + udf_config.set_enable_report_win_url_generation( + buyer_config.enable_report_win_url_generation); + udf_config.set_url_fetch_timeout_ms(70000000); + udf_config.set_url_fetch_period_ms(70000001); + return udf_config; +} + +TEST(BuyerReportingUdfFetchManagerTest, + LoadsHttpFetcherResultIntoV8Dispatcher) { + std::string expected_pa_version = "pa_http://PABuyerOrigin.com"; + std::string expected_pas_version = "pas_http://PASBuyerOrigin.com"; + std::vector> url_response = { + R"JS_CODE(reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals){ +})JS_CODE", + R"JS_CODE(reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals){ +})JS_CODE"}; + + std::string expected_wrapped_code = + R"JS_CODE(reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals){ +})JS_CODE"; + + BuyerConfig buyer_config; + auction_service::SellerCodeFetchConfig udf_config = + GetTestSellerUdfConfig(buyer_config); + MockV8Dispatcher dispatcher; + auto curl_http_fetcher = std::make_unique(); + auto executor = std::make_unique(); + WrapSingleCodeBlobForDispatch buyer_code_wrapper = + [expected_wrapped_code](const std::string& adtech_code_blob) { + return expected_wrapped_code; + }; + absl::BlockingCounter done(2); + EXPECT_CALL(*curl_http_fetcher, FetchUrls) + .WillOnce([&buyer_config, &url_response]( + const std::vector& requests, + absl::Duration timeout, + absl::AnyInvocable>)&&> + done_callback) { + EXPECT_EQ(buyer_config.pa_buyer_udf_url, requests[0].url); + EXPECT_EQ(buyer_config.pas_buyer_udf_url, requests[1].url); + std::move(done_callback)(url_response); + }); + + EXPECT_CALL(dispatcher, LoadSync) + .WillRepeatedly([&done, &expected_wrapped_code, &expected_pa_version, + &expected_pas_version](std::string_view version, + absl::string_view js) { + PS_VLOG(4) << "version:" << version + << " expected:" << expected_pa_version << " or " + << expected_pas_version << "\n"; + EXPECT_TRUE(version == expected_pa_version || + version == expected_pas_version); + EXPECT_EQ(js, expected_wrapped_code); + done.DecrementCount(); + return absl::OkStatus(); + }); + + BuyerReportingUdfFetchManager code_fetcher_manager( + &udf_config, curl_http_fetcher.get(), executor.get(), + std::move(buyer_code_wrapper), &dispatcher); + auto status = code_fetcher_manager.Start(); + ASSERT_TRUE(status.ok()) << status; + done.Wait(); + absl::flat_hash_map + protected_audience_code_blob_per_origin = + code_fetcher_manager + .GetProtectedAudienceReportingByOriginForTesting(); + absl::flat_hash_map + protected_app_signals_code_blob_per_origin = + code_fetcher_manager + .GetProtectedAppSignalsReportingByOriginForTesting(); + EXPECT_EQ(protected_audience_code_blob_per_origin.size(), 1); + EXPECT_EQ(protected_app_signals_code_blob_per_origin.size(), 1); + EXPECT_EQ( + protected_audience_code_blob_per_origin.at(buyer_config.pa_buyer_origin), + expected_wrapped_code); + EXPECT_EQ(protected_app_signals_code_blob_per_origin.at( + buyer_config.pas_buyer_origin), + expected_wrapped_code); + code_fetcher_manager.End(); +} + +TEST(BuyerReportingUdfFetchManagerTest, NoCodeLoadedWhenFlagsTurnedOff) { + BuyerConfig buyer_config = {.enable_report_win_url_generation = false}; + auction_service::SellerCodeFetchConfig udf_config = + GetTestSellerUdfConfig(buyer_config); + MockV8Dispatcher dispatcher; + std::string expected_wrapped_code = + R"JS_CODE(reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals){ +})JS_CODE"; + auto curl_http_fetcher = std::make_unique(); + auto executor = std::make_unique(); + WrapSingleCodeBlobForDispatch buyer_code_wrapper = + [expected_wrapped_code](const std::string& adtech_code_blob) { + return expected_wrapped_code; + }; + EXPECT_CALL(*curl_http_fetcher, FetchUrls).Times(0); + BuyerReportingUdfFetchManager code_fetcher_manager( + &udf_config, curl_http_fetcher.get(), executor.get(), + std::move(buyer_code_wrapper), &dispatcher); + auto status = code_fetcher_manager.Start(); + ASSERT_TRUE(status.ok()) << status; + absl::flat_hash_map + protected_audience_code_blob_per_origin = + code_fetcher_manager + .GetProtectedAudienceReportingByOriginForTesting(); + absl::flat_hash_map + protected_app_signals_code_blob_per_origin = + code_fetcher_manager + .GetProtectedAppSignalsReportingByOriginForTesting(); + EXPECT_EQ(protected_audience_code_blob_per_origin.size(), 0); + EXPECT_EQ(protected_app_signals_code_blob_per_origin.size(), 0); + code_fetcher_manager.End(); +} + +TEST(BuyerReportingUdfFetchManagerTest, NoCodeLoadedWhenNoFetchUrlConfigured) { + std::string expected_wrapped_code = + R"JS_CODE(reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals){ +})JS_CODE"; + + BuyerConfig buyer_config; + auction_service::SellerCodeFetchConfig udf_config = + GetTestSellerUdfConfigWithNoUrlsConfigured(buyer_config); + MockV8Dispatcher dispatcher; + auto curl_http_fetcher = std::make_unique(); + auto executor = std::make_unique(); + WrapSingleCodeBlobForDispatch buyer_code_wrapper = + [expected_wrapped_code](const std::string& adtech_code_blob) { + return expected_wrapped_code; + }; + EXPECT_CALL(*curl_http_fetcher, FetchUrls).Times(0); + BuyerReportingUdfFetchManager code_fetcher_manager( + &udf_config, curl_http_fetcher.get(), executor.get(), + std::move(buyer_code_wrapper), &dispatcher); + auto status = code_fetcher_manager.Start(); + ASSERT_TRUE(status.ok()) << status; + absl::flat_hash_map + protected_audience_code_blob_per_origin = + code_fetcher_manager + .GetProtectedAudienceReportingByOriginForTesting(); + absl::flat_hash_map + protected_app_signals_code_blob_per_origin = + code_fetcher_manager + .GetProtectedAppSignalsReportingByOriginForTesting(); + EXPECT_EQ(protected_audience_code_blob_per_origin.size(), 0); + EXPECT_EQ(protected_app_signals_code_blob_per_origin.size(), 0); + code_fetcher_manager.End(); +} + +TEST(BuyerReportingUdfFetchManagerTest, SkipsBuyerCodeUponFetchError) { + std::string expected_pa_version = "pa_http://PABuyerOrigin.com"; + std::string expected_pas_version = "pas_http://PASBuyerOrigin.com"; + std::vector> url_response = { + R"JS_CODE(reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals){ +})JS_CODE", + absl::Status( + absl::StatusCode::kInternal, + "Error fetching and loading one or more buyer's reportWin() udf.")}; + + std::string expected_wrapped_code = + R"JS_CODE(reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals){ +})JS_CODE"; + + BuyerConfig buyer_config; + auction_service::SellerCodeFetchConfig udf_config = + GetTestSellerUdfConfig(buyer_config); + MockV8Dispatcher dispatcher; + auto curl_http_fetcher = std::make_unique(); + auto executor = std::make_unique(); + WrapSingleCodeBlobForDispatch buyer_code_wrapper = + [expected_wrapped_code](const std::string& adtech_code_blob) { + return expected_wrapped_code; + }; + absl::Notification fetch_done; + EXPECT_CALL(*curl_http_fetcher, FetchUrls) + .WillOnce([&buyer_config, &url_response]( + const std::vector& requests, + absl::Duration timeout, + absl::AnyInvocable>)&&> + done_callback) { + EXPECT_EQ(buyer_config.pa_buyer_udf_url, requests[0].url); + EXPECT_EQ(buyer_config.pas_buyer_udf_url, requests[1].url); + std::move(done_callback)(url_response); + }); + + EXPECT_CALL(dispatcher, LoadSync) + .WillOnce([&fetch_done, &expected_wrapped_code, &expected_pa_version, + &expected_pas_version](std::string_view version, + absl::string_view js) { + PS_VLOG(4) << "version:" << version + << " expected:" << expected_pa_version << " or " + << expected_pas_version << "\n"; + EXPECT_EQ(version, expected_pa_version); + EXPECT_EQ(js, expected_wrapped_code); + fetch_done.Notify(); + return absl::OkStatus(); + }); + + BuyerReportingUdfFetchManager code_fetcher_manager( + &udf_config, curl_http_fetcher.get(), executor.get(), + std::move(buyer_code_wrapper), &dispatcher); + auto status = code_fetcher_manager.Start(); + ASSERT_TRUE(status.ok()) << status; + fetch_done.WaitForNotification(); + absl::flat_hash_map + protected_audience_code_blob_per_origin = + code_fetcher_manager + .GetProtectedAudienceReportingByOriginForTesting(); + absl::flat_hash_map + protected_app_signals_code_blob_per_origin = + code_fetcher_manager + .GetProtectedAppSignalsReportingByOriginForTesting(); + EXPECT_EQ(protected_audience_code_blob_per_origin.size(), 1); + EXPECT_EQ(protected_app_signals_code_blob_per_origin.size(), 0); + EXPECT_EQ( + protected_audience_code_blob_per_origin.at(buyer_config.pa_buyer_origin), + expected_wrapped_code); + code_fetcher_manager.End(); +} +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/seller_code_fetch_manager.cc b/services/auction_service/udf_fetcher/seller_udf_fetch_manager.cc similarity index 68% rename from services/auction_service/seller_code_fetch_manager.cc rename to services/auction_service/udf_fetcher/seller_udf_fetch_manager.cc index 5e449b7c..77429549 100644 --- a/services/auction_service/seller_code_fetch_manager.cc +++ b/services/auction_service/udf_fetcher/seller_udf_fetch_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "services/auction_service/seller_code_fetch_manager.h" +#include "services/auction_service/udf_fetcher/seller_udf_fetch_manager.h" #include #include @@ -20,10 +20,12 @@ #include #include -#include "services/auction_service/auction_code_fetch_config.pb.h" #include "services/auction_service/auction_constants.h" -#include "services/auction_service/code_wrapper/buyer_reporting_fetcher.h" +#include "services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h" #include "services/auction_service/code_wrapper/seller_code_wrapper.h" +#include "services/auction_service/code_wrapper/seller_udf_wrapper.h" +#include "services/auction_service/udf_fetcher/auction_code_fetch_config.pb.h" +#include "services/auction_service/udf_fetcher/buyer_reporting_fetcher.h" #include "services/common/clients/code_dispatcher/v8_dispatcher.h" #include "services/common/code_fetch/code_fetcher_interface.h" #include "services/common/code_fetch/periodic_bucket_fetcher.h" @@ -33,8 +35,6 @@ #include "src/core/interface/errors.h" #include "src/util/status_macro/status_macros.h" -#include "seller_code_fetch_manager.h" - using ::google::scp::core::errors::GetErrorMessage; namespace privacy_sandbox::bidding_auction_servers { @@ -44,23 +44,31 @@ constexpr int kJsBlobIndex = 0; } // namespace -absl::Status SellerCodeFetchManager::Init() { - if (udf_config_.fetch_mode() != auction_service::FETCH_MODE_LOCAL) { - buyer_reporting_fetcher_ = std::make_unique( - udf_config_, &buyer_reporting_http_fetcher_, &executor_); - PS_RETURN_IF_ERROR(buyer_reporting_fetcher_->Start()) - << kBuyerReportingFailedStartup; +absl::Status SellerUdfFetchManager::Init() { + if (udf_config_.fetch_mode() != blob_fetch::FETCH_MODE_LOCAL) { + if (udf_config_.enable_seller_and_buyer_udf_isolation()) { + buyer_reporting_udf_fetch_manager_ = + std::make_unique( + &udf_config_, &buyer_reporting_http_fetcher_, &executor_, + GetUdfWrapperForBuyer(), &dispatcher_); + PS_RETURN_IF_ERROR(buyer_reporting_udf_fetch_manager_->Start()) + << kBuyerReportingFailedStartup; + } else { + buyer_reporting_fetcher_ = std::make_unique( + udf_config_, &buyer_reporting_http_fetcher_, &executor_); + PS_RETURN_IF_ERROR(buyer_reporting_fetcher_->Start()) + << kBuyerReportingFailedStartup; + } } - switch (udf_config_.fetch_mode()) { - case auction_service::FETCH_MODE_LOCAL: { + case blob_fetch::FETCH_MODE_LOCAL: { return InitializeLocalCodeFetch(); } - case auction_service::FETCH_MODE_BUCKET: { + case blob_fetch::FETCH_MODE_BUCKET: { PS_ASSIGN_OR_RETURN(seller_code_fetcher_, InitializeBucketCodeFetch()); return absl::OkStatus(); } - case auction_service::FETCH_MODE_URL: { + case blob_fetch::FETCH_MODE_URL: { PS_ASSIGN_OR_RETURN(seller_code_fetcher_, InitializeUrlCodeFetch()); return absl::OkStatus(); } @@ -70,15 +78,32 @@ absl::Status SellerCodeFetchManager::Init() { } } -absl::Status SellerCodeFetchManager::End() { - if (udf_config_.fetch_mode() != auction_service::FETCH_MODE_LOCAL) { - buyer_reporting_fetcher_->End(); +absl::Status SellerUdfFetchManager::End() { + if (udf_config_.fetch_mode() != blob_fetch::FETCH_MODE_LOCAL) { + if (udf_config_.enable_seller_and_buyer_udf_isolation()) { + buyer_reporting_udf_fetch_manager_->End(); + } else { + buyer_reporting_fetcher_->End(); + } seller_code_fetcher_->End(); } return absl::OkStatus(); } -WrapCodeForDispatch SellerCodeFetchManager::GetUdfWrapper() { +WrapSingleCodeBlobForDispatch SellerUdfFetchManager::GetUdfWrapperForBuyer() { + return [](const std::string& ad_tech_code_blob) { + return GetBuyerWrappedCode(ad_tech_code_blob); + }; +} + +WrapCodeForDispatch SellerUdfFetchManager::GetUdfWrapper() { + if (udf_config_.enable_seller_and_buyer_udf_isolation()) { + return [this](const std::vector& ad_tech_code_blobs) { + return GetSellerWrappedCode( + ad_tech_code_blobs[kJsBlobIndex], + udf_config_.enable_report_result_url_generation()); + }; + } return [this](const std::vector& ad_tech_code_blobs) { auto protected_auction_reporting = buyer_reporting_fetcher_->GetProtectedAuctionReportingByOrigin(); @@ -93,7 +118,7 @@ WrapCodeForDispatch SellerCodeFetchManager::GetUdfWrapper() { }; } -absl::Status SellerCodeFetchManager::InitializeLocalCodeFetch() { +absl::Status SellerUdfFetchManager::InitializeLocalCodeFetch() { if (udf_config_.auction_js_path().empty()) { return absl::UnavailableError( "Local fetch mode requires a non-empty path."); @@ -111,7 +136,7 @@ absl::Status SellerCodeFetchManager::InitializeLocalCodeFetch() { } absl::StatusOr> -SellerCodeFetchManager::InitializeBucketCodeFetch() { +SellerUdfFetchManager::InitializeBucketCodeFetch() { PS_RETURN_IF_ERROR(InitBucketClient()); std::string bucket_name = udf_config_.auction_js_bucket(); @@ -133,7 +158,7 @@ SellerCodeFetchManager::InitializeBucketCodeFetch() { } absl::StatusOr> -SellerCodeFetchManager::InitializeUrlCodeFetch() { +SellerUdfFetchManager::InitializeUrlCodeFetch() { if (udf_config_.auction_js_url().empty()) { return absl::InvalidArgumentError( "URL fetch mode requires a non-empty url."); @@ -150,12 +175,11 @@ SellerCodeFetchManager::InitializeUrlCodeFetch() { return seller_code_fetcher; } -absl::Status SellerCodeFetchManager::InitBucketClient() { +absl::Status SellerUdfFetchManager::InitBucketClient() { PS_RETURN_IF_ERROR(blob_storage_client_->Init()).SetPrepend() << "Failed to init BlobStorageClient: "; PS_RETURN_IF_ERROR(blob_storage_client_->Run()).SetPrepend() << "Failed to run BlobStorageClient: "; return absl::OkStatus(); } - } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/seller_code_fetch_manager.h b/services/auction_service/udf_fetcher/seller_udf_fetch_manager.h similarity index 77% rename from services/auction_service/seller_code_fetch_manager.h rename to services/auction_service/udf_fetcher/seller_udf_fetch_manager.h index 77245d6b..4bec34f7 100644 --- a/services/auction_service/seller_code_fetch_manager.h +++ b/services/auction_service/udf_fetcher/seller_udf_fetch_manager.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef SERVICES_SELLER_CODE_FETCH_MANAGER_H_ -#define SERVICES_SELLER_CODE_FETCH_MANAGER_H_ +#ifndef SERVICES_SELLER_UDF_FETCH_MANAGER_H_ +#define SERVICES_SELLER_UDF_FETCH_MANAGER_H_ #include #include @@ -21,9 +21,9 @@ #include #include -#include "services/auction_service/auction_code_fetch_config.pb.h" -#include "services/auction_service/code_wrapper/buyer_reporting_fetcher.h" -#include "services/auction_service/code_wrapper/seller_code_wrapper.h" +#include "services/auction_service/udf_fetcher/auction_code_fetch_config.pb.h" +#include "services/auction_service/udf_fetcher/buyer_reporting_fetcher.h" +#include "services/auction_service/udf_fetcher/buyer_reporting_udf_fetch_manager.h" #include "services/common/clients/code_dispatcher/v8_dispatcher.h" #include "services/common/clients/http/http_fetcher_async.h" #include "services/common/code_fetch/code_fetcher_interface.h" @@ -38,13 +38,13 @@ constexpr char kBuyerReportingFailedStartup[] = constexpr char kSellerUDFLoadFailedStartup[] = "Failed loading code blob on startup."; -// SellerCodeFetchManager acts as a wrapper for all logic related to fetching +// SellerUdfFetchManager acts as a wrapper for all logic related to fetching // auction service UDFs. This class consumes a SellerCodeFetchConfig and uses // it, along with various other dependencies, to create and own all instances of // CodeFetcherInterface in the auction service. -class SellerCodeFetchManager { +class SellerUdfFetchManager { public: - SellerCodeFetchManager( + SellerUdfFetchManager( std::unique_ptr blob_storage_client, server_common::Executor* executor, HttpFetcherAsync* seller_http_fetcher, @@ -60,8 +60,8 @@ class SellerCodeFetchManager { blob_storage_client_(std::move(blob_storage_client)) {} // Not copyable or movable. - SellerCodeFetchManager(const SellerCodeFetchManager&) = delete; - SellerCodeFetchManager& operator=(const SellerCodeFetchManager&) = delete; + SellerUdfFetchManager(const SellerUdfFetchManager&) = delete; + SellerUdfFetchManager& operator=(const SellerUdfFetchManager&) = delete; absl::Status Init(); @@ -69,6 +69,11 @@ class SellerCodeFetchManager { private: WrapCodeForDispatch GetUdfWrapper(); + WrapSingleCodeBlobForDispatch GetUdfWrapperForBuyer(); + + // This function will replace GetUdfWrapper once seller and buyer code + // isolation is enforced + WrapCodeForDispatch GetUdfWrapperForSeller(); absl::Status InitializeLocalCodeFetch(); @@ -89,9 +94,11 @@ class SellerCodeFetchManager { blob_storage_client_; std::unique_ptr buyer_reporting_fetcher_; + std::unique_ptr + buyer_reporting_udf_fetch_manager_; std::unique_ptr seller_code_fetcher_; }; } // namespace privacy_sandbox::bidding_auction_servers -#endif // SERVICES_SELLER_CODE_FETCH_MANAGER_H_ +#endif // SERVICES_SELLER_UDF_FETCH_MANAGER_H_ diff --git a/services/auction_service/udf_fetcher/seller_udf_fetch_manager_test.cc b/services/auction_service/udf_fetcher/seller_udf_fetch_manager_test.cc new file mode 100644 index 00000000..bfa3f921 --- /dev/null +++ b/services/auction_service/udf_fetcher/seller_udf_fetch_manager_test.cc @@ -0,0 +1,485 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/auction_service/udf_fetcher/seller_udf_fetch_manager.h" + +#include "absl/strings/str_cat.h" +#include "absl/synchronization/blocking_counter.h" +#include "absl/synchronization/notification.h" +#include "absl/time/time.h" +#include "gtest/gtest.h" +#include "services/auction_service/auction_constants.h" +#include "services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h" +#include "services/auction_service/code_wrapper/seller_code_wrapper.h" +#include "services/auction_service/code_wrapper/seller_udf_wrapper.h" +#include "services/common/test/mocks.h" +#include "services/common/util/file_util.h" +#include "src/core/interface/async_context.h" +#include "src/public/cpio/interface/blob_storage_client/blob_storage_client_interface.h" +#include "src/public/cpio/interface/cpio.h" +#include "src/public/cpio/interface/error_codes.h" +#include "src/public/cpio/mock/blob_storage_client/mock_blob_storage_client.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { + +using auction_service::SellerCodeFetchConfig; +using blob_fetch::FetchMode; + +using ::google::cmrt::sdk::blob_storage_service::v1::BlobMetadata; +using ::google::cmrt::sdk::blob_storage_service::v1::GetBlobRequest; +using ::google::cmrt::sdk::blob_storage_service::v1::GetBlobResponse; +using ::google::cmrt::sdk::blob_storage_service::v1::ListBlobsMetadataRequest; +using ::google::cmrt::sdk::blob_storage_service::v1::ListBlobsMetadataResponse; +using ::google::scp::core::AsyncContext; +using ::google::scp::core::ExecutionResult; +using ::google::scp::core::FailureExecutionResult; +using ::google::scp::core::SuccessExecutionResult; +using ::google::scp::cpio::MockBlobStorageClient; +using ::testing::Return; + +struct TestSellerUdfConfig { + std::string pa_buyer_origin = "http://PABuyerOrigin.com"; + std::string pas_buyer_origin = "http://PASBuyerOrigin.com"; + std::string seller_udf_url = "seller.com"; + std::string pa_buyer_udf_url = "foo.com"; + std::string pas_buyer_udf_url = "bar.com"; + bool enable_report_win_url_generation = true; + FetchMode fetch_mode = blob_fetch::FETCH_MODE_URL; + bool enable_seller_and_buyer_udf_isolation = false; + std::string auction_js_bucket = ""; + std::string auction_js_bucket_default_blob = ""; +}; + +class SellerUdfFetchManagerTest : public testing::Test { + protected: + void SetUp() override { + executor_ = std::make_unique(); + http_fetcher_ = std::make_unique(); + dispatcher_ = std::make_unique(); + blob_storage_client_ = std::make_unique(); + } + std::unique_ptr executor_; + std::unique_ptr http_fetcher_; + std::unique_ptr dispatcher_; + std::unique_ptr blob_storage_client_; +}; + +auction_service::SellerCodeFetchConfig GetTestSellerUdfConfig( + const TestSellerUdfConfig& test_seller_udf_config) { + SellerCodeFetchConfig udf_config; + udf_config.set_fetch_mode(test_seller_udf_config.fetch_mode); + udf_config.set_enable_report_win_url_generation( + test_seller_udf_config.enable_report_win_url_generation); + udf_config.set_enable_report_result_url_generation(true); + udf_config.mutable_buyer_report_win_js_urls()->try_emplace( + test_seller_udf_config.pa_buyer_origin, + test_seller_udf_config.pa_buyer_udf_url); + udf_config.mutable_protected_app_signals_buyer_report_win_js_urls() + ->try_emplace(test_seller_udf_config.pas_buyer_origin, + test_seller_udf_config.pas_buyer_udf_url); + udf_config.set_auction_js_url(test_seller_udf_config.seller_udf_url); + udf_config.set_url_fetch_timeout_ms(70000000); + udf_config.set_url_fetch_period_ms(70000001); + udf_config.set_enable_seller_and_buyer_udf_isolation( + test_seller_udf_config.enable_seller_and_buyer_udf_isolation); + udf_config.set_auction_js_bucket(test_seller_udf_config.auction_js_bucket); + udf_config.set_auction_js_bucket_default_blob( + test_seller_udf_config.auction_js_bucket_default_blob); + return udf_config; +} + +TEST_F(SellerUdfFetchManagerTest, FetchModeLocalTriesFileLoad) { + EXPECT_CALL(*blob_storage_client_, Init).Times(0); + EXPECT_CALL(*blob_storage_client_, Run).Times(0); + + SellerCodeFetchConfig udf_config; + udf_config.set_fetch_mode(blob_fetch::FETCH_MODE_LOCAL); + const std::string bad_path = "error"; + const bool enable_protected_app_signals = true; + + udf_config.set_auction_js_path(bad_path); + SellerUdfFetchManager udf_fetcher(std::move(blob_storage_client_), + executor_.get(), http_fetcher_.get(), + http_fetcher_.get(), dispatcher_.get(), + udf_config, enable_protected_app_signals); + absl::Status load_status = udf_fetcher.Init(); + EXPECT_FALSE(load_status.ok()); + EXPECT_EQ(load_status.message(), absl::StrCat(kPathFailed, bad_path)); +} + +TEST_F(SellerUdfFetchManagerTest, + FetchModeBucketSucceedsLoadingWrappedBucketBlobs) { + std::string fake_udf = "udf_data"; + std::string pa_buyer_origin = "pa_origin"; + std::string pas_buyer_origin = "pas_origin"; + std::string pa_reporting_udf_data = "pa_reporting_udf_data"; + std::string pas_reporting_udf_data = "pas_reporting_udf_data"; + const bool enable_protected_app_signals = true; + + SellerCodeFetchConfig udf_config; + udf_config.set_fetch_mode(blob_fetch::FETCH_MODE_BUCKET); + udf_config.set_enable_report_win_url_generation(true); + udf_config.set_enable_report_result_url_generation(true); + udf_config.mutable_buyer_report_win_js_urls()->try_emplace(pa_buyer_origin, + "foo.com"); + udf_config.mutable_protected_app_signals_buyer_report_win_js_urls() + ->try_emplace(pas_buyer_origin, "bar.com"); + udf_config.set_auction_js_bucket("js"); + udf_config.set_auction_js_bucket_default_blob("default"); + + EXPECT_CALL(*blob_storage_client_, Init).WillOnce(Return(absl::OkStatus())); + EXPECT_CALL(*blob_storage_client_, Run).WillOnce(Return(absl::OkStatus())); + EXPECT_CALL(*executor_, RunAfter).Times(2); + EXPECT_CALL(*http_fetcher_, FetchUrls) + .Times(1) + .WillOnce([&udf_config, &pa_buyer_origin, &pas_buyer_origin, + &pa_reporting_udf_data, &pas_reporting_udf_data]( + const std::vector& requests, + absl::Duration timeout, OnDoneFetchUrls done_callback) { + EXPECT_EQ(requests[0].url, + udf_config.buyer_report_win_js_urls().at(pa_buyer_origin)); + EXPECT_EQ( + requests[1].url, + udf_config.protected_app_signals_buyer_report_win_js_urls().at( + pas_buyer_origin)); + + std::move(done_callback)( + {pa_reporting_udf_data, pas_reporting_udf_data}); + }); + + EXPECT_CALL(*blob_storage_client_, ListBlobsMetadata) + .WillOnce( + [&udf_config]( + AsyncContext + context) { + EXPECT_EQ(context.request->blob_metadata().bucket_name(), + udf_config.auction_js_bucket()); + BlobMetadata md; + md.set_bucket_name(udf_config.auction_js_bucket()); + md.set_blob_name(udf_config.auction_js_bucket_default_blob()); + + context.response = std::make_shared(); + context.response->mutable_blob_metadatas()->Add(std::move(md)); + context.Finish(SuccessExecutionResult()); + return absl::OkStatus(); + }); + + EXPECT_CALL(*blob_storage_client_, GetBlob) + .WillOnce( + [&udf_config, &fake_udf]( + AsyncContext async_context) { + const auto& async_bucket_name = + async_context.request->blob_metadata().bucket_name(); + const auto& async_blob_name = + async_context.request->blob_metadata().blob_name(); + EXPECT_EQ(async_bucket_name, udf_config.auction_js_bucket()); + EXPECT_EQ(async_blob_name, + udf_config.auction_js_bucket_default_blob()); + + async_context.response = std::make_shared(); + async_context.response->mutable_blob()->set_data(fake_udf); + async_context.result = SuccessExecutionResult(); + async_context.Finish(); + + return absl::OkStatus(); + }); + + EXPECT_CALL(*dispatcher_, LoadSync) + .WillOnce([&udf_config, &fake_udf, &pa_buyer_origin, &pas_buyer_origin, + &pa_reporting_udf_data, &pas_reporting_udf_data]( + std::string_view version, absl::string_view blob_data) { + EXPECT_EQ(version, udf_config.auction_js_bucket_default_blob()); + absl::flat_hash_map pa_reporting_udfs; + pa_reporting_udfs.try_emplace(pa_buyer_origin, pa_reporting_udf_data); + absl::flat_hash_map pas_reporting_udfs; + pas_reporting_udfs.try_emplace(pas_buyer_origin, + pas_reporting_udf_data); + + EXPECT_EQ( + blob_data, + GetSellerWrappedCode( + fake_udf, udf_config.enable_report_result_url_generation(), + enable_protected_app_signals, + udf_config.enable_report_win_url_generation(), + pa_reporting_udfs, pas_reporting_udfs)); + return absl::OkStatus(); + }); + + SellerUdfFetchManager udf_fetcher(std::move(blob_storage_client_), + executor_.get(), http_fetcher_.get(), + http_fetcher_.get(), dispatcher_.get(), + udf_config, enable_protected_app_signals); + absl::Status load_status = udf_fetcher.Init(); + EXPECT_TRUE(load_status.ok()); +} + +TEST_F(SellerUdfFetchManagerTest, + FetchModeBucketSucceedsWithSellerAndBuyerUdfIsolation) { + std::string expected_seller_udf = R"JS_CODE( +scoreAd = function(ad_metadata, bid, auction_config, scoring_signals, bid_metadata, directFromSellerSignals){ +} +reportResult = function(auctionConfig, sellerReportingSignals, directFromSellerSignals){ +} +)JS_CODE"; + std::string expected_buyer_udf = R"JS_CODE( +reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals){ +} +)JS_CODE"; + std::string expected_pa_version = "pa_http://PABuyerOrigin.com"; + std::string expected_pas_version = "pas_http://PASBuyerOrigin.com"; + bool enable_protected_app_signals = true; + TestSellerUdfConfig test_udf_config = { + .fetch_mode = blob_fetch::FETCH_MODE_BUCKET, + .enable_seller_and_buyer_udf_isolation = true, + .auction_js_bucket = "js", + .auction_js_bucket_default_blob = "default"}; + SellerCodeFetchConfig udf_config = GetTestSellerUdfConfig(test_udf_config); + + EXPECT_CALL(*blob_storage_client_, Init).WillOnce(Return(absl::OkStatus())); + EXPECT_CALL(*blob_storage_client_, Run).WillOnce(Return(absl::OkStatus())); + EXPECT_CALL(*executor_, RunAfter).Times(2); + EXPECT_CALL(*http_fetcher_, FetchUrls) + .Times(1) + .WillOnce([&test_udf_config, &expected_buyer_udf]( + const std::vector& requests, + absl::Duration timeout, OnDoneFetchUrls done_callback) { + EXPECT_EQ(requests[0].url, test_udf_config.pa_buyer_udf_url); + EXPECT_EQ(requests[1].url, test_udf_config.pas_buyer_udf_url); + + std::move(done_callback)({expected_buyer_udf, expected_buyer_udf}); + }); + + EXPECT_CALL(*blob_storage_client_, ListBlobsMetadata) + .WillOnce( + [&udf_config]( + AsyncContext + context) { + EXPECT_EQ(context.request->blob_metadata().bucket_name(), + udf_config.auction_js_bucket()); + BlobMetadata md; + md.set_bucket_name(udf_config.auction_js_bucket()); + md.set_blob_name(udf_config.auction_js_bucket_default_blob()); + + context.response = std::make_shared(); + context.response->mutable_blob_metadatas()->Add(std::move(md)); + context.Finish(SuccessExecutionResult()); + return absl::OkStatus(); + }); + + EXPECT_CALL(*blob_storage_client_, GetBlob) + .WillOnce( + [&udf_config, &expected_seller_udf]( + AsyncContext async_context) { + const auto& async_bucket_name = + async_context.request->blob_metadata().bucket_name(); + const auto& async_blob_name = + async_context.request->blob_metadata().blob_name(); + EXPECT_EQ(async_bucket_name, udf_config.auction_js_bucket()); + EXPECT_EQ(async_blob_name, + udf_config.auction_js_bucket_default_blob()); + + async_context.response = std::make_shared(); + async_context.response->mutable_blob()->set_data( + expected_seller_udf); + async_context.result = SuccessExecutionResult(); + async_context.Finish(); + + return absl::OkStatus(); + }); + + EXPECT_CALL(*dispatcher_, LoadSync) + .Times(3) + .WillOnce([&expected_buyer_udf, &expected_pa_version]( + std::string_view version, absl::string_view blob_data) { + EXPECT_EQ(version, expected_pa_version); + + EXPECT_EQ(blob_data, GetBuyerWrappedCode(expected_buyer_udf)); + return absl::OkStatus(); + }) + .WillOnce([&expected_buyer_udf, &expected_pas_version]( + std::string_view version, absl::string_view blob_data) { + EXPECT_EQ(version, expected_pas_version); + + EXPECT_EQ(blob_data, GetBuyerWrappedCode(expected_buyer_udf)); + return absl::OkStatus(); + }) + .WillOnce([&udf_config, &expected_seller_udf]( + std::string_view version, absl::string_view blob_data) { + EXPECT_EQ(version, udf_config.auction_js_bucket_default_blob()); + EXPECT_EQ(blob_data, + GetSellerWrappedCode( + expected_seller_udf, + udf_config.enable_report_result_url_generation())); + return absl::OkStatus(); + }); + + SellerUdfFetchManager udf_fetcher(std::move(blob_storage_client_), + executor_.get(), http_fetcher_.get(), + http_fetcher_.get(), dispatcher_.get(), + udf_config, enable_protected_app_signals); + absl::Status load_status = udf_fetcher.Init(); + EXPECT_TRUE(load_status.ok()); +} + +TEST_F(SellerUdfFetchManagerTest, + FetchModeUrlSucceedsWithSellerAndBuyerCodeIsolation) { + std::string expected_seller_udf = R"JS_CODE( +scoreAd = function(ad_metadata, bid, auction_config, scoring_signals, bid_metadata, directFromSellerSignals){ +} +reportResult = function(auctionConfig, sellerReportingSignals, directFromSellerSignals){ +} +)JS_CODE"; + std::string expected_buyer_udf = R"JS_CODE( +reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals){ +} +)JS_CODE"; + std::string expected_pa_version = "pa_http://PABuyerOrigin.com"; + std::string expected_pas_version = "pas_http://PASBuyerOrigin.com"; + bool enable_protected_app_signals = true; + TestSellerUdfConfig test_udf_config = { + .enable_seller_and_buyer_udf_isolation = true}; + SellerCodeFetchConfig udf_config = GetTestSellerUdfConfig(test_udf_config); + EXPECT_CALL(*blob_storage_client_, Init).Times(0); + EXPECT_CALL(*blob_storage_client_, Run).Times(0); + EXPECT_CALL(*executor_, RunAfter).Times(2); + EXPECT_CALL(*http_fetcher_, FetchUrls) + .Times(2) + .WillOnce([&test_udf_config, &expected_buyer_udf]( + const std::vector& requests, + absl::Duration timeout, OnDoneFetchUrls done_callback) { + EXPECT_EQ(requests[0].url, test_udf_config.pa_buyer_udf_url); + EXPECT_EQ(requests[1].url, test_udf_config.pas_buyer_udf_url); + + std::move(done_callback)({expected_buyer_udf, expected_buyer_udf}); + }) + .WillOnce([&udf_config, &expected_seller_udf]( + const std::vector& requests, + absl::Duration timeout, OnDoneFetchUrls done_callback) { + EXPECT_EQ(requests[0].url, udf_config.auction_js_url()); + std::move(done_callback)({expected_seller_udf}); + }); + + EXPECT_CALL(*dispatcher_, LoadSync) + .Times(3) + .WillOnce([&expected_buyer_udf, &expected_pa_version]( + std::string_view version, absl::string_view blob_data) { + EXPECT_EQ(version, expected_pa_version); + + EXPECT_EQ(blob_data, GetBuyerWrappedCode(expected_buyer_udf)); + return absl::OkStatus(); + }) + .WillOnce([&expected_buyer_udf, &expected_pas_version]( + std::string_view version, absl::string_view blob_data) { + EXPECT_EQ(version, expected_pas_version); + + EXPECT_EQ(blob_data, GetBuyerWrappedCode(expected_buyer_udf)); + return absl::OkStatus(); + }) + .WillOnce([&udf_config, &expected_seller_udf]( + std::string_view version, absl::string_view blob_data) { + EXPECT_EQ(version, kScoreAdBlobVersion); + EXPECT_EQ(blob_data, + GetSellerWrappedCode( + expected_seller_udf, + udf_config.enable_report_result_url_generation())); + return absl::OkStatus(); + }); + + SellerUdfFetchManager udf_fetcher(std::move(blob_storage_client_), + executor_.get(), http_fetcher_.get(), + http_fetcher_.get(), dispatcher_.get(), + udf_config, enable_protected_app_signals); + absl::Status load_status = udf_fetcher.Init(); + EXPECT_TRUE(load_status.ok()); +} + +TEST_F(SellerUdfFetchManagerTest, FetchModeUrlSucceedsLoadingWrappedUrlBlobs) { + std::string fake_udf = "udf_data"; + std::string pa_buyer_origin = "pa_origin"; + std::string pas_buyer_origin = "pas_origin"; + std::string pa_reporting_udf_data = "pa_reporting_udf_data"; + std::string pas_reporting_udf_data = "pas_reporting_udf_data"; + const bool enable_protected_app_signals = true; + + SellerCodeFetchConfig udf_config; + udf_config.set_fetch_mode(blob_fetch::FETCH_MODE_URL); + udf_config.set_enable_report_win_url_generation(true); + udf_config.set_enable_report_result_url_generation(true); + udf_config.mutable_buyer_report_win_js_urls()->try_emplace(pa_buyer_origin, + "foo.com"); + udf_config.mutable_protected_app_signals_buyer_report_win_js_urls() + ->try_emplace(pas_buyer_origin, "bar.com"); + udf_config.set_auction_js_url("auction.com"); + udf_config.set_url_fetch_timeout_ms(70000000); + udf_config.set_url_fetch_period_ms(70000001); + EXPECT_CALL(*blob_storage_client_, Init).Times(0); + EXPECT_CALL(*blob_storage_client_, Run).Times(0); + EXPECT_CALL(*executor_, RunAfter).Times(2); + EXPECT_CALL(*http_fetcher_, FetchUrls) + .Times(2) + .WillOnce([&udf_config, &pa_buyer_origin, &pas_buyer_origin, + &pa_reporting_udf_data, &pas_reporting_udf_data]( + const std::vector& requests, + absl::Duration timeout, OnDoneFetchUrls done_callback) { + EXPECT_EQ(requests[0].url, + udf_config.buyer_report_win_js_urls().at(pa_buyer_origin)); + EXPECT_EQ( + requests[1].url, + udf_config.protected_app_signals_buyer_report_win_js_urls().at( + pas_buyer_origin)); + + std::move(done_callback)( + {pa_reporting_udf_data, pas_reporting_udf_data}); + }) + .WillOnce([&udf_config, &fake_udf]( + const std::vector& requests, + absl::Duration timeout, OnDoneFetchUrls done_callback) { + EXPECT_EQ(requests[0].url, udf_config.auction_js_url()); + std::move(done_callback)({fake_udf}); + }); + + EXPECT_CALL(*dispatcher_, LoadSync) + .WillOnce([&udf_config, &fake_udf, &pa_buyer_origin, &pas_buyer_origin, + &pa_reporting_udf_data, &pas_reporting_udf_data]( + std::string_view version, absl::string_view blob_data) { + EXPECT_EQ(version, kScoreAdBlobVersion); + absl::flat_hash_map pa_reporting_udfs; + pa_reporting_udfs.try_emplace(pa_buyer_origin, pa_reporting_udf_data); + absl::flat_hash_map pas_reporting_udfs; + pas_reporting_udfs.try_emplace(pas_buyer_origin, + pas_reporting_udf_data); + + EXPECT_EQ( + blob_data, + GetSellerWrappedCode( + fake_udf, udf_config.enable_report_result_url_generation(), + enable_protected_app_signals, + udf_config.enable_report_win_url_generation(), + pa_reporting_udfs, pas_reporting_udfs)); + return absl::OkStatus(); + }); + + SellerUdfFetchManager udf_fetcher(std::move(blob_storage_client_), + executor_.get(), http_fetcher_.get(), + http_fetcher_.get(), dispatcher_.get(), + udf_config, enable_protected_app_signals); + absl::Status load_status = udf_fetcher.Init(); + EXPECT_TRUE(load_status.ok()); +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/utils/BUILD b/services/auction_service/utils/BUILD index 79fdf25e..cf52d2a0 100644 --- a/services/auction_service/utils/BUILD +++ b/services/auction_service/utils/BUILD @@ -29,13 +29,13 @@ cc_library( "//services/auction_service:auction_constants", "//services/auction_service/code_wrapper:seller_code_wrapper", "//services/common/clients/code_dispatcher:code_dispatch_client", + "//services/common/loggers:request_log_context", "//services/common/util:json_util", "//services/common/util:reporting_util", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", "@rapidjson", ], diff --git a/services/auction_service/utils/proto_utils.cc b/services/auction_service/utils/proto_utils.cc index f19c4c6d..a1ab0e1d 100644 --- a/services/auction_service/utils/proto_utils.cc +++ b/services/auction_service/utils/proto_utils.cc @@ -29,7 +29,6 @@ using AdWithBidMetadata = ScoreAdsRequest::ScoreAdsRawRequest::AdWithBidMetadata; using ProtectedAppSignalsAdWithBidMetadata = ScoreAdsRequest::ScoreAdsRawRequest::ProtectedAppSignalsAdWithBidMetadata; -using server_common::log::ContextImpl; namespace { @@ -185,7 +184,7 @@ std::string MakeOpenBidMetadataJson( } else { absl::StrAppend(&bid_metadata, R"JSON(")JSON", kBidCurrencyPropertyForScoreAd, R"JSON(":")JSON", - kEmptyBidCurrencyCode, R"JSON(",)JSON"); + kUnknownBidCurrencyCode, R"JSON(",)JSON"); } return bid_metadata; } @@ -227,7 +226,7 @@ inline absl::string_view GetDebugUrlIfInLimit( } // namespace void MayLogScoreAdsInput(const std::vector>& input, - ContextImpl& log_context) { + RequestLogContext& log_context) { PS_VLOG(kDispatch, log_context) << "\n\nScore Ad Input Args:" << "\nAdMetadata:\n" << *(input[ScoreArgIndex(ScoreAdArgs::kAdMetadata)]) << "\nBid:\n" @@ -293,7 +292,7 @@ std::string MakeBidMetadataForTopLevelAuction( absl::StatusOr> BuildTrustedScoringSignals( const ScoreAdsRequest::ScoreAdsRawRequest& raw_request, - ContextImpl& log_context) { + RequestLogContext& log_context) { rapidjson::Document trusted_scoring_signals_value; // TODO (b/285214424): De-nest, use a guard. if (!raw_request.scoring_signals().empty()) { @@ -391,7 +390,7 @@ void MayPopulateScoringSignalsForProtectedAppSignals( const ScoreAdsRequest::ScoreAdsRawRequest& raw_request, absl::flat_hash_map& render_url_signals, absl::flat_hash_map& combined_signals, - ContextImpl& log_context) { + RequestLogContext& log_context) { PS_VLOG(8, log_context) << __func__; for (const auto& protected_app_signals_ad_bid : raw_request.protected_app_signals_ad_bids()) { @@ -420,7 +419,7 @@ void MayPopulateScoringSignalsForProtectedAppSignals( absl::StatusOr ParseAndGetScoreAdResponseJson( bool enable_ad_tech_code_logging, const std::string& response, - ContextImpl& log_context) { + RequestLogContext& log_context) { PS_ASSIGN_OR_RETURN(rapidjson::Document document, ParseJsonString(response)); MayVlogAdTechCodeLogs(enable_ad_tech_code_logging, document, log_context); rapidjson::Document response_obj; @@ -439,7 +438,7 @@ std::optional ParseAdRejectionReason(const rapidjson::Document& score_ad_resp, absl::string_view interest_group_owner, absl::string_view interest_group_name, - ContextImpl& log_context) { + RequestLogContext& log_context) { auto reject_reason_itr = score_ad_resp.FindMember(kRejectReasonPropertyForScoreAd); if (reject_reason_itr == score_ad_resp.MemberEnd() || @@ -574,8 +573,7 @@ absl::StatusOr BuildScoreAdRequest( absl::string_view ad_render_url, absl::string_view ad_metadata_json, absl::string_view scoring_signals, float ad_bid, const std::shared_ptr& auction_config, - absl::string_view bid_metadata, - server_common::log::ContextImpl& log_context, + absl::string_view bid_metadata, RequestLogContext& log_context, const bool enable_adtech_code_logging, const bool enable_debug_reporting, absl::string_view code_version) { // Construct the wrapper struct for our V8 Dispatch Request. diff --git a/services/auction_service/utils/proto_utils.h b/services/auction_service/utils/proto_utils.h index da9cc69a..53ca9f3a 100644 --- a/services/auction_service/utils/proto_utils.h +++ b/services/auction_service/utils/proto_utils.h @@ -32,7 +32,7 @@ #include "services/auction_service/auction_constants.h" #include "services/auction_service/code_wrapper/seller_code_wrapper.h" #include "services/common/clients/code_dispatcher/v8_dispatcher.h" -#include "src/logger/request_context_impl.h" +#include "services/common/loggers/request_log_context.h" #include "src/util/status_macro/status_macros.h" namespace privacy_sandbox::bidding_auction_servers { @@ -57,26 +57,26 @@ std::shared_ptr BuildAuctionConfig( absl::StatusOr> BuildTrustedScoringSignals( const ScoreAdsRequest::ScoreAdsRawRequest& raw_request, - server_common::log::ContextImpl& log_context); + RequestLogContext& log_context); void MayPopulateScoringSignalsForProtectedAppSignals( const ScoreAdsRequest::ScoreAdsRawRequest& raw_request, absl::flat_hash_map& render_url_signals, absl::flat_hash_map& combined_signals, - server_common::log::ContextImpl& log_context); + RequestLogContext& log_context); void MayLogScoreAdsInput(const std::vector>& input, - server_common::log::ContextImpl& log_context); + RequestLogContext& log_context); absl::StatusOr ParseAndGetScoreAdResponseJson( bool enable_ad_tech_code_logging, const std::string& response, - server_common::log::ContextImpl& log_context); + RequestLogContext& log_context); std::optional ParseAdRejectionReason(const rapidjson::Document& score_ad_resp, absl::string_view interest_group_owner, absl::string_view interest_group_name, - server_common::log::ContextImpl& log_context); + RequestLogContext& log_context); absl::StatusOr ParseScoreAdResponse( const rapidjson::Document& score_ad_resp, @@ -116,8 +116,7 @@ absl::StatusOr BuildScoreAdRequest( absl::string_view ad_render_url, absl::string_view ad_metadata_json, absl::string_view scoring_signals, float ad_bid, const std::shared_ptr& auction_config, - absl::string_view bid_metadata, - server_common::log::ContextImpl& log_context, + absl::string_view bid_metadata, RequestLogContext& log_context, const bool enable_adtech_code_logging, const bool enable_debug_reporting, absl::string_view code_version); @@ -129,8 +128,7 @@ absl::StatusOr BuildScoreAdRequest( const T& ad, const std::shared_ptr& auction_config, const absl::flat_hash_map& scoring_signals, - const bool enable_debug_reporting, - server_common::log::ContextImpl& log_context, + const bool enable_debug_reporting, RequestLogContext& log_context, const bool enable_adtech_code_logging, absl::string_view bid_metadata, absl::string_view code_version) { std::string ad_object_json; diff --git a/services/auction_service/utils/proto_utils_test.cc b/services/auction_service/utils/proto_utils_test.cc index c20eaca6..b17c83ee 100644 --- a/services/auction_service/utils/proto_utils_test.cc +++ b/services/auction_service/utils/proto_utils_test.cc @@ -142,8 +142,7 @@ constexpr absl::string_view kTestAdMetadataJson = R"JSON( constexpr char kTestScoringSignals[] = R"JSON({"RandomScoringSignals":{}})JSON"; constexpr char kTestAuctionConfig[] = R"JSON({"RandomAuctionConfig":{}})JSON"; constexpr char kTestBidMetadata[] = R"JSON({"RandomBidMetadata":{}})JSON"; -server_common::log::ContextImpl log_context{ - {}, server_common::ConsentedDebugConfiguration()}; +RequestLogContext log_context{{}, server_common::ConsentedDebugConfiguration()}; google::protobuf::Value GetTestAdMetadata() { auto ads_metadata = JsonStringToValue(kTestAdMetadataJson); diff --git a/services/bidding_service/BUILD b/services/bidding_service/BUILD index e37212ac..d497926e 100644 --- a/services/bidding_service/BUILD +++ b/services/bidding_service/BUILD @@ -13,6 +13,11 @@ # limitations under the License. load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library", "cc_test") +load( + "@rules_pkg//pkg:mappings.bzl", + "pkg_attributes", + "pkg_files", +) load("@rules_proto//proto:defs.bzl", "proto_library") load("//:config.bzl", "ENABLE_CORE_DUMPS_DEFINES") @@ -40,15 +45,16 @@ cc_library( deps = [ ":bidding_code_fetch_config_cc_proto", "//services/bidding_service/code_wrapper:buyer_code_wrapper", + "//services/common/blob_fetch:fetch_mode_cc_proto", "//services/common/clients/code_dispatcher:v8_dispatcher", "//services/common/code_fetch:periodic_bucket_fetcher", "//services/common/code_fetch:periodic_code_fetcher", + "//services/common/loggers:request_log_context", "//services/common/util:file_util", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", "@google_privacysandbox_servers_common//src/concurrent:executor", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", "@google_privacysandbox_servers_common//src/public/core/interface:errors", "@google_privacysandbox_servers_common//src/public/core/interface:execution_result", "@google_privacysandbox_servers_common//src/public/cpio/interface:cpio", @@ -103,6 +109,7 @@ cc_library( "//services/common/clients/code_dispatcher:code_dispatch_client", "//services/common/clients/kv_server:kv_async_client", "//services/common/code_dispatch:code_dispatch_reactor", + "//services/common/loggers:request_log_context", "//services/common/metric:server_definition", "//services/common/util:json_util", "//services/common/util:request_metadata", @@ -110,7 +117,6 @@ cc_library( "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", "@google_privacysandbox_servers_common//src/util/status_macro:status_util", "@rapidjson", @@ -171,6 +177,9 @@ cc_test( srcs = [ "bidding_service_integration_test.cc", ], + tags = [ + "flaky", + ], deps = [ ":bidding_service", ":generate_bids_reactor", @@ -183,6 +192,7 @@ cc_test( "//services/common/encryption:mock_crypto_client_wrapper", "//services/common/test:mocks", "//services/common/test:random", + "@com_google_absl//absl/debugging:failure_signal_handler", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", @@ -220,6 +230,7 @@ proto_library( name = "bidding_code_fetch_config_proto", srcs = ["bidding_code_fetch_config.proto"], deps = [ + "//services/common/blob_fetch:fetch_mode_proto", "@com_google_googleapis//google/api:annotations_proto", "@com_google_protobuf//:struct_proto", ], @@ -340,3 +351,11 @@ cc_library( "@service_value_key_fledge_privacysandbox//public/query/v2:get_values_v2_cc_proto", ], ) + +pkg_files( + name = "packaged_cddl_specs", + srcs = ["//services/bidding_service/egress_cddl_spec:1.0.0"], + attributes = pkg_attributes(mode = "0555"), + prefix = "/egress_cddl_spec/", + visibility = ["//visibility:public"], +) diff --git a/services/bidding_service/base_generate_bids_reactor.h b/services/bidding_service/base_generate_bids_reactor.h index e9ab11cc..de3ceafa 100644 --- a/services/bidding_service/base_generate_bids_reactor.h +++ b/services/bidding_service/base_generate_bids_reactor.h @@ -24,8 +24,8 @@ #include "services/bidding_service/data/runtime_config.h" #include "services/common/clients/code_dispatcher/request_context.h" #include "services/common/code_dispatch/code_dispatch_reactor.h" +#include "services/common/loggers/request_log_context.h" #include "services/common/util/request_response_constants.h" -#include "src/logger/request_context_impl.h" namespace privacy_sandbox::bidding_auction_servers { @@ -58,12 +58,13 @@ class BaseGenerateBidsReactor roma_timeout_ms_(runtime_config.roma_timeout_ms), roma_request_context_factory_( GetLoggingContext(this->raw_request_), - this->raw_request_.consented_debug_config()), - enable_adtech_code_logging_(runtime_config.enable_adtech_code_logging), + this->raw_request_.consented_debug_config(), + [this]() { return this->raw_response_.mutable_debug_info(); }), log_context_( GetLoggingContext(this->raw_request_), this->raw_request_.consented_debug_config(), [this]() { return this->raw_response_.mutable_debug_info(); }), + enable_adtech_code_logging_(log_context_.is_consented()), max_allowed_size_debug_url_chars_( runtime_config.max_allowed_size_debug_url_bytes), max_allowed_size_all_debug_urls_chars_( @@ -90,8 +91,8 @@ class BaseGenerateBidsReactor bool enable_buyer_debug_url_generation_; std::string roma_timeout_ms_; RomaRequestContextFactory roma_request_context_factory_; + RequestLogContext log_context_; bool enable_adtech_code_logging_; - server_common::log::ContextImpl log_context_; int max_allowed_size_debug_url_chars_; long max_allowed_size_all_debug_urls_chars_; }; diff --git a/services/bidding_service/benchmarking/generate_bids_reactor_benchmarks.cc b/services/bidding_service/benchmarking/generate_bids_reactor_benchmarks.cc index 46c3bf98..5885cb7b 100644 --- a/services/bidding_service/benchmarking/generate_bids_reactor_benchmarks.cc +++ b/services/bidding_service/benchmarking/generate_bids_reactor_benchmarks.cc @@ -159,11 +159,9 @@ absl::Status CodeDispatchClientStub::BatchExecute( DispatchResponse dispatch_response; dispatch_response.resp = absl::StrFormat( R"JSON({ - "response": { - "render": "fake_url", - "bid": %f - } - })JSON", + "render": "fake_url", + "bid": %f + })JSON", absl::Uniform(bit_gen, 0, 100.0)); dispatch_response.id = request.id; responses.emplace_back(dispatch_response); @@ -243,8 +241,7 @@ static void BM_ProtectedAudience(benchmark::State& state) { GenerateBidsResponse response; CodeDispatchClientStub dispatcher; BiddingServiceRuntimeConfig runtime_config = { - .enable_buyer_debug_url_generation = true, - .enable_adtech_code_logging = false}; + .enable_buyer_debug_url_generation = true}; for (auto _ : state) { // This code gets timed. metric::MetricContextMap( diff --git a/services/bidding_service/bidding_code_fetch_config.proto b/services/bidding_service/bidding_code_fetch_config.proto index fdbfe340..0c311e0f 100644 --- a/services/bidding_service/bidding_code_fetch_config.proto +++ b/services/bidding_service/bidding_code_fetch_config.proto @@ -18,16 +18,7 @@ syntax = "proto3"; package privacy_sandbox.bidding_auction_servers.bidding_service; -// Specify how to fetch the code blob. -enum FetchMode { - // Fetch a single blob from an arbitrary url. - FETCH_MODE_URL = 0; - // Fetch all blobs from a specified bucket - // and rely on a default blob specification. - FETCH_MODE_BUCKET = 1; - // Fetch a blob from a local path. Dev-only. - FETCH_MODE_LOCAL = 2; -} +import "services/common/blob_fetch/fetch_mode.proto"; message BuyerCodeFetchConfig { @@ -96,6 +87,6 @@ message BuyerCodeFetchConfig { // The default will be used if the bid request does not specify a version. string ads_retrieval_bucket_default_blob = 19; - FetchMode fetch_mode = 20; + blob_fetch.FetchMode fetch_mode = 20; } diff --git a/services/bidding_service/bidding_main.cc b/services/bidding_service/bidding_main.cc index fd974e75..f941fc06 100644 --- a/services/bidding_service/bidding_main.cc +++ b/services/bidding_service/bidding_main.cc @@ -254,6 +254,17 @@ absl::Status RunServer() { config.RegisterFunctionBinding(std::move(run_inference_function_object)); PS_LOG(INFO) << "RunInference registered."; + PS_LOG(INFO) << "Register getModelPaths API."; + auto get_model_paths_function_object = + std::make_unique>(); + get_model_paths_function_object->function_name = + std::string(inference::kGetModelPathsFunctionName); + get_model_paths_function_object->function = inference::GetModelPaths; + config.RegisterFunctionBinding( + std::move(get_model_paths_function_object)); + PS_LOG(INFO) << "getModelPaths registered."; + PS_LOG(INFO) << "Start the inference sidecar."; // This usage of the following two flags is not consistent with rest of // the codebase, where we use the parameter from the config client @@ -290,7 +301,6 @@ absl::Status RunServer() { bool enable_buyer_debug_url_generation = udf_config.enable_buyer_debug_url_generation(); - bool enable_adtech_code_logging = udf_config.enable_adtech_code_logging(); const bool enable_protected_audience = config_client.HasParameter(ENABLE_PROTECTED_AUDIENCE) && config_client.GetBooleanParameter(ENABLE_PROTECTED_AUDIENCE); @@ -396,7 +406,6 @@ absl::Status RunServer() { .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation, .roma_timeout_ms = config_client.GetStringParameter(ROMA_TIMEOUT_MS).data(), - .enable_adtech_code_logging = enable_adtech_code_logging, .is_protected_app_signals_enabled = enable_protected_app_signals, .is_protected_audience_enabled = enable_protected_audience, .ad_retrieval_timeout_ms = @@ -410,7 +419,7 @@ absl::Status RunServer() { .kv_server_egress_tls = config_client.GetBooleanParameter(KV_SERVER_EGRESS_TLS)}; - if (udf_config.fetch_mode() == bidding_service::FETCH_MODE_BUCKET) { + if (udf_config.fetch_mode() == blob_fetch::FETCH_MODE_BUCKET) { if (enable_protected_audience) { runtime_config.default_protected_auction_generate_bid_version = udf_config.protected_auction_bidding_js_bucket_default_blob(); diff --git a/services/bidding_service/bidding_service_integration_test.cc b/services/bidding_service/bidding_service_integration_test.cc index bc9eb2fd..3a1f3c09 100644 --- a/services/bidding_service/bidding_service_integration_test.cc +++ b/services/bidding_service/bidding_service_integration_test.cc @@ -14,6 +14,8 @@ #include +#include "absl/debugging/failure_signal_handler.h" +#include "absl/debugging/symbolize.h" #include "absl/strings/escaping.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" @@ -47,6 +49,7 @@ constexpr char kKeyId[] = "key_id"; constexpr char kSecret[] = "secret"; constexpr char kAdRenderUrlPrefixForTest[] = "https://advertising.net/ad?"; constexpr char kTopLevelSeller[] = "top_level_seller"; +constexpr char kTestConsentToken[] = "testConsentToken"; // While Roma demands JSON input and enforces it strictly, we follow the // javascript style guide for returning objects here, so object keys are @@ -479,7 +482,8 @@ absl::StatusOr BuildGenerateBidsRequestFromBrowser( absl::flat_hash_map>* interest_group_to_ad, - int desired_bid_count = 5, bool set_enable_debug_reporting = false) { + int desired_bid_count = 5, bool set_enable_debug_reporting = false, + bool enable_adtech_code_logging = false) { GenerateBidsRequest::GenerateBidsRawRequest raw_request; raw_request.set_enable_debug_reporting(set_enable_debug_reporting); for (int i = 0; i < desired_bid_count; i++) { @@ -494,17 +498,26 @@ BuildGenerateBidsRequestFromBrowser( PS_ASSIGN_OR_RETURN(std::string bidding_signals, MakeRandomTrustedBiddingSignals(raw_request)); raw_request.set_bidding_signals(std::move(bidding_signals)); + if (enable_adtech_code_logging) { + raw_request.mutable_consented_debug_config()->set_token(kTestConsentToken); + raw_request.mutable_consented_debug_config()->set_is_consented(true); + } return raw_request; } class GenerateBidsReactorIntegrationTest : public ::testing::Test { protected: void SetUp() override { + absl::InitializeSymbolizer("GenerateBidsReactorIntegrationTest"); + absl::FailureSignalHandlerOptions options; + absl::InstallFailureSignalHandler(options); + // initialize server_common::telemetry::TelemetryConfig config_proto; config_proto.set_mode(server_common::telemetry::TelemetryConfig::PROD); metric::MetricContextMap( server_common::telemetry::BuildDependentConfig(config_proto)); + server_common::log::ServerToken(kTestConsentToken); TrustedServersConfigClient config_client({}); config_client.SetFlagForTest(kTrue, TEST_MODE); @@ -1113,7 +1126,8 @@ void GenerateBidCodeWrapperTestHelper( absl::flat_hash_map> interest_group_to_ad; auto req = BuildGenerateBidsRequestFromBrowser( - &interest_group_to_ad, desired_bid_count, enable_debug_reporting); + &interest_group_to_ad, desired_bid_count, enable_debug_reporting, + enable_adtech_code_logging); ASSERT_TRUE(req.ok()) << req.status(); if (component_auction) { req.value().set_top_level_seller(kTopLevelSeller); @@ -1159,8 +1173,7 @@ void GenerateBidCodeWrapperTestHelper( config_client, /* public_key_fetcher= */ nullptr); BiddingServiceRuntimeConfig runtime_config{ - .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation, - .enable_adtech_code_logging = enable_adtech_code_logging}; + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation}; BiddingService service( std::move(generate_bids_reactor_factory), std::move(key_fetcher_manager), std::move(crypto_client), std::move(runtime_config), @@ -1332,12 +1345,11 @@ TEST_F(GenerateBidsReactorIntegrationTest, GenerateBidsResponse response; bool enable_debug_reporting = false; bool enable_buyer_debug_url_generation = false; - bool enable_adtech_code_logging = true; GenerateBidCodeWrapperTestHelper( &response, absl::StrFormat(js_code_with_logs_template, kAdRenderUrlPrefixForTest), enable_debug_reporting, enable_buyer_debug_url_generation, - enable_adtech_code_logging); + /*enable_adtech_code_logging=*/true); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); EXPECT_GT(raw_response.bids_size(), 0); @@ -1348,14 +1360,13 @@ TEST_F(GenerateBidsReactorIntegrationTest, GenerateBidsResponse response; bool enable_debug_reporting = false; bool enable_buyer_debug_url_generation = false; - bool enable_adtech_code_logging = true; std::string raw_wasm_bytes; ASSERT_TRUE(absl::Base64Unescape(base64_wasm_plus_one, &raw_wasm_bytes)); - GenerateBidCodeWrapperTestHelper(&response, js_code_runs_wasm_helper, - enable_debug_reporting, - enable_buyer_debug_url_generation, - enable_adtech_code_logging, raw_wasm_bytes); + GenerateBidCodeWrapperTestHelper( + &response, js_code_runs_wasm_helper, enable_debug_reporting, + enable_buyer_debug_url_generation, + /*enable_adtech_code_logging=*/true, raw_wasm_bytes); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); EXPECT_GT(raw_response.bids_size(), 0); diff --git a/services/bidding_service/bidding_service_test.cc b/services/bidding_service/bidding_service_test.cc index 003d832c..f53823e5 100644 --- a/services/bidding_service/bidding_service_test.cc +++ b/services/bidding_service/bidding_service_test.cc @@ -276,9 +276,10 @@ TEST_F(BiddingProtectedAppSignalsTest, BadAdRetrievalResponseFinishesRpc) { .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { return absl::InternalError(kInternalServerError); }); auto bidding_service = CreateBiddingService(); diff --git a/services/bidding_service/buyer_code_fetch_manager.cc b/services/bidding_service/buyer_code_fetch_manager.cc index a3a8c619..5eb9c47e 100644 --- a/services/bidding_service/buyer_code_fetch_manager.cc +++ b/services/bidding_service/buyer_code_fetch_manager.cc @@ -54,11 +54,11 @@ BuyerCodeFetchManager::~BuyerCodeFetchManager() { absl::Status BuyerCodeFetchManager::Init() { switch (udf_config_.fetch_mode()) { - case bidding_service::FETCH_MODE_LOCAL: + case blob_fetch::FETCH_MODE_LOCAL: return InitializeLocalCodeFetch(); - case bidding_service::FETCH_MODE_BUCKET: + case blob_fetch::FETCH_MODE_BUCKET: return InitializeBucketCodeFetch(); - case bidding_service::FETCH_MODE_URL: + case blob_fetch::FETCH_MODE_URL: return InitializeUrlCodeFetch(); default: return absl::InvalidArgumentError(kFetchModeInvalid); diff --git a/services/bidding_service/buyer_code_fetch_manager_test.cc b/services/bidding_service/buyer_code_fetch_manager_test.cc index ed24f43b..3182457f 100644 --- a/services/bidding_service/buyer_code_fetch_manager_test.cc +++ b/services/bidding_service/buyer_code_fetch_manager_test.cc @@ -37,7 +37,6 @@ namespace privacy_sandbox::bidding_auction_servers { namespace { using bidding_service::BuyerCodeFetchConfig; -using bidding_service::FetchMode; using ::google::cmrt::sdk::blob_storage_service::v1::BlobMetadata; using ::google::cmrt::sdk::blob_storage_service::v1::GetBlobRequest; @@ -71,7 +70,7 @@ TEST_F(BuyerCodeFetchManagerTest, FetchModeLocalTriesFileLoad) { EXPECT_CALL(*blob_storage_client_, Run).Times(0); BuyerCodeFetchConfig udf_config; - udf_config.set_fetch_mode(FetchMode::FETCH_MODE_LOCAL); + udf_config.set_fetch_mode(blob_fetch::FETCH_MODE_LOCAL); const std::string bad_path = "error"; udf_config.set_bidding_js_path(bad_path); BuyerCodeFetchManager udf_fetcher(executor_.get(), http_fetcher_.get(), @@ -88,7 +87,7 @@ TEST_F(BuyerCodeFetchManagerTest, FetchModeBucketTriesBucketLoad) { EXPECT_CALL(*blob_storage_client_, Init).WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*blob_storage_client_, Run).WillOnce(Return(absl::OkStatus())); BuyerCodeFetchConfig udf_config; - udf_config.set_fetch_mode(FetchMode::FETCH_MODE_BUCKET); + udf_config.set_fetch_mode(blob_fetch::FETCH_MODE_BUCKET); BuyerCodeFetchManager udf_fetcher(executor_.get(), http_fetcher_.get(), dispatcher_.get(), std::move(blob_storage_client_), udf_config, @@ -102,7 +101,7 @@ TEST_F(BuyerCodeFetchManagerTest, FetchModeBucketTriesBucketLoad) { TEST_F(BuyerCodeFetchManagerTest, TriesBucketFetchForProtectedAuction) { BuyerCodeFetchConfig udf_config; - udf_config.set_fetch_mode(FetchMode::FETCH_MODE_BUCKET); + udf_config.set_fetch_mode(blob_fetch::FETCH_MODE_BUCKET); udf_config.set_url_fetch_period_ms(1); udf_config.set_protected_auction_bidding_js_bucket("pa"); udf_config.set_protected_auction_bidding_js_bucket_default_blob("pa_test"); @@ -132,7 +131,7 @@ TEST_F(BuyerCodeFetchManagerTest, TriesBucketFetchForProtectedAuction) { TEST_F(BuyerCodeFetchManagerTest, TriesBucketFetchForProtectedAppSignals) { BuyerCodeFetchConfig udf_config; - udf_config.set_fetch_mode(FetchMode::FETCH_MODE_BUCKET); + udf_config.set_fetch_mode(blob_fetch::FETCH_MODE_BUCKET); udf_config.set_url_fetch_period_ms(1); udf_config.set_protected_app_signals_bidding_js_bucket("pas"); udf_config.set_protected_app_signals_bidding_js_bucket_default_blob( @@ -170,7 +169,7 @@ TEST_F(BuyerCodeFetchManagerTest, const std::string ads_object = "ads_test"; BuyerCodeFetchConfig udf_config; - udf_config.set_fetch_mode(FetchMode::FETCH_MODE_BUCKET); + udf_config.set_fetch_mode(blob_fetch::FETCH_MODE_BUCKET); udf_config.set_url_fetch_period_ms(1); udf_config.set_protected_app_signals_bidding_js_bucket(pas_bucket); udf_config.set_protected_app_signals_bidding_js_bucket_default_blob( @@ -256,7 +255,7 @@ TEST_F(BuyerCodeFetchManagerTest, FetchModeUrlTriesUrlLoad) { const std::string ads_retrieval_wasm_url = "f"; BuyerCodeFetchConfig udf_config; - udf_config.set_fetch_mode(FetchMode::FETCH_MODE_URL); + udf_config.set_fetch_mode(blob_fetch::FETCH_MODE_URL); udf_config.set_bidding_js_url(pa_url); udf_config.set_bidding_wasm_helper_url(pa_wasm_url); diff --git a/services/bidding_service/code_wrapper/BUILD b/services/bidding_service/code_wrapper/BUILD index 120973cd..5912d4d4 100644 --- a/services/bidding_service/code_wrapper/BUILD +++ b/services/bidding_service/code_wrapper/BUILD @@ -28,10 +28,10 @@ cc_library( "//services/bidding_service:runtime_flags", "//services/common/clients/config:config_client", "//services/common/constants:common_service_flags", + "//services/common/loggers:request_log_context", "//services/common/util:reporting_util", "//services/common/util:request_response_constants", "@com_google_absl//absl/strings", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", ], ) diff --git a/services/bidding_service/code_wrapper/buyer_code_wrapper.cc b/services/bidding_service/code_wrapper/buyer_code_wrapper.cc index 5a32887e..a324c0a1 100644 --- a/services/bidding_service/code_wrapper/buyer_code_wrapper.cc +++ b/services/bidding_service/code_wrapper/buyer_code_wrapper.cc @@ -21,8 +21,8 @@ #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "services/bidding_service/constants.h" +#include "services/common/loggers/request_log_context.h" #include "services/common/util/reporting_util.h" -#include "src/logger/request_context_impl.h" namespace privacy_sandbox::bidding_auction_servers { namespace { diff --git a/services/bidding_service/code_wrapper/buyer_code_wrapper.h b/services/bidding_service/code_wrapper/buyer_code_wrapper.h index 83e026ad..0202688d 100644 --- a/services/bidding_service/code_wrapper/buyer_code_wrapper.h +++ b/services/bidding_service/code_wrapper/buyer_code_wrapper.h @@ -49,7 +49,7 @@ std::string GetProtectedAppSignalsGenericBuyerWrappedCode( //- Exporting logs to Bidding Service using console.log //- Hooks in wasm module inline constexpr absl::string_view kEntryFunction = R"JS_CODE( - function generateBidEntryFunction($0, featureFlags){ + async function generateBidEntryFunction($0, featureFlags){ var ps_logs = []; var ps_errors = []; var ps_warns = []; @@ -78,7 +78,7 @@ inline constexpr absl::string_view kEntryFunction = R"JS_CODE( var generateBidResponse = {}; try { - generateBidResponse = generateBid($0); + generateBidResponse = await generateBid($0); if( featureFlags.enable_debug_url_generation && (forDebuggingOnly_auction_loss_url || forDebuggingOnly_auction_win_url)) { @@ -92,12 +92,16 @@ inline constexpr absl::string_view kEntryFunction = R"JS_CODE( console.error("[Error: " + error + "; Message: " + message + "]"); } } - return { - response: generateBidResponse !== undefined ? generateBidResponse : {}, - logs: ps_logs, - errors: ps_errors, - warnings: ps_warns + var result = generateBidResponse !== undefined ? generateBidResponse : {}; + if (featureFlags.enable_logging) { + return { + response: result, + logs: ps_logs, + errors: ps_errors, + warnings: ps_warns + }; } + return result; } )JS_CODE"; @@ -107,7 +111,7 @@ inline constexpr absl::string_view kEntryFunction = R"JS_CODE( //- Hooks in wasm module inline constexpr absl::string_view kPrepareDataForAdRetrievalEntryFunction = R"JS_CODE( - function $0EntryFunction(onDeviceEncodedSignalsHexString, $1, featureFlags){ + async function $0EntryFunction(onDeviceEncodedSignalsHexString, $1, featureFlags){ var ps_logs = []; var ps_errors = []; var ps_warns = []; @@ -127,7 +131,7 @@ inline constexpr absl::string_view kPrepareDataForAdRetrievalEntryFunction = Uint8Array.from(encodedOnDeviceSignalsIn.match(/.{1,2}/g).map((byte) => parseInt(byte, 16))); return { - response: $0(convertToUint8Array(onDeviceEncodedSignalsHexString), $1), + response: await $0(convertToUint8Array(onDeviceEncodedSignalsHexString), $1), logs: ps_logs, errors: ps_errors, warnings: ps_warns diff --git a/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h b/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h index 38379f52..438314e0 100644 --- a/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h +++ b/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h @@ -44,7 +44,7 @@ constexpr absl::string_view kExpectedGenerateBidCode_template = R"JS_CODE( const globalWasmHex = []; const globalWasmHelper = globalWasmHex.length ? new WebAssembly.Module(Uint8Array.from(globalWasmHex)) : null; - function generateBidEntryFunction(interest_group, auction_signals, buyer_signals, trusted_bidding_signals, device_signals, featureFlags){ + async function generateBidEntryFunction(interest_group, auction_signals, buyer_signals, trusted_bidding_signals, device_signals, featureFlags){ var ps_logs = []; var ps_errors = []; var ps_warns = []; @@ -73,7 +73,7 @@ constexpr absl::string_view kExpectedGenerateBidCode_template = R"JS_CODE( var generateBidResponse = {}; try { - generateBidResponse = generateBid(interest_group, auction_signals, buyer_signals, trusted_bidding_signals, device_signals); + generateBidResponse = await generateBid(interest_group, auction_signals, buyer_signals, trusted_bidding_signals, device_signals); if( featureFlags.enable_debug_url_generation && (forDebuggingOnly_auction_loss_url || forDebuggingOnly_auction_win_url)) { @@ -87,12 +87,16 @@ constexpr absl::string_view kExpectedGenerateBidCode_template = R"JS_CODE( console.error("[Error: " + error + "; Message: " + message + "]"); } } - return { - response: generateBidResponse !== undefined ? generateBidResponse : {}, - logs: ps_logs, - errors: ps_errors, - warnings: ps_warns + var result = generateBidResponse !== undefined ? generateBidResponse : {}; + if (featureFlags.enable_logging) { + return { + response: result, + logs: ps_logs, + errors: ps_errors, + warnings: ps_warns + }; } + return result; } function fibonacci(num) { @@ -117,7 +121,7 @@ constexpr absl::string_view const globalWasmHex = []; const globalWasmHelper = globalWasmHex.length ? new WebAssembly.Module(Uint8Array.from(globalWasmHex)) : null; - function generateBidEntryFunction(ads, sellerAuctionSignals, buyerSignals, preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion, featureFlags){ + async function generateBidEntryFunction(ads, sellerAuctionSignals, buyerSignals, preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion, featureFlags){ var ps_logs = []; var ps_errors = []; var ps_warns = []; @@ -154,7 +158,7 @@ constexpr absl::string_view var generateBidResponse = {}; try { - generateBidResponse = generateBid(ads, sellerAuctionSignals, buyerSignals, preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion); + generateBidResponse = await generateBid(ads, sellerAuctionSignals, buyerSignals, preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion); if( featureFlags.enable_debug_url_generation && (forDebuggingOnly_auction_loss_url || forDebuggingOnly_auction_win_url)) { @@ -168,12 +172,16 @@ constexpr absl::string_view console.error("[Error: " + error + "; Message: " + message + "]"); } } - return { - response: generateBidResponse !== undefined ? generateBidResponse : {}, - logs: ps_logs, - errors: ps_errors, - warnings: ps_warns + var result = generateBidResponse !== undefined ? generateBidResponse : {}; + if (featureFlags.enable_logging) { + return { + response: result, + logs: ps_logs, + errors: ps_errors, + warnings: ps_warns + }; } + return result; } function fibonacci(num) { @@ -198,7 +206,7 @@ constexpr absl::string_view kExpectedPrepareDataForAdRetrievalTemplate = const globalWasmHex = []; const globalWasmHelper = globalWasmHex.length ? new WebAssembly.Module(Uint8Array.from(globalWasmHex)) : null; - function prepareDataForAdRetrievalEntryFunction(onDeviceEncodedSignalsHexString, testArg, featureFlags){ + async function prepareDataForAdRetrievalEntryFunction(onDeviceEncodedSignalsHexString, testArg, featureFlags){ var ps_logs = []; var ps_errors = []; var ps_warns = []; @@ -218,7 +226,7 @@ constexpr absl::string_view kExpectedPrepareDataForAdRetrievalTemplate = Uint8Array.from(encodedOnDeviceSignalsIn.match(/.{1,2}/g).map((byte) => parseInt(byte, 16))); return { - response: prepareDataForAdRetrieval(convertToUint8Array(onDeviceEncodedSignalsHexString), testArg), + response: await prepareDataForAdRetrieval(convertToUint8Array(onDeviceEncodedSignalsHexString), testArg), logs: ps_logs, errors: ps_errors, warnings: ps_warns diff --git a/services/bidding_service/data/runtime_config.h b/services/bidding_service/data/runtime_config.h index da4d838d..c96b0ae6 100644 --- a/services/bidding_service/data/runtime_config.h +++ b/services/bidding_service/data/runtime_config.h @@ -39,8 +39,6 @@ struct BiddingServiceRuntimeConfig { // - Exporting console.logs from Roma // - Event level debug win and loss reporting bool enable_buyer_code_wrapper = false; - // Enables exporting console.logs from Roma to Bidding Service - bool enable_adtech_code_logging = false; // Indicates whether or not protected app signals support is enabled. bool is_protected_app_signals_enabled = false; // Indicates whether or not Protected Audience support is enabled. diff --git a/services/bidding_service/egress_cddl_spec/1.0.0 b/services/bidding_service/egress_cddl_spec/1.0.0 new file mode 100644 index 00000000..969de1f8 --- /dev/null +++ b/services/bidding_service/egress_cddl_spec/1.0.0 @@ -0,0 +1,99 @@ +; An ad-tech controlled `schema` of `feature`s to egress. Ad-techs provide +; their `schema` as a JSON file alongside their `generateBid()` implementation, +; and it is used during validation to ensure that `payload`s are formatted as +; expected. +schema = { + features: [* feature_types], + ; Version of the CDDL used when constructing this `schema` instance. + cddl_version: cddl_version, + ; Version of this schema instance itself. Will be prepended to the wire format + ; of the `payload` after validation. 3 bits, so 8 maximum concurrent `schema`s + ; are supported. + version: uint .le 7 +} + +; A platform-defined enumeration of all `feature` values we support. During +; validation, we will enforce that all `feature`s in a `payload` are in this +; enumeration. We will also analyze `feature` instances to determine the number +; of bits of information they contain so that we can be sure the sum of the +; information does not overflow the system-controlled maximum. +; +; Each concrete `feature` will have a `type` field; see `feature_types`. +feature /= boolean-feature / signed-integer-feature / + unsigned-integer-feature / bucket-feature / + histogram-feature ; Additional `feature` types here. + +; All supported `feature_types`. `feature`s hold values, and each one has a +; matching `_type`, used in `schema` to define payload structure. +; +; All `_type`s must have a `name`, and may optionally have other fields. +feature_types /= boolean-feature-type / signed-integer-feature-type / + unsigned-integer-feature-type / bucket-feature-type / + histogram-feature-type + +boolean-feature-type = { + name: text .regexp "^boolean-feature$" +} + +boolean-feature = { + type: boolean-feature-type, + value: bool, +} + +unsigned-integer-feature-type = { + name: text .regexp "^unsigned-integer-feature$" + size: uint, +} + +unsigned-integer-feature = { + type: unsigned-integer-feature-type, + value: uint, +} + +signed-integer-feature-type = { + name: text .regexp "^signed-integer-feature$" + size: uint, +} + +signed-integer-feature = { + type: signed-integer-feature-type, + value: int, +} + +bucket-feature-type = { + name: text .regexp "^bucket-feature$" + buckets: uint, ; number of buckets. +} + +bucket-feature = { + type: bucket-feature-type, + value: [* bool], +} + +histogram-feature-type = { + name: text .regexp "^histogram-feature$" + size: uint, + value: [1* histogram-feature-subtype], +} + +histogram-feature-subtype /= unsigned-integer-feature-type / signed-integer-feature-type + +histogram-feature = { + type: histogram-feature-type, + value: [1* unsigned-integer-feature / signed-integer-feature ], +} + +; Version of this CDDL used when validating `schema`s and `payload`s. +cddl_version /= "1.0.0" + +; An ad-tech controlled ordered list of `feature` values to egress. +; +; We will cap the amount of information contained in a `payload` instance to a +; system-controlled value (for example, 20 bits). This is not expressed in the +; schema because enforcement is applied to the amount of information in the +; resulting payload on the wire, modulo bookkeeping information. +payload = { + features: [* feature], ; Any number of features, subject to size limits. + ; No version here -- we will attach the `schema` version used during + ; validation so the ad-tech knows which schema to use during deserialization. +} diff --git a/services/bidding_service/egress_cddl_spec/BUILD b/services/bidding_service/egress_cddl_spec/BUILD new file mode 100644 index 00000000..8c05b473 --- /dev/null +++ b/services/bidding_service/egress_cddl_spec/BUILD @@ -0,0 +1,15 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exports_files(["1.0.0"]) diff --git a/services/bidding_service/generate_bids_reactor.cc b/services/bidding_service/generate_bids_reactor.cc index fb0813f0..f53b0b45 100644 --- a/services/bidding_service/generate_bids_reactor.cc +++ b/services/bidding_service/generate_bids_reactor.cc @@ -220,7 +220,7 @@ absl::StatusOr SerializeIG(const IGForBidding& ig) { // in a loop for all Interest Groups. absl::StatusOr SerializeTrustedBiddingSignalsPerIG( const GenerateBidsRequest::GenerateBidsRawRequest& raw_request, - server_common::log::ContextImpl& log_context) { + RequestLogContext& log_context) { // Parse into JSON. auto start_parse_time = absl::Now(); PS_ASSIGN_OR_RETURN((rapidjson::Document parsed_signals), @@ -285,8 +285,8 @@ absl::StatusOr BuildGenerateBidRequest( const std::vector>& base_input, const TrustedBiddingSignalsByIg& ig_trusted_signals_map, const bool enable_buyer_debug_url_generation, - server_common::log::ContextImpl& log_context, - const bool enable_adtech_code_logging, const std::string& version) { + RequestLogContext& log_context, const bool enable_adtech_code_logging, + const std::string& version) { // Construct the wrapper struct for our V8 Dispatch Request. DispatchRequest generate_bid_request; generate_bid_request.id = interest_group.name(); @@ -446,8 +446,10 @@ void GenerateBidsReactor::Execute() { PS_VLOG(kEncrypted, log_context_) << "Encrypted GenerateBidsRequest:\n" << request_->ShortDebugString(); + log_context_.SetEventMessageField(*request_); PS_VLOG(kPlain, log_context_) << "GenerateBidsRawRequest:\n" << raw_request_.ShortDebugString(); + log_context_.SetEventMessageField(raw_request_); auto interest_groups = raw_request_.interest_group_for_bidding(); @@ -482,7 +484,18 @@ void GenerateBidsReactor::Execute() { } else { auto dispatch_request = generate_bid_request.value(); dispatch_request.tags[kTimeoutMs] = roma_timeout_ms_; - dispatch_request.metadata = roma_request_context_factory_.Create(); + RomaRequestSharedContext shared_context = + roma_request_context_factory_.Create(); + auto roma_shared_context = shared_context.GetRomaRequestContext(); + if (roma_shared_context.ok()) { + std::shared_ptr roma_request_context = + roma_shared_context.value(); + roma_request_context->SetIsProtectedAudienceRequest(true); + } else { + PS_LOG(ERROR, log_context_) << "Failed to retrieve RomaRequestContext: " + << roma_shared_context.status(); + } + dispatch_request.metadata = shared_context; dispatch_requests_.push_back(dispatch_request); } } @@ -548,8 +561,10 @@ void GenerateBidsReactor::GenerateBidsCallback( if (result.ok()) { AdWithBid bid; absl::StatusOr generate_bid_response = - ParseAndGetResponseJson(enable_adtech_code_logging_, result->resp, - log_context_); + enable_adtech_code_logging_ + ? ParseAndGetResponseJson(enable_adtech_code_logging_, + result->resp, log_context_) + : result->resp; if (!generate_bid_response.ok()) { PS_LOG(ERROR, log_context_) << "Failed to parse response from Roma " @@ -620,6 +635,8 @@ void GenerateBidsReactor::GenerateBidsCallback( void GenerateBidsReactor::EncryptResponseAndFinish(grpc::Status status) { PS_VLOG(kPlain, log_context_) << "GenerateBidsRawResponse:\n" << raw_response_.ShortDebugString(); + log_context_.SetEventMessageField(raw_response_); + log_context_.ExportEventMessage(); if (!EncryptResponse()) { PS_LOG(ERROR, log_context_) diff --git a/services/bidding_service/generate_bids_reactor_test.cc b/services/bidding_service/generate_bids_reactor_test.cc index 0c21ca66..4b8f5d3c 100644 --- a/services/bidding_service/generate_bids_reactor_test.cc +++ b/services/bidding_service/generate_bids_reactor_test.cc @@ -45,6 +45,7 @@ namespace { constexpr char kTestBiddingSignals[] = R"json({"keys":{"trusted_bidding_signal_key": "some_trusted_bidding_signal_value"}})json"; constexpr char kTopLevelSeller[] = "https://www.example-top-ssp.com"; +constexpr char kTestConsentToken[] = "testConsentToken"; using ::google::protobuf::TextFormat; using ::google::protobuf::util::MessageToJsonString; @@ -70,60 +71,99 @@ absl::Status FakeExecute(std::vector& batch, return absl::OkStatus(); } -std::string GetTestResponse(absl::string_view render, float bid) { +std::string GetTestResponse(absl::string_view render, float bid, + bool enable_adtech_code_logging = false) { + if (enable_adtech_code_logging) { + return absl::Substitute(R"JSON({ + "response": { + "render": "$0", + "bid": $1 + }, + "logs": ["test log"], + "errors": ["test.error"], + "warnings":["test.warn"] + })JSON", + render, bid); + } + return absl::Substitute(R"JSON({ - "response": { - "render": "$0", - "bid": $1 - }, - "logs": ["test log"], - "errors": ["test.error"], - "warnings":["test.warn"] + "render": "$0", + "bid": $1 })JSON", render, bid); } std::string GetTestResponseWithBuyerReportingId( - absl::string_view render, float bid, absl::string_view buyer_reporting_id) { + absl::string_view render, float bid, absl::string_view buyer_reporting_id, + bool enable_adtech_code_logging = false) { + if (enable_adtech_code_logging) { + return absl::Substitute(R"JSON({ + "response": { + "render": "$0", + "bid": $1, + "buyerReportingId": "$2" + }, + "logs": [], + "errors": [], + "warnings":[] + })JSON", + render, bid, buyer_reporting_id); + } + return absl::Substitute(R"JSON({ - "response": { - "render": "$0", - "bid": $1, - "buyerReportingId": "$2" - }, - "logs": [], - "errors": [], - "warnings":[] + "render": "$0", + "bid": $1, + "buyerReportingId": "$2" })JSON", render, bid, buyer_reporting_id); } -std::string GetTestResponseWithUnknownField(absl::string_view render, - float bid) { +std::string GetTestResponseWithUnknownField( + absl::string_view render, float bid, + bool enable_adtech_code_logging = false) { + if (enable_adtech_code_logging) { + return absl::Substitute(R"JSON({ + "response": { + "render": "$0", + "bid": $1, + "buyer_reporting_ids": "abcdef" + }, + "logs": [], + "errors": [], + "warnings":[] + })JSON", + render, bid); + } + return absl::Substitute(R"JSON({ - "response": { - "render": "$0", - "bid": $1, - "buyer_reporting_ids": "abcdef" - }, - "logs": [], - "errors": [], - "warnings":[] + "render": "$0", + "bid": $1, + "buyer_reporting_ids": "abcdef" })JSON", render, bid); } -std::string GetComponentAuctionResponse(absl::string_view render, float bid, - bool allow_component_auction) { +std::string GetComponentAuctionResponse( + absl::string_view render, float bid, bool allow_component_auction, + bool enable_adtech_code_logging = false) { + if (enable_adtech_code_logging) { + return absl::Substitute(R"JSON({ + "response": { + "render": "$0", + "bid": $1, + "allowComponentAuction": $2 + }, + "logs": ["test log"], + "errors": ["test.error"], + "warnings":["test.warn"] + })JSON", + render, bid, allow_component_auction); + } + return absl::Substitute(R"JSON({ - "response": { - "render": "$0", - "bid": $1, - "allowComponentAuction": $2 - }, - "logs": ["test log"], - "errors": ["test.error"], - "warnings":["test.warn"] + "render": "$0", + "bid": $1, + "allowComponentAuction": $2 })JSON", render, bid, allow_component_auction); } @@ -141,6 +181,7 @@ class GenerateBidsReactorTest : public testing::Test { metric::MetricContextMap( server_common::telemetry::BuildDependentConfig(config_proto)) ->Get(&request_); + server_common::log::ServerToken(kTestConsentToken); TrustedServersConfigClient config_client({}); config_client.SetFlagForTest(kTrue, TEST_MODE); @@ -154,14 +195,12 @@ class GenerateBidsReactorTest : public testing::Test { void CheckGenerateBids(const RawRequest& raw_request, const Response& expected_response, - bool enable_buyer_debug_url_generation = false, - bool enable_adtech_code_logging = false) { + bool enable_buyer_debug_url_generation = false) { Response response; std::unique_ptr benchmarkingLogger = std::make_unique(); BiddingServiceRuntimeConfig runtime_config = { - .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation, - .enable_adtech_code_logging = enable_adtech_code_logging}; + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation}; request_.set_request_ciphertext(raw_request.SerializeAsString()); GenerateBidsReactor reactor( dispatcher_, &request_, &response, std::move(benchmarkingLogger), @@ -286,7 +325,8 @@ void BuildRawRequest(const std::vector& interest_groups_to_add, absl::string_view auction_signals, absl::string_view buyer_signals, absl::string_view bidding_signals, RawRequest& raw_request, - bool enable_debug_reporting = false) { + bool enable_debug_reporting = false, + bool enable_adtech_code_logging = false) { for (int i = 0; i < interest_groups_to_add.size(); i++) { *raw_request.mutable_interest_group_for_bidding()->Add() = interest_groups_to_add[i]; @@ -297,6 +337,10 @@ void BuildRawRequest(const std::vector& interest_groups_to_add, raw_request.set_enable_debug_reporting(enable_debug_reporting); raw_request.set_seller(kSeller); raw_request.set_publisher_name(kPublisherName); + if (enable_adtech_code_logging) { + raw_request.mutable_consented_debug_config()->set_token(kTestConsentToken); + raw_request.mutable_consented_debug_config()->set_is_consented(true); + } } void BuildRawRequestForComponentAuction( @@ -341,7 +385,8 @@ TEST_F(GenerateBidsReactorTest, GenerateBidSuccessfulWithCodeWrapper) { bool enable_debug_reporting = false; bool enable_buyer_debug_url_generation = false; bool enable_adtech_code_logging = true; - std::string response_json = GetTestResponse(kTestRenderUrl, 1); + std::string response_json = + GetTestResponse(kTestRenderUrl, 1, enable_adtech_code_logging); AdWithBid bid; bid.set_render(kTestRenderUrl); bid.set_bid(1); @@ -360,9 +405,9 @@ TEST_F(GenerateBidsReactorTest, GenerateBidSuccessfulWithCodeWrapper) { }); RawRequest raw_request; BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request, enable_debug_reporting); - CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation, - enable_adtech_code_logging); + kTestBiddingSignals, raw_request, enable_debug_reporting, + enable_adtech_code_logging); + CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation); } TEST_F(GenerateBidsReactorTest, BuyerReportingIdSetInResponse) { @@ -370,7 +415,7 @@ TEST_F(GenerateBidsReactorTest, BuyerReportingIdSetInResponse) { bool enable_buyer_debug_url_generation = false; bool enable_adtech_code_logging = true; std::string response_json = GetTestResponseWithBuyerReportingId( - kTestRenderUrl, 1, kTestBuyerReportingId); + kTestRenderUrl, 1, kTestBuyerReportingId, enable_adtech_code_logging); AdWithBid bid; bid.set_render(kTestRenderUrl); bid.set_bid(1); @@ -390,17 +435,17 @@ TEST_F(GenerateBidsReactorTest, BuyerReportingIdSetInResponse) { }); RawRequest raw_request; BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request, enable_debug_reporting); - CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation, - enable_adtech_code_logging); + kTestBiddingSignals, raw_request, enable_debug_reporting, + enable_adtech_code_logging); + CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation); } TEST_F(GenerateBidsReactorTest, UnknownFieldInResponseParsedSuccessfully) { bool enable_debug_reporting = false; bool enable_buyer_debug_url_generation = false; bool enable_adtech_code_logging = true; - std::string response_json = - GetTestResponseWithUnknownField(kTestRenderUrl, 1); + std::string response_json = GetTestResponseWithUnknownField( + kTestRenderUrl, 1, enable_adtech_code_logging); AdWithBid bid; bid.set_render(kTestRenderUrl); bid.set_bid(1); @@ -419,9 +464,9 @@ TEST_F(GenerateBidsReactorTest, UnknownFieldInResponseParsedSuccessfully) { }); RawRequest raw_request; BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request, enable_debug_reporting); - CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation, - enable_adtech_code_logging); + kTestBiddingSignals, raw_request, enable_debug_reporting, + enable_adtech_code_logging); + CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation); } TEST_F(GenerateBidsReactorTest, @@ -907,15 +952,12 @@ TEST_F(GenerateBidsReactorTest, GenerateBidResponseWithDebugUrls) { bool enable_buyer_debug_url_generation = true; const std::string response_json = R"JSON( { - "response": { - "render": "https://adTech.com/ad?id=123", - "bid": 1, - "debug_report_urls": { - "auction_debug_loss_url": "test.com/debugLoss", - "auction_debug_win_url": "test.com/debugWin" - } - }, - "logs": [] + "render": "https://adTech.com/ad?id=123", + "bid": 1, + "debug_report_urls": { + "auction_debug_loss_url": "test.com/debugLoss", + "auction_debug_win_url": "test.com/debugWin" + } } )JSON"; diff --git a/services/bidding_service/generate_bids_reactor_test_utils.cc b/services/bidding_service/generate_bids_reactor_test_utils.cc index 4b49a425..4016f86e 100644 --- a/services/bidding_service/generate_bids_reactor_test_utils.cc +++ b/services/bidding_service/generate_bids_reactor_test_utils.cc @@ -161,14 +161,11 @@ std::string CreateGenerateBidsUdfResponse( &base64_encoded_temporary_features_bytes); return absl::Substitute(R"JSON( { - "response": { - "render": "$0", - "bid": $1, - "egressPayload": "$2", - "debugReportUrls": $3, - "temporaryUnlimitedEgressPayload": "$4" - }, - "logs": [] + "render": "$0", + "bid": $1, + "egressPayload": "$2", + "debugReportUrls": $3, + "temporaryUnlimitedEgressPayload": "$4" } )JSON", render, bid, base64_encoded_features_bytes, @@ -249,15 +246,17 @@ void SetupAdRetrievalClientExpectations( std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = ads_retrieval_response.has_value() ? *ads_retrieval_response : CreateAdsRetrievalOrKvLookupResponse(); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); } diff --git a/services/bidding_service/inference/BUILD b/services/bidding_service/inference/BUILD index f746a709..9e1330da 100644 --- a/services/bidding_service/inference/BUILD +++ b/services/bidding_service/inference/BUILD @@ -13,6 +13,7 @@ # limitations under the License. load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load("//:config.bzl", "IS_PROD_BUILD_DEFINES") cc_library( name = "inference_utils", @@ -22,6 +23,7 @@ cc_library( hdrs = [ "inference_utils.h", ], + local_defines = IS_PROD_BUILD_DEFINES, visibility = ["//visibility:public"], deps = [ ":inference_flags", @@ -41,6 +43,7 @@ cc_library( "@inference_common//proto:inference_sidecar_cc_proto", "@inference_common//sandbox:sandbox_executor", "@inference_common//utils:file_util", + "@rapidjson", ], ) diff --git a/services/bidding_service/inference/inference_flags.cc b/services/bidding_service/inference/inference_flags.cc index ab65f6e5..1b69dcf7 100644 --- a/services/bidding_service/inference/inference_flags.cc +++ b/services/bidding_service/inference/inference_flags.cc @@ -38,7 +38,7 @@ ABSL_FLAG(std::optional, inference_model_bucket_paths, // "num_interop_threads": , // "num_intraop_threads": , // "module_name": , -// "cpuset": +// "cpuset": , // ... // } // where each property corresponds to a field of the diff --git a/services/bidding_service/inference/inference_utils.cc b/services/bidding_service/inference/inference_utils.cc index 54341fbe..157a59df 100644 --- a/services/bidding_service/inference/inference_utils.cc +++ b/services/bidding_service/inference/inference_utils.cc @@ -33,6 +33,10 @@ #include "absl/strings/str_split.h" #include "absl/synchronization/mutex.h" #include "proto/inference_sidecar.grpc.pb.h" +#include "rapidjson/document.h" +#include "rapidjson/error/en.h" +#include "rapidjson/pointer.h" +#include "rapidjson/writer.h" #include "services/bidding_service/inference/inference_flags.h" #include "services/common/clients/code_dispatcher/request_context.h" #include "services/common/util/request_response_constants.h" @@ -144,6 +148,20 @@ void RunInference( PredictRequest predict_request; predict_request.set_input(payload); + absl::StatusOr> roma_request_context = + wrapper.metadata.GetRomaRequestContext(); + if (roma_request_context.ok()) { + // Check if it is a Protected Audience request and the build flavor is prod + if ((*roma_request_context)->IsProtectedAudienceRequest() && + PS_IS_PROD_BUILD) { + PS_LOG(ERROR, (*roma_request_context)->GetLogContext()) + << "Inference is not supported for Protected Audience requests in " + "production build."; + return; + } + predict_request.set_is_consented((*roma_request_context)->IsConsented()); + } + grpc::ClientContext context; PredictResponse predict_response; grpc::Status rpc_status = @@ -152,11 +170,64 @@ void RunInference( wrapper.io_proto.set_output_string(predict_response.output()); PS_VLOG(10) << "Inference response received: " << predict_response.DebugString(); + if (roma_request_context.ok()) { + PS_VLOG(kNoisyInfo, (*roma_request_context)->GetLogContext()) + << "Inference sidecar consented debugging log: " + << predict_response.debug_info(); + } return; } absl::Status status = server_common::ToAbslStatus(rpc_status); // TODO(b/321284008): Communicate inference failure with JS caller. - PS_LOG(ERROR) << "Response error: " << status.message(); + if (roma_request_context.ok()) { + PS_LOG(ERROR, (*roma_request_context)->GetLogContext()) + << "Response error: " << status.message(); + } +} + +std::string GetModelResponseToJson(const GetModelPathsResponse& response) { + rapidjson::Document document; + document.SetArray(); + rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); + + for (const ModelSpec& spec : response.model_specs()) { + rapidjson::Value value; + value.SetString(spec.model_path().c_str(), spec.model_path().length(), + allocator); + document.PushBack(value, allocator); + } + + rapidjson::StringBuffer strbuf; + rapidjson::Writer writer(strbuf); + document.Accept(writer); + + return strbuf.GetString(); +} + +void GetModelPaths( + google::scp::roma::FunctionBindingPayload& + wrapper) { + SandboxExecutor& executor = Executor(); + std::unique_ptr stub = + InferenceService::NewStub(InferenceChannel(executor)); + + PS_VLOG(kNoisyInfo) << "GetModelPaths called"; + GetModelPathsRequest get_model_paths_request; + + grpc::ClientContext context; + GetModelPathsResponse get_model_paths_response; + grpc::Status rpc_status = stub->GetModelPaths( + &context, get_model_paths_request, &get_model_paths_response); + if (rpc_status.ok()) { + wrapper.io_proto.set_output_string( + GetModelResponseToJson(get_model_paths_response)); + PS_VLOG(10) << "GetModelPaths response received: " + << get_model_paths_response.DebugString(); + return; + } + + absl::Status status = server_common::ToAbslStatus(rpc_status); + PS_LOG(ERROR) << "GetModelPaths response error: " << status.message(); } } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/bidding_service/inference/inference_utils.h b/services/bidding_service/inference/inference_utils.h index 3d9a7050..e24b893d 100644 --- a/services/bidding_service/inference/inference_utils.h +++ b/services/bidding_service/inference/inference_utils.h @@ -29,6 +29,7 @@ namespace privacy_sandbox::bidding_auction_servers::inference { constexpr absl::string_view kInferenceFunctionName = "runInference"; +constexpr absl::string_view kGetModelPathsFunctionName = "getModelPaths"; // Accesses a sandbox exectuor that uses static storage. SandboxExecutor& Executor(); @@ -48,6 +49,15 @@ void RunInference( google::scp::roma::FunctionBindingPayload& wrapper); +// Registered with Roma to provide an API to query the currently available +// models from JS code. +void GetModelPaths( + google::scp::roma::FunctionBindingPayload& + wrapper); + +// Converts a GetModelPaths response to Json string +std::string GetModelResponseToJson(const GetModelPathsResponse& response); + } // namespace privacy_sandbox::bidding_auction_servers::inference #endif // SERVICES_BIDDING_SERVICE_INFERENCE_INFERENCE_UTILS_H_ diff --git a/services/bidding_service/inference/inference_utils_test.cc b/services/bidding_service/inference/inference_utils_test.cc index 467c0945..77772d79 100644 --- a/services/bidding_service/inference/inference_utils_test.cc +++ b/services/bidding_service/inference/inference_utils_test.cc @@ -54,10 +54,11 @@ class InferenceUtilsTest : public ::testing::Test { // TODO(b/322030670): Making static SandboxExecutor compatible with multiple // tests. -TEST_F(InferenceUtilsTest, ReturnValueIsSet) { +TEST_F(InferenceUtilsTest, TestAPIOutputs) { SandboxExecutor& inference_executor = Executor(); CHECK_EQ(inference_executor.StartSandboxee().code(), absl::StatusCode::kOk); + // register a model ASSERT_TRUE(RegisterModelsFromLocal({std::string(kTestModelPath)}).ok()); google::scp::roma::proto::FunctionBindingIoProto input_output_proto; google::scp::roma::FunctionBindingPayload wrapper{ @@ -70,6 +71,14 @@ TEST_F(InferenceUtilsTest, ReturnValueIsSet) { // populate the output string. ASSERT_EQ(wrapper.io_proto.output_string(), "0.57721"); + // make sure GetModelPaths returns the registered model + google::scp::roma::proto::FunctionBindingIoProto input_output_proto_1; + google::scp::roma::FunctionBindingPayload wrapper_1{ + input_output_proto_1, {}}; + GetModelPaths(wrapper_1); + ASSERT_EQ(wrapper_1.io_proto.output_string(), + "[\"" + std::string(kTestModelPath) + "\"]"); + absl::StatusOr result = inference_executor.StopSandboxee(); ASSERT_TRUE(result.ok()); ASSERT_EQ(result->final_status(), sandbox2::Result::EXTERNAL_KILL); @@ -98,5 +107,21 @@ TEST_F(InferenceUtilsTest, RegisterModelsFromBucket_Error) { absl::StatusCode::kNotFound); } +TEST_F(InferenceUtilsTest, GetModelResponseToJsonOuput) { + GetModelPathsResponse get_model_paths_response; + EXPECT_EQ("[]", GetModelResponseToJson(get_model_paths_response)); + ModelSpec* spec; + + spec = get_model_paths_response.add_model_specs(); + spec->set_model_path("a"); + + EXPECT_EQ("[\"a\"]", GetModelResponseToJson(get_model_paths_response)); + + spec = get_model_paths_response.add_model_specs(); + spec->set_model_path("b"); + + EXPECT_EQ("[\"a\",\"b\"]", GetModelResponseToJson(get_model_paths_response)); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor.cc b/services/bidding_service/protected_app_signals_generate_bids_reactor.cc index edf01550..12b2f932 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor.cc +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor.cc @@ -157,9 +157,11 @@ void ProtectedAppSignalsGenerateBidsReactor::FetchAds( const std::string& prepare_data_for_ads_retrieval_response) { PS_VLOG(8, log_context_) << __func__; auto status = ad_retrieval_async_client_->ExecuteInternal( - CreateAdsRetrievalRequest(prepare_data_for_ads_retrieval_response), {}, + CreateAdsRetrievalRequest(prepare_data_for_ads_retrieval_response), + /* metadata= */ {}, [this, prepare_data_for_ads_retrieval_response]( - KVLookUpResult ad_retrieval_result) { + KVLookUpResult ad_retrieval_result, + ResponseMetadata response_metadata) { if (!ad_retrieval_result.ok()) { PS_VLOG(kNoisyWarn, log_context_) << "Ad retrieval request failed: " << ad_retrieval_result.status(); @@ -231,11 +233,18 @@ absl::StatusOr ProtectedAppSignalsGenerateBidsReactor:: ParseProtectedSignalsGenerateBidsResponse(const std::string& response) { PS_VLOG(8, log_context_) << __func__; - PS_ASSIGN_OR_RETURN(auto generate_bid_response, - ParseAndGetResponseJson(enable_adtech_code_logging_, - response, log_context_), - _ << "Failed to parse ProtectedAppSignalsAdWithBid JSON " - "response from Roma"); + std::string generate_bid_response; + if (enable_adtech_code_logging_) { + PS_ASSIGN_OR_RETURN( + generate_bid_response, + ParseAndGetResponseJson(enable_adtech_code_logging_, response, + log_context_), + _ << "Failed to parse ProtectedAppSignalsAdWithBid JSON " + "response from Roma"); + } else { + generate_bid_response = response; + } + ProtectedAppSignalsAdWithBid bid; PS_RETURN_IF_ERROR( google::protobuf::util::JsonStringToMessage(generate_bid_response, &bid)); @@ -379,9 +388,10 @@ void ProtectedAppSignalsGenerateBidsReactor::FetchAdsMetadata( .ad_render_ids(), ", "); auto status = kv_async_client_->ExecuteInternal( - CreateKVLookupRequest(ad_render_ids), {}, + CreateKVLookupRequest(ad_render_ids), /* metadata= */ {}, [this, prepare_data_for_ads_retrieval_response]( - KVLookUpResult kv_look_up_result) { + KVLookUpResult kv_look_up_result, + ResponseMetadata response_metadata) { PS_VLOG(8) << "On KV response"; if (!kv_look_up_result.ok()) { PS_VLOG(kNoisyWarn, log_context_) @@ -413,8 +423,10 @@ void ProtectedAppSignalsGenerateBidsReactor::Execute() { PS_VLOG(8, log_context_) << __func__; PS_VLOG(kEncrypted, log_context_) << "GenerateBidsRequest:\n" << request_->ShortDebugString(); + log_context_.SetEventMessageField(*request_); PS_VLOG(kPlain, log_context_) << "GenerateBidsRawRequest:\n" << raw_request_.ShortDebugString(); + log_context_.SetEventMessageField(raw_request_); if (IsContextualRetrievalRequest()) { StartContextualAdsRetrieval(); @@ -435,6 +447,11 @@ void ProtectedAppSignalsGenerateBidsReactor::OnCancel() {} void ProtectedAppSignalsGenerateBidsReactor::EncryptResponseAndFinish( grpc::Status status) { PS_VLOG(8, log_context_) << __func__; + PS_VLOG(kPlain, log_context_) + << "GenerateProtectedAppSignalsBidsRawResponse:\n" + << raw_response_.ShortDebugString(); + log_context_.SetEventMessageField(raw_response_); + log_context_.ExportEventMessage(); if (!EncryptResponse()) { PS_LOG(ERROR, log_context_) << "Failed to encrypt the generate app signals bids response."; diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc b/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc index fcdf92f1..2cf47e0e 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc @@ -82,7 +82,6 @@ class GenerateBidsReactorTest : public ::testing::Test { if (!runtime_config.has_value()) { runtime_config = { .enable_buyer_debug_url_generation = false, - .enable_adtech_code_logging = false, }; } // Create a request. @@ -117,13 +116,15 @@ TEST_F(GenerateBidsReactorTest, WinningBidIsGenerated) { .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = CreateAdsRetrievalOrKvLookupResponse(); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); @@ -153,9 +154,10 @@ TEST_F(GenerateBidsReactorTest, AdsRetrievalTimeoutIsUsed) { .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { EXPECT_EQ(timeout, absl::Milliseconds(kTestAdRetrievalTimeoutMs)); return absl::OkStatus(); }); @@ -167,7 +169,6 @@ TEST_F(GenerateBidsReactorTest, AdsRetrievalTimeoutIsUsed) { RunReactorWithRequest( raw_request, BiddingServiceRuntimeConfig({ .enable_buyer_debug_url_generation = false, - .enable_adtech_code_logging = false, .ad_retrieval_timeout_ms = kTestAdRetrievalTimeoutMs, })); } @@ -224,9 +225,10 @@ TEST_F(GenerateBidsReactorTest, AdRetrievalClientInputIsCorrect) { .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { EXPECT_EQ(raw_request->partitions().size(), 1); const auto& udf_arguments = raw_request->partitions()[0].arguments(); EXPECT_EQ(udf_arguments.size(), kNumAdRetrievalUdfArguments); @@ -252,7 +254,8 @@ TEST_F(GenerateBidsReactorTest, AdRetrievalClientInputIsCorrect) { auto response = CreateAdsRetrievalOrKvLookupResponse(); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); @@ -317,13 +320,15 @@ TEST_F(GenerateBidsReactorTest, GenerateBidsInputIsCorrect) { .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = CreateAdsRetrievalOrKvLookupResponse(); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); @@ -346,13 +351,15 @@ TEST_F(GenerateBidsReactorTest, egressPayloadAreNotPopulated) { .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = CreateAdsRetrievalOrKvLookupResponse(); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); @@ -404,13 +411,15 @@ TEST_F(GenerateBidsReactorTest, ZeroBidsAreFiltered) { .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = CreateAdsRetrievalOrKvLookupResponse(); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); @@ -450,13 +459,15 @@ TEST_F(GenerateBidsReactorTest, .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = CreateAdsRetrievalOrKvLookupResponse(""); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); @@ -478,13 +489,15 @@ TEST_F(GenerateBidsReactorTest, NoContextualAdsMeansAdRetrievalServiceInvoked) { .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = CreateAdsRetrievalOrKvLookupResponse(); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); EXPECT_CALL(kv_async_client_, ExecuteInternal).Times(0); @@ -517,13 +530,15 @@ TEST_F(GenerateBidsReactorTest, ContextualAdsMeansKVServiceInvoked) { .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = CreateAdsRetrievalOrKvLookupResponse(); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); ContextualProtectedAppSignalsData contextual_pas_data; @@ -553,9 +568,10 @@ TEST_F(GenerateBidsReactorTest, KvInputIsCorrect) { .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { EXPECT_EQ(raw_request->partitions().size(), 1); const auto& udf_arguments = raw_request->partitions()[0].arguments(); EXPECT_EQ(udf_arguments.size(), kNumKVLookupUdfArguments); @@ -568,7 +584,8 @@ TEST_F(GenerateBidsReactorTest, KvInputIsCorrect) { auto response = CreateAdsRetrievalOrKvLookupResponse(); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); @@ -600,13 +617,15 @@ TEST_F(GenerateBidsReactorTest, TemporaryEgressVectorGetsPopulated) { .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = CreateAdsRetrievalOrKvLookupResponse(); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); @@ -639,13 +658,15 @@ TEST_F(GenerateBidsReactorTest, .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = CreateAdsRetrievalOrKvLookupResponse(); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); @@ -678,13 +699,15 @@ TEST_F(GenerateBidsReactorTest, .WillOnce([](std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable>)&&> + absl::StatusOr>, + ResponseMetadata)&&> on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = CreateAdsRetrievalOrKvLookupResponse(); EXPECT_TRUE(response.ok()) << response.status(); std::move(on_done)( - std::make_unique(*std::move(response))); + std::make_unique(*std::move(response)), + /* response_metadata= */ {}); return absl::OkStatus(); }); diff --git a/services/bidding_service/utils/BUILD b/services/bidding_service/utils/BUILD new file mode 100644 index 00000000..ffa8c1ce --- /dev/null +++ b/services/bidding_service/utils/BUILD @@ -0,0 +1,51 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") + +cc_library( + name = "egress", + srcs = [ + "egress.cc", + "//third_party/cddl:cddl_h", + ], + hdrs = [ + "egress.h", + ], + includes = [ + "include", + ], + visibility = ["//visibility:public"], + deps = [ + "@cddl_lib//:cddl", + "@com_google_absl//absl/strings", + ], +) + +cc_test( + name = "egress_test", + size = "small", + srcs = ["egress_test.cc"], + data = [ + "//services/bidding_service/egress_cddl_spec:1.0.0", + ], + deps = [ + ":egress", + "//services/common/util:file_util", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/services/bidding_service/utils/egress.cc b/services/bidding_service/utils/egress.cc new file mode 100644 index 00000000..8564d4d2 --- /dev/null +++ b/services/bidding_service/utils/egress.cc @@ -0,0 +1,27 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/bidding_service/utils/egress.h" + +#include "absl/strings/string_view.h" +#include "third_party/cddl/include/cddl.h" + +namespace privacy_sandbox::bidding_auction_servers { + +bool AdtechEgresSchemaValid(absl::string_view adtech_schema, + absl::string_view cddl_spec) { + return validate_json_str(cddl_spec.data(), adtech_schema.data()); +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/bidding_service/utils/egress.h b/services/bidding_service/utils/egress.h new file mode 100644 index 00000000..0889dce9 --- /dev/null +++ b/services/bidding_service/utils/egress.h @@ -0,0 +1,28 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_BIDDING_SERVICE_UTILS_EGRESS_H_ +#define SERVICES_BIDDING_SERVICE_UTILS_EGRESS_H_ + +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers { + +// Verifies adtech provided schema conformance with the give CDDL spec. +bool AdtechEgresSchemaValid(absl::string_view adtech_schema, + absl::string_view cddl_spec); + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_BIDDING_SERVICE_UTILS_EGRESS_H_ diff --git a/services/bidding_service/utils/egress_test.cc b/services/bidding_service/utils/egress_test.cc new file mode 100644 index 00000000..fc3c1e18 --- /dev/null +++ b/services/bidding_service/utils/egress_test.cc @@ -0,0 +1,325 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/bidding_service/utils/egress.h" + +#include +#include + +#include "absl/log/check.h" +#include "absl/strings/string_view.h" +#include "gtest/gtest.h" +#include "services/common/util/file_util.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { + +inline constexpr absl::string_view kTestCddlSpec = R"""( +schema = { + features: [* feature_types], + ; Version of the CDDL used when constructing this `schema` instance. + cddl_version: cddl_version, + ; Version of this schema instance itself. Will be prepended to the wire format + ; of the `payload` after validation. 3 bits, so 8 maximum concurrent `schema`s + ; are supported. See wire format for details. + version: uint .le 7 +} + +feature = boolean-feature / bucket-feature ; Additional `feature` types here. + +feature_types /= boolean-feature-type / bucket-feature-type ; More here too. + +; Type for `boolean-feature`. +boolean-feature-type = { + name: text .regexp "^boolean-feature$" +} + +; A `feature` representing a single boolean value. +; +; 1 bit of information. +boolean-feature = { + type: boolean-feature-type, + value: bool, +} + +; Type for `bucket-feature`. +bucket-feature-type = { + name: text .regexp "^bucket-feature$" + buckets: uint, ; number of buckets. +} + +; A `feature` representing a bucketized value. +bucket-feature = { + type: bucket-feature-type, + value: [* bool], +} + +; Version of this CDDL used when validating `schema`s and `payload`s. +cddl_version /= "1.0.0" ; In v1.0.1, this would be "1.0.0" / "1.0.1". + +)"""; + +TEST(ValidateAdtechSchema, SucceedsWithConformingSchema) { + std::string adtech_schema = R"JSON( + { + "cddl_version": "1.0.0", + "version": 7, + "features": [ + { + "name": "boolean-feature" + }, + { + "name": "bucket-feature", + "buckets": 2 + } + ] + } + )JSON"; + EXPECT_TRUE(AdtechEgresSchemaValid(adtech_schema, kTestCddlSpec)) + << "Adtech provided schema:\n" + << adtech_schema << "\n is not conformant with spec:\n" + << kTestCddlSpec; +} + +TEST(ValidateAdtechSchema, SucceedsWithOptionalFeatureMissing) { + std::string adtech_schema = R"JSON( + { + "cddl_version": "1.0.0", + "version": 7, + "features": [ + { + "name": "bucket-feature", + "buckets": 2 + } + ] + } + )JSON"; + EXPECT_TRUE(AdtechEgresSchemaValid(adtech_schema, kTestCddlSpec)) + << "Adtech provided schema:\n" + << adtech_schema << "\n is not conformant with spec:\n" + << kTestCddlSpec; +} + +TEST(ValidateAdtechSchema, FailsWithMismatchingFeatureName) { + std::string adtech_schema = R"JSON( + { + "cddl_version": "1.0.0", + "version": 7, + "features": [ + { + "name": "unknown-feature", + } + ] + } + )JSON"; + EXPECT_FALSE(AdtechEgresSchemaValid(adtech_schema, kTestCddlSpec)) + << "Adtech provided schema:\n" + << adtech_schema << "\n is unexpectedly conformant with spec:\n" + << kTestCddlSpec; +} + +TEST(ValidateAdtechSchema, FailsWithUnsupportedCddlVersion) { + std::string adtech_schema = R"JSON( + { + "cddl_version": "10.0.1", + "version": 7, + "features": [ + { + "name": "bucket-feature", + } + ] + } + )JSON"; + EXPECT_FALSE(AdtechEgresSchemaValid(adtech_schema, kTestCddlSpec)) + << "Adtech provided schema:\n" + << adtech_schema << "\n is unexpectedly conformant with spec:\n" + << kTestCddlSpec; +} + +TEST(ValidateAdtechSchema, FailsWithUnsupportedSchemaVersion) { + std::string adtech_schema = R"JSON( + { + "cddl_version": "1.0.0", + "version": 9, + "features": [ + { + "name": "bucket-feature", + } + ] + } + )JSON"; + EXPECT_FALSE(AdtechEgresSchemaValid(adtech_schema, kTestCddlSpec)) + << "Adtech provided schema:\n" + << adtech_schema << "\n is unexpectedly conformant with spec:\n" + << kTestCddlSpec; +} + +constexpr absl::string_view kCddlSpec100Path = + "services/bidding_service/egress_cddl_spec/1.0.0"; + +class CddlSpecTest : public ::testing::Test { + protected: + void SetUp() override { + auto file_content = GetFileContent(kCddlSpec100Path, /*log_on_error=*/true); + CHECK_OK(file_content); + cddl_spec_ = *std::move(file_content); + } + + const std::string& CddlSpec() { return cddl_spec_; } + + private: + std::string cddl_spec_; +}; + +TEST_F(CddlSpecTest, SucceedsWithValidAdtechSchema) { + std::string adtech_schema = R"JSON( + { + "cddl_version": "1.0.0", + "version": 2, + "features": [ + { + "name": "boolean-feature" + }, + { + "name": "unsigned-integer-feature", + "size": 4 + }, + { + "name": "signed-integer-feature", + "size": 4 + }, + { + "name": "bucket-feature", + "buckets": 2 + }, + { + "name": "histogram-feature", + "size": 3, + "value": [ + { + "name": "unsigned-integer-feature", + "size": 4 + }, + { + "name": "signed-integer-feature", + "size": 2 + }, + { + "name": "unsigned-integer-feature", + "size": 4 + } + ] + } + ] + } + )JSON"; + EXPECT_TRUE(AdtechEgresSchemaValid(adtech_schema, CddlSpec())) + << "Adtech provided schema:\n" + << adtech_schema << "\n is not conformant with spec:\n" + << CddlSpec(); +} + +TEST_F(CddlSpecTest, FailsIfSizeNotSpecifiedForUnsignedInteger) { + std::string adtech_schema = R"JSON( + { + "cddl_version": "1.0.0", + "version": 2, + "features": [ + { + "name": "unsigned-integer-feature" + } + ] + } + )JSON"; + EXPECT_FALSE(AdtechEgresSchemaValid(adtech_schema, CddlSpec())) + << "Adtech provided schema:\n" + << adtech_schema << "\n is not unexpectedly conformant with spec:\n" + << CddlSpec(); +} + +TEST_F(CddlSpecTest, FailsIfSizeNotSpecifiedForSignedInteger) { + std::string adtech_schema = R"JSON( + { + "cddl_version": "1.0.0", + "version": 2, + "features": [ + { + "name": "signed-integer-feature" + } + ] + } + )JSON"; + EXPECT_FALSE(AdtechEgresSchemaValid(adtech_schema, CddlSpec())) + << "Adtech provided schema:\n" + << adtech_schema << "\n is not unexpectedly conformant with spec:\n" + << CddlSpec(); +} + +TEST_F(CddlSpecTest, FailsIfBucketsNotSpecifiedForBucketFeature) { + std::string adtech_schema = R"JSON( + { + "cddl_version": "1.0.0", + "version": 2, + "features": [ + { + "name": "bucket-feature" + } + ] + } + )JSON"; + EXPECT_FALSE(AdtechEgresSchemaValid(adtech_schema, CddlSpec())) + << "Adtech provided schema:\n" + << adtech_schema << "\n is unexpectedly conformant with spec:\n" + << CddlSpec(); +} + +TEST_F(CddlSpecTest, FailsIfSizeNotSpecifiedForHistogramFeature) { + std::string adtech_schema = R"JSON( + { + "cddl_version": "1.0.0", + "version": 2, + "features": [ + { + "name": "histogram-feature" + } + ] + } + )JSON"; + EXPECT_FALSE(AdtechEgresSchemaValid(adtech_schema, CddlSpec())) + << "Adtech provided schema:\n" + << adtech_schema << "\n is unexpectedly conformant with spec:\n" + << CddlSpec(); +} + +TEST_F(CddlSpecTest, FailsIfCddlVersionMismatches) { + std::string adtech_schema = R"JSON( + { + "cddl_version": "1.0.1", + "version": 2, + "features": [ + { + "name": "histogram-feature", + "size": 2 + } + ] + } + )JSON"; + EXPECT_FALSE(AdtechEgresSchemaValid(adtech_schema, CddlSpec())) + << "Adtech provided schema:\n" + << adtech_schema << "\n is unexpectedly conformant with spec:\n" + << CddlSpec(); +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/buyer_frontend_service/buyer_frontend_main.cc b/services/buyer_frontend_service/buyer_frontend_main.cc index 1e20d9ee..c3240d8b 100644 --- a/services/buyer_frontend_service/buyer_frontend_main.cc +++ b/services/buyer_frontend_service/buyer_frontend_main.cc @@ -55,6 +55,8 @@ ABSL_FLAG(std::optional, healthcheck_port, std::nullopt, ABSL_FLAG(std::optional, bidding_server_addr, std::nullopt, "Bidding Server Address"); +ABSL_FLAG(std::optional, grpc_arg_default_authority, std::nullopt, + "Domain for bidding server, or other domain for TLS Authority"); ABSL_FLAG(std::optional, buyer_kv_server_addr, std::nullopt, "Buyer KV Server Address"); // Added for performance/benchmark testing of both types of http clients. @@ -108,6 +110,8 @@ absl::StatusOr GetConfigClient( config_client.SetFlag(FLAGS_port, PORT); config_client.SetFlag(FLAGS_healthcheck_port, HEALTHCHECK_PORT); config_client.SetFlag(FLAGS_bidding_server_addr, BIDDING_SERVER_ADDR); + config_client.SetFlag(FLAGS_grpc_arg_default_authority, + GRPC_ARG_DEFAULT_AUTHORITY_VAL); config_client.SetFlag(FLAGS_buyer_kv_server_addr, BUYER_KV_SERVER_ADDR); config_client.SetFlag(FLAGS_generate_bid_timeout_ms, GENERATE_BID_TIMEOUT_MS); config_client.SetFlag(FLAGS_protected_app_signals_generate_bid_timeout_ms, @@ -203,6 +207,8 @@ absl::Status RunServer() { int port = config_client.GetIntParameter(PORT); std::string bidding_server_addr = std::string(config_client.GetStringParameter(BIDDING_SERVER_ADDR)); + std::string grpc_arg_default_authority = std::string( + config_client.GetStringParameter(GRPC_ARG_DEFAULT_AUTHORITY_VAL)); std::string buyer_kv_server_addr = std::string(config_client.GetStringParameter(BUYER_KV_SERVER_ADDR)); bool enable_buyer_frontend_benchmarking = @@ -245,7 +251,8 @@ absl::Status RunServer() { .secure_client = config_client.GetBooleanParameter(BIDDING_EGRESS_TLS), .is_pas_enabled = - config_client.GetBooleanParameter(ENABLE_PROTECTED_APP_SIGNALS)}, + config_client.GetBooleanParameter(ENABLE_PROTECTED_APP_SIGNALS), + .grpc_arg_default_authority = grpc_arg_default_authority}, CreateKeyFetcherManager(config_client, CreatePublicKeyFetcher(config_client)), CreateCryptoClient(), diff --git a/services/buyer_frontend_service/buyer_frontend_service.cc b/services/buyer_frontend_service/buyer_frontend_service.cc index acacbe8f..af78f3c5 100644 --- a/services/buyer_frontend_service/buyer_frontend_service.cc +++ b/services/buyer_frontend_service/buyer_frontend_service.cc @@ -39,9 +39,10 @@ BuyerFrontEndService::BuyerFrontEndService( enable_benchmarking_(enable_benchmarking), key_fetcher_manager_(std::move(key_fetcher_manager)), crypto_client_(std::move(crypto_client)), - stub_(Bidding::NewStub(CreateChannel(client_config.server_addr, - client_config.compression, - client_config.secure_client))), + stub_(Bidding::NewStub( + CreateChannel(client_config.server_addr, client_config.compression, + client_config.secure_client, + client_config.grpc_arg_default_authority))), bidding_async_client_(std::make_unique( key_fetcher_manager_.get(), crypto_client_.get(), client_config, stub_.get())) { diff --git a/services/buyer_frontend_service/buyer_frontend_service_test.cc b/services/buyer_frontend_service/buyer_frontend_service_test.cc index 0a975af7..0f7dd3c3 100644 --- a/services/buyer_frontend_service/buyer_frontend_service_test.cc +++ b/services/buyer_frontend_service/buyer_frontend_service_test.cc @@ -167,16 +167,18 @@ std::unique_ptr GetValidBiddingAsyncClientMock() { ExecuteInternal( An>(), An(), - An>) &&>>(), - An())) + An>, + ResponseMetadata) &&>>(), + An(), An())) .WillOnce([](std::unique_ptr get_values_raw_request, const RequestMetadata& metadata, auto on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto raw_response = std::make_unique(); *raw_response->mutable_bids()->Add() = CreateAdWithBid(); - std::move(on_done)(std::move(raw_response)); + std::move(on_done)(std::move(raw_response), + /* response_metadata= */ {}); return absl::OkStatus(); }); return bidding_async_client; @@ -191,9 +193,10 @@ GetValidBiddingAsyncClientMockNotCalled() { ExecuteInternal( An>(), An(), - An>) &&>>(), - An())) + An>, + ResponseMetadata) &&>>(), + An(), An())) .Times(0); return bidding_async_client; } @@ -209,17 +212,19 @@ GetValidProtectedAppSignalsBiddingClientMock() { An(), An>) &&>>(), - An())) + GenerateProtectedAppSignalsBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .WillOnce([](std::unique_ptr get_values_raw_request, const RequestMetadata& metadata, auto on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto raw_response = std::make_unique(); *raw_response->mutable_bids()->Add() = CreateProtectedAppSignalsAdWithBid(); - std::move(on_done)(std::move(raw_response)); + std::move(on_done)(std::move(raw_response), + /* response_metadata= */ {}); return absl::OkStatus(); }); return protected_app_signals_bidding_async_client; @@ -340,15 +345,17 @@ std::unique_ptr GetErrorBiddingAsyncClientMock() { ExecuteInternal( An>(), An(), - An>) &&>>(), - An())) + An>, + ResponseMetadata) &&>>(), + An(), An())) .WillOnce([](std::unique_ptr get_values_raw_request, const RequestMetadata& metadata, auto on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { std::move(on_done)(absl::Status(absl::StatusCode::kInvalidArgument, - kRejectionReasonBidBelowAuctionFloor)); + kRejectionReasonBidBelowAuctionFloor), + /* response_metadata= */ {}); return absl::OkStatus(); }); return bidding_async_client; @@ -394,14 +401,16 @@ GetErrorProtectedAppSignalsBiddingClientMock() { An(), An>) &&>>(), - An())) + GenerateProtectedAppSignalsBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .WillOnce([](std::unique_ptr get_values_raw_request, const RequestMetadata& metadata, auto on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { std::move(on_done)(absl::Status(absl::StatusCode::kInvalidArgument, - kRejectionReasonCategoryExclusions)); + kRejectionReasonCategoryExclusions), + /* response_metadata= */ {}); return absl::OkStatus(); }); return protected_app_signals_bidding_async_client; @@ -444,14 +453,16 @@ std::unique_ptr GetEmptyBiddingAsyncClientMock() { ExecuteInternal( An>(), An(), - An>) &&>>(), - An())) + An>, + ResponseMetadata) &&>>(), + An(), An())) .WillOnce([](std::unique_ptr get_values_raw_request, const RequestMetadata& metadata, auto on_done, - absl::Duration timeout) { - std::move(on_done)(std::make_unique()); + absl::Duration timeout, RequestConfig request_config) { + std::move(on_done)(std::make_unique(), + /* response_metadata= */ {}); return absl::OkStatus(); }); return bidding_async_client; @@ -497,14 +508,16 @@ GetEmptyProtectedAppSignalsBiddingClientMock() { An(), An>) &&>>(), - An())) + GenerateProtectedAppSignalsBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .WillOnce([](std::unique_ptr get_values_raw_request, const RequestMetadata& metadata, auto on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { std::move(on_done)( - std::make_unique()); + std::make_unique(), + /* response_metadata= */ {}); return absl::OkStatus(); }); return protected_app_signals_bidding_async_client; @@ -642,8 +655,9 @@ GetProtectedAppSignalsBiddingClientMockThatWillNotBeCalled() { An(), An>) &&>>(), - An())) + GenerateProtectedAppSignalsBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .Times(0); return protected_app_signals_bidding_async_client; } @@ -688,9 +702,10 @@ TEST_F(BuyerFrontEndServiceTest, ExecuteInternal( An>(), An(), - An>) &&>>(), - An())) + An>, + ResponseMetadata) &&>>(), + An(), An())) .Times(0); auto protected_app_signals_bidding_async_client = GetProtectedAppSignalsBiddingClientMockThatWillNotBeCalled(); diff --git a/services/buyer_frontend_service/get_bids_unary_reactor.cc b/services/buyer_frontend_service/get_bids_unary_reactor.cc index f594abfa..f3c59686 100644 --- a/services/buyer_frontend_service/get_bids_unary_reactor.cc +++ b/services/buyer_frontend_service/get_bids_unary_reactor.cc @@ -103,7 +103,7 @@ GetBidsUnaryReactor::GetBidsUnaryReactor( crypto_client_(crypto_client), log_context_([this]() { decrypt_status_ = DecryptRequest(); - return server_common::log::ContextImpl( + return RequestLogContext( GetLoggingContext(), raw_request_.consented_debug_config(), [this]() { return get_bids_raw_response_->mutable_debug_info(); }); }()), @@ -155,6 +155,8 @@ void GetBidsUnaryReactor::OnAllBidsDone(bool any_successful_bids) { PS_VLOG(kPlain, log_context_) << "GetBidsRawResponse:\n" << get_bids_raw_response_->ShortDebugString(); + log_context_.SetEventMessageField(*get_bids_raw_response_); + log_context_.ExportEventMessage(); if (auto encryption_status = EncryptResponse(); !encryption_status.ok()) { PS_LOG(ERROR, log_context_) << "Failed to encrypt the response"; @@ -224,6 +226,7 @@ void GetBidsUnaryReactor::Execute() { benchmarking_logger_->Begin(); PS_VLOG(kEncrypted, log_context_) << "Encrypted GetBidsRequest:\n" << request_->ShortDebugString(); + log_context_.SetEventMessageField(*request_); PS_VLOG(kPlain, log_context_) << "Headers:\n" << absl::StrJoin(context_->client_metadata(), "\n", @@ -239,6 +242,7 @@ void GetBidsUnaryReactor::Execute() { PS_VLOG(5, log_context_) << "Successfully decrypted the request"; PS_VLOG(kPlain, log_context_) << "GetBidsRawRequest:\n" << raw_request_.ShortDebugString(); + log_context_.SetEventMessageField(raw_request_); const int num_bidding_calls = GetNumberOfBiddingCalls(); if (num_bidding_calls == 0) { @@ -291,9 +295,11 @@ void GetBidsUnaryReactor::MayGetProtectedSignalsBids() { absl::Status execute_result = protected_app_signals_bidding_async_client_->ExecuteInternal( std::move(protected_app_signals_bid_request), bidding_metadata_, - [this](absl::StatusOr< - std::unique_ptr> - raw_response) { + [this]( + absl::StatusOr< + std::unique_ptr> + raw_response, + ResponseMetadata response_metadata) { HandleSingleBidCompletion< GenerateProtectedAppSignalsBidsRawResponse>( std::move(raw_response), @@ -429,7 +435,8 @@ void GetBidsUnaryReactor::PrepareAndGenerateProtectedAudienceBid( [this, bidding_request = std::move(bidding_request)]( absl::StatusOr< std::unique_ptr> - raw_response) mutable { + raw_response, + ResponseMetadata response_metadata) mutable { { int response_size = raw_response.ok() ? (int)raw_response->get()->ByteSizeLong() : 0; @@ -506,7 +513,6 @@ void GetBidsUnaryReactor::FinishWithStatus(const grpc::Status& status) { if (status.error_code() != grpc::StatusCode::OK) { metric_context_->SetRequestResult(server_common::ToAbslStatus(status)); } - Finish(status); } diff --git a/services/buyer_frontend_service/get_bids_unary_reactor.h b/services/buyer_frontend_service/get_bids_unary_reactor.h index 334f20bc..b2fa3976 100644 --- a/services/buyer_frontend_service/get_bids_unary_reactor.h +++ b/services/buyer_frontend_service/get_bids_unary_reactor.h @@ -35,10 +35,10 @@ #include "services/common/clients/bidding_server/bidding_async_client.h" #include "services/common/encryption/crypto_client_wrapper_interface.h" #include "services/common/loggers/benchmarking_logger.h" +#include "services/common/loggers/request_log_context.h" #include "services/common/metric/server_definition.h" #include "services/common/util/async_task_tracker.h" #include "src/encryption/key_fetcher/interface/key_fetcher_manager_interface.h" -#include "src/logger/request_context_impl.h" namespace privacy_sandbox::bidding_auction_servers { @@ -150,7 +150,7 @@ class GetBidsUnaryReactor : public grpc::ServerUnaryReactor { std::string hpke_secret_; grpc::Status decrypt_status_; - server_common::log::ContextImpl log_context_; + RequestLogContext log_context_; // Used to log metric, same life time as reactor. std::unique_ptr metric_context_; diff --git a/services/buyer_frontend_service/get_bids_unary_reactor_test.cc b/services/buyer_frontend_service/get_bids_unary_reactor_test.cc index 858212ed..651751fd 100644 --- a/services/buyer_frontend_service/get_bids_unary_reactor_test.cc +++ b/services/buyer_frontend_service/get_bids_unary_reactor_test.cc @@ -176,15 +176,17 @@ TEST_F(GetBidUnaryReactorTest, LoadsBiddingSignalsAndCallsBiddingServer) { An(), An>) &&>>(), - An())) + GenerateBidsResponse::GenerateBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .WillOnce([¬ification]( std::unique_ptr get_values_raw_request, const RequestMetadata& metadata, auto on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { std::move(on_done)( - std::make_unique()); + std::make_unique(), + /* response_metadata= */ {}); notification.Notify(); return absl::OkStatus(); }); @@ -214,17 +216,19 @@ TEST_F(GetBidUnaryReactorTest, An(), An>) &&>>(), - An())) + GenerateBidsResponse::GenerateBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .WillOnce([¬ification]( std::unique_ptr get_values_request, const RequestMetadata& metadata, auto on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto raw_response = std::make_unique(); raw_response->mutable_bids()->Add(); - std::move(on_done)(std::move(raw_response)); + std::move(on_done)(std::move(raw_response), + /* response_metadata= */ {}); notification.Notify(); return absl::OkStatus(); }); @@ -276,7 +280,7 @@ TEST_F(GetBidUnaryReactorTest, VerifyLogContextPropagates) { EXPECT_CALL(bidding_client_mock_, ExecuteInternal(Pointee(EqGenerateBidsRawRequestWithLogContext( expected_generate_bids_raw_request)), - _, _, _)); + _, _, _, _)); GetBidsUnaryReactor get_bids_unary_reactor( context_, request_, response_, bidding_signals_provider_, @@ -381,19 +385,19 @@ TEST_F(GetProtectedAppSignalsTest, CorrectGenerateBidSentToBiddingService) { EXPECT_CALL( protected_app_signals_bidding_client_mock_, - ExecuteInternal( - Pointee( - EqGenerateProtectedAppSignalsBidsRawRequest(expected_request)), - An(), - An>) &&>>(), - An())) + ExecuteInternal(Pointee(EqGenerateProtectedAppSignalsBidsRawRequest( + expected_request)), + An(), + An>, + ResponseMetadata) &&>>(), + An(), An())) .Times(1); // No protected audience buyer input and hence no outbound call to bidding // service. - EXPECT_CALL(bidding_client_mock_, ExecuteInternal(_, _, _, _)).Times(0); + EXPECT_CALL(bidding_client_mock_, ExecuteInternal).Times(0); GetBidsUnaryReactor class_under_test( context_, request_, response_, bidding_signals_provider_, @@ -415,8 +419,9 @@ TEST_F(GetProtectedAppSignalsTest, TimeoutIsRespected) { An(), An>) &&>>(), - Eq(kTestProtectedAppSignalsGenerateBidTimeout))); + GenerateProtectedAppSignalsBidsRawResponse>>, + ResponseMetadata) &&>>(), + Eq(kTestProtectedAppSignalsGenerateBidTimeout), An())); GetBidsUnaryReactor class_under_test( context_, request_, response_, bidding_signals_provider_, @@ -441,8 +446,9 @@ TEST_F(GetProtectedAppSignalsTest, RespectsFeatureFlagOff) { An(), An>) &&>>(), - An())) + GenerateProtectedAppSignalsBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .Times(0); EXPECT_CALL( bidding_client_mock_, @@ -451,8 +457,9 @@ TEST_F(GetProtectedAppSignalsTest, RespectsFeatureFlagOff) { An(), An>) &&>>(), - An())) + GenerateBidsResponse::GenerateBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .Times(1); GetBidsUnaryReactor class_under_test( context_, request_, response_, bidding_signals_provider_, @@ -478,8 +485,9 @@ TEST_F(GetProtectedAppSignalsTest, RespectsProtectedAudienceFeatureFlagOff) { An(), An>) &&>>(), - An())) + GenerateBidsResponse::GenerateBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .Times(0); EXPECT_CALL( protected_app_signals_bidding_client_mock_, @@ -488,8 +496,9 @@ TEST_F(GetProtectedAppSignalsTest, RespectsProtectedAudienceFeatureFlagOff) { An(), An>) &&>>(), - An())) + GenerateProtectedAppSignalsBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .Times(1); GetBidsUnaryReactor class_under_test( context_, request_, response_, bidding_signals_provider_, @@ -517,17 +526,19 @@ TEST_F(GetProtectedAppSignalsTest, GetBidsResponseAggregatedBackToSfe) { An(), An>) &&>>(), - An())) + GenerateBidsResponse::GenerateBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .WillOnce([&bids_counter]( std::unique_ptr get_values_request, const RequestMetadata& metadata, auto on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto raw_response = std::make_unique(); *raw_response->mutable_bids()->Add() = CreateAdWithBid(); - std::move(on_done)(std::move(raw_response)); + std::move(on_done)(std::move(raw_response), + /* response_metadata= */ {}); bids_counter.DecrementCount(); return absl::OkStatus(); }); @@ -539,18 +550,20 @@ TEST_F(GetProtectedAppSignalsTest, GetBidsResponseAggregatedBackToSfe) { An(), An>) &&>>(), - An())) + GenerateProtectedAppSignalsBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .WillOnce([&bids_counter]( std::unique_ptr get_values_raw_request, const RequestMetadata& metadata, auto on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto raw_response = std::make_unique(); *raw_response->mutable_bids()->Add() = CreateProtectedAppSignalsAdWithBid(); - std::move(on_done)(std::move(raw_response)); + std::move(on_done)(std::move(raw_response), + /* response_metadata= */ {}); bids_counter.DecrementCount(); return absl::OkStatus(); }); @@ -624,8 +637,9 @@ TEST_F(GetProtectedAppSignalsTest, An(), An>) &&>>(), - An())) + GenerateBidsResponse::GenerateBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .Times(0); EXPECT_CALL( @@ -635,18 +649,20 @@ TEST_F(GetProtectedAppSignalsTest, An(), An>) &&>>(), - An())) + GenerateProtectedAppSignalsBidsRawResponse>>, + ResponseMetadata) &&>>(), + An(), An())) .WillOnce([&bids_counter]( std::unique_ptr get_values_raw_request, const RequestMetadata& metadata, auto on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto raw_response = std::make_unique(); *raw_response->mutable_bids()->Add() = CreateProtectedAppSignalsAdWithBid(); - std::move(on_done)(std::move(raw_response)); + std::move(on_done)(std::move(raw_response), + /* response_metadata= */ {}); bids_counter.DecrementCount(); return absl::OkStatus(); }); diff --git a/services/buyer_frontend_service/runtime_flags.h b/services/buyer_frontend_service/runtime_flags.h index d29ff736..01e02fc8 100644 --- a/services/buyer_frontend_service/runtime_flags.h +++ b/services/buyer_frontend_service/runtime_flags.h @@ -27,6 +27,8 @@ inline constexpr absl::string_view PORT = "BUYER_FRONTEND_PORT"; inline constexpr absl::string_view HEALTHCHECK_PORT = "BUYER_FRONTEND_HEALTHCHECK_PORT"; inline constexpr absl::string_view BIDDING_SERVER_ADDR = "BIDDING_SERVER_ADDR"; +inline constexpr absl::string_view GRPC_ARG_DEFAULT_AUTHORITY_VAL = + "GRPC_ARG_DEFAULT_AUTHORITY"; inline constexpr absl::string_view BUYER_KV_SERVER_ADDR = "BUYER_KV_SERVER_ADDR"; inline constexpr absl::string_view GENERATE_BID_TIMEOUT_MS = @@ -52,11 +54,12 @@ inline constexpr absl::string_view inline constexpr absl::string_view BFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES = "BFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES"; -inline constexpr int kNumRuntimeFlags = 16; +inline constexpr int kNumRuntimeFlags = 17; inline constexpr std::array kFlags = { PORT, HEALTHCHECK_PORT, BIDDING_SERVER_ADDR, + GRPC_ARG_DEFAULT_AUTHORITY_VAL, BUYER_KV_SERVER_ADDR, GENERATE_BID_TIMEOUT_MS, PROTECTED_APP_SIGNALS_GENERATE_BID_TIMEOUT_MS, diff --git a/services/common/blob_fetch/BUILD b/services/common/blob_fetch/BUILD index 7aa7dade..a581333a 100644 --- a/services/common/blob_fetch/BUILD +++ b/services/common/blob_fetch/BUILD @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_proto_library", "cc_test") +load("@rules_proto//proto:defs.bzl", "proto_library") package(default_visibility = ["//:__subpackages__"]) @@ -54,3 +55,16 @@ cc_test( "@google_privacysandbox_servers_common//src/public/cpio/mock/blob_storage_client:blob_storage_client_mock", ], ) + +proto_library( + name = "fetch_mode_proto", + srcs = ["fetch_mode.proto"], + deps = [ + "@com_google_googleapis//google/api:annotations_proto", + ], +) + +cc_proto_library( + name = "fetch_mode_cc_proto", + deps = [":fetch_mode_proto"], +) diff --git a/services/common/blob_fetch/fetch_mode.proto b/services/common/blob_fetch/fetch_mode.proto new file mode 100644 index 00000000..1f1b505f --- /dev/null +++ b/services/common/blob_fetch/fetch_mode.proto @@ -0,0 +1,31 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package privacy_sandbox.bidding_auction_servers.blob_fetch; + +// Note: this FetchMode enum will have to be placed in a common place. +// Specify how to fetch the code blob. +enum FetchMode { + // Fetch a single blob from an arbitrary url. + FETCH_MODE_URL = 0; + // Fetch all blobs from a specified bucket + // and rely on a default blob specification. + FETCH_MODE_BUCKET = 1; + // Fetch a blob from a local path. Dev-only. + FETCH_MODE_LOCAL = 2; +} diff --git a/services/common/chaffing/BUILD b/services/common/chaffing/BUILD new file mode 100644 index 00000000..c80135ce --- /dev/null +++ b/services/common/chaffing/BUILD @@ -0,0 +1,30 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//:__subpackages__"]) + +cc_library( + name = "transcoding_utils", + hdrs = ["transcoding_utils.h"], + deps = [ + "//api:bidding_auction_servers_cc_grpc_proto", + "@com_github_google_quiche//quiche:quiche_unstable_api", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@google_privacysandbox_servers_common//src/logger:request_context_impl", + ], +) diff --git a/services/common/chaffing/transcoding_utils.h b/services/common/chaffing/transcoding_utils.h new file mode 100644 index 00000000..9746a567 --- /dev/null +++ b/services/common/chaffing/transcoding_utils.h @@ -0,0 +1,126 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_COMMON_CHAFFING_TRANSCODING_UTILS_H_ +#define SERVICES_COMMON_CHAFFING_TRANSCODING_UTILS_H_ + +#include +#include +#include + +#include "api/bidding_auction_servers.grpc.pb.h" +#include "api/bidding_auction_servers.pb.h" +#include "quiche/common/quiche_data_reader.h" +#include "quiche/common/quiche_data_writer.h" +#include "src/logger/request_context_impl.h" + +namespace privacy_sandbox::bidding_auction_servers { + +inline constexpr int kVersionAndCompressionTypeSizeBytes = 1; +inline constexpr int kDataSizeBytes = 4; +inline constexpr int kTotalMetadataSizeBytes = + kVersionAndCompressionTypeSizeBytes + kDataSizeBytes; + +template +struct DecodedGetBidsPayload { + uint8_t version_and_compression_num; + uint32_t payload_length; + GetBidsProto get_bids_proto; +}; + +// Encodes a GetBids request payload in following format: +// - 1 byte containing: +// - 4 bits for the framing version (the format/structure of the payload) +// - 4 bits for the compression algorithm used to compress the payload +// - 4 bytes for the size of the data +// - X bytes of data (e.g. the payload) +// - Y bytes of padding +template +std::string EncodeGetBidsPayload(const GetBidsProto& raw_proto, + size_t padded_request_size = 0) { + const bool is_get_bids_proto = + std::is_base_of::value || + std::is_base_of::value; + static_assert( + is_get_bids_proto, + "raw_proto should be either a GetBids RawRequest or RawResponse"); + + std::string plaintext = raw_proto.SerializeAsString(); + int payload_size = kTotalMetadataSizeBytes; + payload_size += std::max(plaintext.length(), padded_request_size); + + // Create backing array for QuicheDataWriter and initialize the writer with + // it. + const int kEncodedDataSize = payload_size; + char encoded_payload[kEncodedDataSize]; + quiche::QuicheDataWriter writer(kEncodedDataSize, encoded_payload); + + // Write '0' byte for version and compression algorithm. + writer.WriteUInt8('\0'); + // Write the length of the payload using 4 bytes. + writer.WriteUInt32(plaintext.length()); + // Write the actual payload. + writer.WriteStringPiece(plaintext); + // Fill the rest of the array with padding. + writer.WritePadding(); + + PS_VLOG(5) << "Payload successfully encoded..."; + return std::string(encoded_payload, kEncodedDataSize); +} + +template +absl::StatusOr> DecodeGetBidsPayload( + absl::string_view encoded_payload) { + const bool is_get_bids_proto = + std::is_base_of::value || + std::is_base_of::value; + static_assert( + is_get_bids_proto, + "raw_proto should be either a GetBids RawRequest or RawResponse"); + + quiche::QuicheDataReader reader(encoded_payload); + uint8_t first_byte; + if (!reader.ReadUInt8(&first_byte)) { + return absl::InvalidArgumentError("Cannot read version/compression byte"); + } + + uint32_t payload_length; + if (!reader.ReadUInt32(&payload_length)) { + return absl::InvalidArgumentError("Cannot read payload length bytes"); + } + + absl::string_view payload; + if (!reader.ReadStringPiece(&payload, payload_length)) { + return absl::InvalidArgumentError("Cannot read payload"); + } + + GetBidsProto get_bids_proto; + if (!get_bids_proto.ParseFromString(payload)) { + return absl::InvalidArgumentError( + "Cannot parse proto from GetBids payload"); + } + + DecodedGetBidsPayload decoded_payload = { + .version_and_compression_num = first_byte, + .payload_length = payload_length, + .get_bids_proto = std::move(get_bids_proto)}; + + return decoded_payload; +} + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_COMMON_CHAFFING_TRANSCODING_UTILS_H_ diff --git a/services/common/clients/BUILD b/services/common/clients/BUILD index 84a27473..96ece25e 100644 --- a/services/common/clients/BUILD +++ b/services/common/clients/BUILD @@ -26,6 +26,7 @@ cc_library( linkstatic = True, deps = [ ":client_params_template", + "//services/common/clients/async_grpc:request_config", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/status", "@com_google_absl//absl/time", @@ -38,6 +39,7 @@ cc_library( # header only library for interface linkstatic = True, deps = [ + "//services/common/clients/async_grpc:request_config", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/status", diff --git a/services/common/clients/async_client.h b/services/common/clients/async_client.h index c744f63c..d565c07a 100644 --- a/services/common/clients/async_client.h +++ b/services/common/clients/async_client.h @@ -22,6 +22,7 @@ #include "absl/functional/any_invocable.h" #include "absl/status/statusor.h" #include "absl/time/time.h" +#include "services/common/clients/async_grpc/request_config.h" namespace privacy_sandbox::bidding_auction_servers { @@ -54,9 +55,10 @@ class AsyncClient { // Executes the inter service GRPC request asynchronously. virtual absl::Status ExecuteInternal( std::unique_ptr request, const RequestMetadata& metadata, - absl::AnyInvocable>) &&> + absl::AnyInvocable>, + ResponseMetadata) &&> on_done, - absl::Duration timeout) const { + absl::Duration timeout, RequestConfig request_config = {}) const { return absl::NotFoundError("Method not implemented."); } }; diff --git a/services/common/clients/async_grpc/BUILD b/services/common/clients/async_grpc/BUILD index c86f2691..1c361509 100644 --- a/services/common/clients/async_grpc/BUILD +++ b/services/common/clients/async_grpc/BUILD @@ -16,8 +16,16 @@ load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") package(default_visibility = ["//:__subpackages__"]) +cc_library( + name = "request_config", + hdrs = [ + "request_config.h", + ], +) + cc_library( name = "default_async_grpc_client", + srcs = ["default_async_grpc_client.cc"], hdrs = [ "default_async_grpc_client.h", ], @@ -38,9 +46,11 @@ cc_library( ), deps = [ ":grpc_client_utils", + ":request_config", "//api:bidding_auction_servers_cc_grpc_proto", "//api:bidding_auction_servers_cc_proto", "//services/common/clients:async_client", + "//services/common/constants:common_constants", "//services/common/encryption:crypto_client_factory", "//services/common/encryption:crypto_client_wrapper_interface", "//services/common/encryption:key_fetcher_factory", diff --git a/services/common/clients/async_grpc/bidding_async_grpc_client_stub_test.h b/services/common/clients/async_grpc/bidding_async_grpc_client_stub_test.h index c3d80687..867fa493 100644 --- a/services/common/clients/async_grpc/bidding_async_grpc_client_stub_test.h +++ b/services/common/clients/async_grpc/bidding_async_grpc_client_stub_test.h @@ -158,9 +158,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, CallsServerWithRequest) { auto status = class_under_test.ExecuteInternal( std::move(input_request_ptr), {}, [¬ification]( - absl::StatusOr> get_values_response) { - notification.Notify(); - }); + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { notification.Notify(); }); CHECK_OK(status); notification.WaitForNotification(); EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals( @@ -214,9 +213,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, CallsServerWithMetadata) { auto status = class_under_test.ExecuteInternal( std::make_unique(), sent_metadata, [¬ification]( - absl::StatusOr> get_values_response) { - notification.Notify(); - }); + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { notification.Notify(); }); CHECK_OK(status); notification.WaitForNotification(); @@ -267,7 +265,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, PassesStatusToCallback) { auto status = class_under_test.ExecuteInternal( std::move(input_request_ptr), {}, [¬ification]( - absl::StatusOr> get_values_response) { + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { EXPECT_EQ(get_values_response.status().code(), absl::StatusCode::kInvalidArgument); notification.Notify(); @@ -325,9 +324,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, CallsServerWithTimeout) { auto status = class_under_test.ExecuteInternal( std::move(input_request_ptr), {}, [¬ification]( - absl::StatusOr> get_values_response) { - notification.Notify(); - }, + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { notification.Notify(); }, timeout); CHECK_OK(status); notification.WaitForNotification(); @@ -377,7 +375,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, PassesResponseToCallback) { auto status = class_under_test.ExecuteInternal( std::move(input_request_ptr), {}, [¬ification, &expected_output]( - absl::StatusOr> get_values_response) { + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals( **get_values_response, expected_output)); notification.Notify(); @@ -426,7 +425,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, DoesNotExecuteCallbackOnSyncError) { auto status = class_under_test.ExecuteInternal( std::move(input_request_ptr), {}, [¬ification]( - absl::StatusOr> get_values_response) { + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { EXPECT_FALSE(get_values_response.ok()); notification.Notify(); }, @@ -476,7 +476,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, ExecutesCallbackOnTimeout) { auto status = class_under_test.ExecuteInternal( std::move(input_request_ptr), {}, [¬ification]( - absl::StatusOr> get_values_response) { + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { EXPECT_FALSE(get_values_response.ok()); notification.Notify(); }, diff --git a/services/common/clients/async_grpc/default_async_grpc_client.cc b/services/common/clients/async_grpc/default_async_grpc_client.cc new file mode 100644 index 00000000..9598f683 --- /dev/null +++ b/services/common/clients/async_grpc/default_async_grpc_client.cc @@ -0,0 +1,43 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/common/clients/async_grpc/default_async_grpc_client.h" + +#include + +namespace privacy_sandbox::bidding_auction_servers { + +std::shared_ptr CreateChannel( + absl::string_view server_addr, bool compression, bool secure, + absl::string_view grpc_arg_default_authority) { + std::shared_ptr creds = + secure ? grpc::SslCredentials(grpc::SslCredentialsOptions()) + : grpc::InsecureChannelCredentials(); + grpc::ChannelArguments args; + // Set max message size to 256 MB. + args.SetMaxSendMessageSize(256L * 1024L * 1024L); + args.SetMaxReceiveMessageSize(256L * 1024L * 1024L); + if (!grpc_arg_default_authority.empty() && + grpc_arg_default_authority != kIgnoredPlaceholderValue) { + args.SetString(GRPC_ARG_DEFAULT_AUTHORITY, + grpc_arg_default_authority.data()); + } + if (compression) { + // Set the default compression algorithm for the channel. + args.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP); + } + return grpc::CreateCustomChannel(server_addr.data(), creds, args); +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/clients/async_grpc/default_async_grpc_client.h b/services/common/clients/async_grpc/default_async_grpc_client.h index 8a476d19..c1ea4fb6 100644 --- a/services/common/clients/async_grpc/default_async_grpc_client.h +++ b/services/common/clients/async_grpc/default_async_grpc_client.h @@ -23,7 +23,9 @@ #include "absl/log/check.h" #include "services/common/clients/async_client.h" #include "services/common/clients/async_grpc/grpc_client_utils.h" +#include "services/common/clients/async_grpc/request_config.h" #include "services/common/clients/client_params.h" +#include "services/common/constants/common_constants.h" #include "services/common/encryption/crypto_client_wrapper_interface.h" #include "services/common/util/error_categories.h" #include "src/encryption/key_fetcher/key_fetcher_manager.h" @@ -35,7 +37,7 @@ namespace privacy_sandbox::bidding_auction_servers { using ::google::cmrt::sdk::public_key_service::v1::PublicKey; // This can be made configurable -inline constexpr absl::Duration max_timeout = absl::Milliseconds(60000); +inline constexpr absl::Duration kMaxClientTimeout = absl::Seconds(60); // This class acts as a template for a basic asynchronous grpc client. template raw_request, const RequestMetadata& metadata, - absl::AnyInvocable>) &&> + absl::AnyInvocable>, + ResponseMetadata) &&> on_done, - absl::Duration timeout = max_timeout) const override { + absl::Duration timeout = kMaxClientTimeout, + RequestConfig request_config = {}) const override { PS_VLOG(6) << "Raw request:\n" << raw_request->DebugString(); PS_VLOG(5) << "Encrypting request ..."; - auto secret_request = EncryptRequestWithHpke( - std::move(raw_request), *crypto_client_, *key_fetcher_manager_, - cloud_platform_); + + return EncryptPayloadAndSendRpc(raw_request->SerializeAsString(), metadata, + std::move(on_done), timeout, + request_config); + } + + protected: + using SecretRequest = std::pair>; + + absl::Status EncryptPayloadAndSendRpc( + const std::string& plaintext, const RequestMetadata& metadata, + absl::AnyInvocable>, + ResponseMetadata) &&> + on_done, + absl::Duration timeout, RequestConfig request_config = {}) const { + auto secret_request = EncryptRequestWithHpke( + plaintext, *crypto_client_, *key_fetcher_manager_, cloud_platform_); if (!secret_request.ok()) { PS_LOG(ERROR) << "Failed to encrypt the request: " << secret_request.status(); @@ -90,16 +108,13 @@ class DefaultAsyncGrpcClient auto params = std::make_unique>( - std::move(request), std::move(on_done), metadata); - params->SetDeadline(std::min(max_timeout, timeout)); + std::move(request), std::move(on_done), metadata, request_config); + params->SetDeadline(std::min(kMaxClientTimeout, timeout)); PS_VLOG(5) << "Sending RPC ..."; SendRpc(hpke_secret, params.release()); return absl::OkStatus(); } - protected: - using SecretRequest = std::pair>; - // Sends an asynchronous request via grpc. This method must be implemented // by classes implementing this interface. // @@ -150,24 +165,11 @@ class DefaultAsyncGrpcClient // server_addr: the URL or IP for the server DNS // compression: flag to enable gRPC level compression for the client. // Disabled by default. -inline std::shared_ptr CreateChannel( +std::shared_ptr CreateChannel( // Const string reference to prevent copies. string_view cannot be casted // to argument for grpc::CreateChannel. - absl::string_view server_addr, bool compression = false, - bool secure = true) { - std::shared_ptr creds = - secure ? grpc::SslCredentials(grpc::SslCredentialsOptions()) - : grpc::InsecureChannelCredentials(); - grpc::ChannelArguments args; - // Set max message size to 256 MB. - args.SetMaxSendMessageSize(256L * 1024L * 1024L); - args.SetMaxReceiveMessageSize(256L * 1024L * 1024L); - if (compression) { - // Set the default compression algorithm for the channel. - args.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP); - } - return grpc::CreateCustomChannel(server_addr.data(), creds, args); -} + absl::string_view server_addr, bool compression = false, bool secure = true, + absl::string_view grpc_arg_default_authority = ""); } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/clients/async_grpc/default_async_grpc_client_stub_test.h b/services/common/clients/async_grpc/default_async_grpc_client_stub_test.h index fa93a82b..1b2b98d9 100644 --- a/services/common/clients/async_grpc/default_async_grpc_client_stub_test.h +++ b/services/common/clients/async_grpc/default_async_grpc_client_stub_test.h @@ -113,6 +113,29 @@ void SetupMockCryptoClientWrapper(T request, .WillOnce(testing::Return(aead_decrypt_response)); } +void MockHpkeEncryptCall(const MockCryptoClientWrapper& crypto_client, + absl::string_view encoded_payload) { + google::cmrt::sdk::crypto_service::v1::HpkeEncryptResponse + hpke_encrypt_response; + hpke_encrypt_response.set_secret(kTestSecret); + hpke_encrypt_response.mutable_encrypted_data()->set_key_id(kTestKeyId); + hpke_encrypt_response.mutable_encrypted_data()->set_ciphertext( + encoded_payload); + EXPECT_CALL(crypto_client, HpkeEncrypt) + .Times(testing::AnyNumber()) + .WillOnce(testing::Return(hpke_encrypt_response)); +} + +void MockAeadDecryptCall(const MockCryptoClientWrapper& crypto_client, + absl::string_view encoded_payload) { + google::cmrt::sdk::crypto_service::v1::AeadDecryptResponse + aead_decrypt_response; + aead_decrypt_response.set_payload(encoded_payload); + EXPECT_CALL(crypto_client, AeadDecrypt) + .Times(AnyNumber()) + .WillOnce(testing::Return(aead_decrypt_response)); +} + TYPED_TEST_P(AsyncGrpcClientStubTest, CallsServerWithRequest) { using ServiceThread = typename TypeParam::ServiceThreadType; using Request = typename TypeParam::RequestType; @@ -150,9 +173,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, CallsServerWithRequest) { auto status = class_under_test.ExecuteInternal( std::move(input_request_ptr), {}, [¬ification]( - absl::StatusOr> get_values_response) { - notification.Notify(); - }); + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { notification.Notify(); }); CHECK_OK(status); notification.WaitForNotification(); EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals( @@ -197,16 +219,16 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, CallsServerWithMetadata) { MockCryptoClientWrapper crypto_client; SetupMockCryptoClientWrapper(raw_request, crypto_client); auto key_fetcher_manager = - CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr); + CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ + nullptr); TestClient class_under_test(key_fetcher_manager.get(), &crypto_client, client_config); absl::Notification notification; auto status = class_under_test.ExecuteInternal( std::make_unique(), sent_metadata, [¬ification]( - absl::StatusOr> get_values_response) { - notification.Notify(); - }); + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { notification.Notify(); }); CHECK_OK(status); notification.WaitForNotification(); @@ -247,7 +269,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, PassesStatusToCallback) { MockCryptoClientWrapper crypto_client; SetupMockCryptoClientWrapper(raw_request, crypto_client); auto key_fetcher_manager = - CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr); + CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ + nullptr); TestClient class_under_test(key_fetcher_manager.get(), &crypto_client, client_config); absl::Notification notification; @@ -255,7 +278,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, PassesStatusToCallback) { auto status = class_under_test.ExecuteInternal( std::move(input_request_ptr), {}, [¬ification]( - absl::StatusOr> get_values_response) { + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { EXPECT_EQ(get_values_response.status().code(), absl::StatusCode::kInvalidArgument); notification.Notify(); @@ -305,15 +329,15 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, CallsServerWithTimeout) { MockCryptoClientWrapper crypto_client; SetupMockCryptoClientWrapper(raw_request, crypto_client); auto key_fetcher_manager = - CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr); + CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ + nullptr); TestClient class_under_test(key_fetcher_manager.get(), &crypto_client, client_config); auto status = class_under_test.ExecuteInternal( std::move(input_request_ptr), {}, [¬ification]( - absl::StatusOr> get_values_response) { - notification.Notify(); - }, + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { notification.Notify(); }, timeout); CHECK_OK(status); notification.WaitForNotification(); @@ -352,7 +376,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, PassesResponseToCallback) { MockCryptoClientWrapper crypto_client; SetupMockCryptoClientWrapper(raw_request, crypto_client); auto key_fetcher_manager = - CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr); + CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ + nullptr); TestClient class_under_test(key_fetcher_manager.get(), &crypto_client, client_config); absl::Notification notification; @@ -361,7 +386,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, PassesResponseToCallback) { auto status = class_under_test.ExecuteInternal( std::move(input_request_ptr), {}, [¬ification, &expected_output]( - absl::StatusOr> get_values_response) { + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals( **get_values_response, expected_output)); notification.Notify(); @@ -399,7 +425,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, DoesNotExecuteCallbackOnSyncError) { MockCryptoClientWrapper crypto_client; SetupMockCryptoClientError(raw_request, crypto_client); auto key_fetcher_manager = - CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr); + CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ + nullptr); TestClient class_under_test(key_fetcher_manager.get(), &crypto_client, client_config); absl::Notification notification; @@ -408,7 +435,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, DoesNotExecuteCallbackOnSyncError) { auto status = class_under_test.ExecuteInternal( std::move(input_request_ptr), {}, [¬ification]( - absl::StatusOr> get_values_response) { + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { EXPECT_FALSE(get_values_response.ok()); notification.Notify(); }, @@ -447,7 +475,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, ExecutesCallbackOnTimeout) { MockCryptoClientWrapper crypto_client; SetupMockCryptoClientWrapper(raw_request, crypto_client); auto key_fetcher_manager = - CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr); + CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ + nullptr); TestClient class_under_test(key_fetcher_manager.get(), &crypto_client, client_config); absl::Notification notification; @@ -456,7 +485,8 @@ TYPED_TEST_P(AsyncGrpcClientStubTest, ExecutesCallbackOnTimeout) { auto status = class_under_test.ExecuteInternal( std::move(input_request_ptr), {}, [¬ification]( - absl::StatusOr> get_values_response) { + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { EXPECT_FALSE(get_values_response.ok()); notification.Notify(); }, diff --git a/services/common/clients/async_grpc/default_async_grpc_client_test.cc b/services/common/clients/async_grpc/default_async_grpc_client_test.cc index c5a8ad14..7b6a7adc 100644 --- a/services/common/clients/async_grpc/default_async_grpc_client_test.cc +++ b/services/common/clients/async_grpc/default_async_grpc_client_test.cc @@ -166,7 +166,8 @@ TEST(TestDefaultAsyncGrpcClient, SendsMessageWithCorrectParams) { auto status = client.ExecuteInternal( std::make_unique(raw_request), metadata, - [](absl::StatusOr> result) {}, + [](absl::StatusOr> result, + ResponseMetadata response_metadata) {}, timeout_ms); CHECK_OK(status); notification.WaitForNotification(); diff --git a/services/common/clients/async_grpc/grpc_client_utils.h b/services/common/clients/async_grpc/grpc_client_utils.h index 6a4bfd7d..c0a9f1b1 100644 --- a/services/common/clients/async_grpc/grpc_client_utils.h +++ b/services/common/clients/async_grpc/grpc_client_utils.h @@ -30,14 +30,12 @@ namespace privacy_sandbox::bidding_auction_servers { -template +template absl::StatusOr>> EncryptRequestWithHpke( - std::unique_ptr raw_request, - CryptoClientWrapperInterface& crypto_client, + const std::string& plaintext, CryptoClientWrapperInterface& crypto_client, server_common::KeyFetcherManagerInterface& key_fetcher_manager, server_common::CloudPlatform cloud_platform) { - std::string plaintext = raw_request->SerializeAsString(); PS_ASSIGN_OR_RETURN(HpkeMessage encrypted_request, HpkeEncrypt(plaintext, crypto_client, key_fetcher_manager, cloud_platform)); diff --git a/services/common/clients/async_grpc/request_config.h b/services/common/clients/async_grpc/request_config.h new file mode 100644 index 00000000..124305ba --- /dev/null +++ b/services/common/clients/async_grpc/request_config.h @@ -0,0 +1,35 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_COMMON_CLIENTS_ASYNC_GRPC_REQUEST_CONFIG_ +#define SERVICES_COMMON_CLIENTS_ASYNC_GRPC_REQUEST_CONFIG_ + +#include + +namespace privacy_sandbox::bidding_auction_servers { + +// This can be extended such that each service can have its own specific +// RequestConfig object. That struct type can be added to client_params.h's +// template to avoid casting issues +struct RequestConfig { + size_t chaff_request_size = 0; +}; + +struct ResponseMetadata { + size_t response_size = 0; +}; + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_COMMON_CLIENTS_ASYNC_GRPC_REQUEST_CONFIG_ diff --git a/services/common/clients/auction_server/BUILD b/services/common/clients/auction_server/BUILD index d94a43ea..120c08e4 100644 --- a/services/common/clients/auction_server/BUILD +++ b/services/common/clients/auction_server/BUILD @@ -54,6 +54,9 @@ cc_test( srcs = [ "scoring_async_client_test.cc", ], + tags = [ + "flaky", + ], deps = [ ":async_client", "//services/common/encryption:mock_crypto_client_wrapper", diff --git a/services/common/clients/auction_server/scoring_async_client.cc b/services/common/clients/auction_server/scoring_async_client.cc index 97a04de9..290a45e3 100644 --- a/services/common/clients/auction_server/scoring_async_client.cc +++ b/services/common/clients/auction_server/scoring_async_client.cc @@ -26,9 +26,9 @@ ScoringAsyncGrpcClient::ScoringAsyncGrpcClient( CryptoClientWrapperInterface* crypto_client, const AuctionServiceClientConfig& client_config) : DefaultAsyncGrpcClient(key_fetcher_manager, crypto_client) { - stub_ = Auction::NewStub(CreateChannel(client_config.server_addr, - client_config.compression, - client_config.secure_client)); + stub_ = Auction::NewStub(CreateChannel( + client_config.server_addr, client_config.compression, + client_config.secure_client, client_config.grpc_arg_default_authority)); } void ScoringAsyncGrpcClient::SendRpc( diff --git a/services/common/clients/auction_server/scoring_async_client.h b/services/common/clients/auction_server/scoring_async_client.h index 42e3950c..a8d6ba7c 100644 --- a/services/common/clients/auction_server/scoring_async_client.h +++ b/services/common/clients/auction_server/scoring_async_client.h @@ -37,6 +37,7 @@ struct AuctionServiceClientConfig { std::string server_addr; bool compression = false; bool secure_client = true; + std::string grpc_arg_default_authority = ""; }; // This class is an async grpc client for the Fledge Auction (Scoring) Service. diff --git a/services/common/clients/auction_server/scoring_async_client_test.cc b/services/common/clients/auction_server/scoring_async_client_test.cc index ebc8c97a..46c40997 100644 --- a/services/common/clients/auction_server/scoring_async_client_test.cc +++ b/services/common/clients/auction_server/scoring_async_client_test.cc @@ -95,7 +95,8 @@ TEST(ScoringAsyncClientTest, CallServerWithEncryptionEnabled_Success) { std::make_unique(std::move(request)), {}, [&](absl::StatusOr> - callback_response) { + callback_response, + ResponseMetadata response_metadata) { std::string difference; google::protobuf::util::MessageDifferencer differencer; differencer.ReportDifferencesToString(&difference); @@ -144,7 +145,8 @@ TEST(ScoringAsyncClientTest, absl::Status execute_result = class_under_test.ExecuteInternal( std::make_unique(), {}, [](absl::StatusOr> - callback_response) { // Never called + callback_response, + ResponseMetadata response_metadata) { // Never called }); EXPECT_FALSE(execute_result.ok()); } @@ -204,7 +206,8 @@ TEST(ScoringAsyncClientTest, std::make_unique(std::move(request)), {}, [&](absl::StatusOr> - callback_response) { + callback_response, + ResponseMetadata response_metadata) { EXPECT_FALSE(callback_response.ok()); notification.Notify(); }); diff --git a/services/common/clients/bidding_server/bidding_async_client.h b/services/common/clients/bidding_server/bidding_async_client.h index 88452169..2b16cff1 100644 --- a/services/common/clients/bidding_server/bidding_async_client.h +++ b/services/common/clients/bidding_server/bidding_async_client.h @@ -46,6 +46,7 @@ struct BiddingServiceClientConfig { bool compression = false; bool secure_client = true; bool is_pas_enabled = false; + std::string grpc_arg_default_authority = ""; }; // This class is an async grpc client for the Fledge Bidding Service. diff --git a/services/common/clients/buyer_frontend_server/BUILD b/services/common/clients/buyer_frontend_server/BUILD index 409f3fd3..992e420e 100644 --- a/services/common/clients/buyer_frontend_server/BUILD +++ b/services/common/clients/buyer_frontend_server/BUILD @@ -24,7 +24,9 @@ cc_library( hdrs = ["buyer_frontend_async_client.h"], deps = [ "//api:bidding_auction_servers_cc_grpc_proto", + "//services/common/chaffing:transcoding_utils", "//services/common/clients/async_grpc:default_async_grpc_client", + "@com_github_google_quiche//quiche:quiche_unstable_api", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/time", @@ -44,9 +46,12 @@ cc_test( ], deps = [ ":buyer_frontend_async_client", + "//services/common/chaffing:transcoding_utils", "//services/common/clients/async_grpc:default_async_grpc_client_integration_test_utils", "//services/common/test:mocks", "//services/common/test:random", + "//services/seller_frontend_service:runtime_flags", + "@com_github_google_quiche//quiche:quiche_unstable_api", "@com_github_grpc_grpc//:grpc++", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", diff --git a/services/common/clients/buyer_frontend_server/buyer_frontend_async_client.cc b/services/common/clients/buyer_frontend_server/buyer_frontend_async_client.cc index 68cdd84b..17167fb1 100644 --- a/services/common/clients/buyer_frontend_server/buyer_frontend_async_client.cc +++ b/services/common/clients/buyer_frontend_server/buyer_frontend_async_client.cc @@ -14,6 +14,9 @@ #include "services/common/clients/buyer_frontend_server/buyer_frontend_async_client.h" +#include + +#include "services/common/chaffing/transcoding_utils.h" #include "src/public/cpio/interface/crypto_client/crypto_client_interface.h" namespace privacy_sandbox::bidding_auction_servers { @@ -27,7 +30,8 @@ BuyerFrontEndAsyncGrpcClient::BuyerFrontEndAsyncGrpcClient( std::unique_ptr stub) : DefaultAsyncGrpcClient(key_fetcher_manager, crypto_client, client_config.cloud_platform), - stub_(std::move(stub)) { + stub_(std::move(stub)), + chaffing_enabled_(client_config.chaffing_enabled) { if (!stub_) { stub_ = BuyerFrontEnd::NewStub(CreateChannel(client_config.server_addr, client_config.compression, @@ -35,6 +39,83 @@ BuyerFrontEndAsyncGrpcClient::BuyerFrontEndAsyncGrpcClient( } } +absl::Status BuyerFrontEndAsyncGrpcClient::ExecuteInternal( + std::unique_ptr raw_request, + const RequestMetadata& metadata, + absl::AnyInvocable>, + ResponseMetadata) &&> + on_done, + absl::Duration timeout, RequestConfig request_config) const { + if (!chaffing_enabled_) { + return DefaultAsyncGrpcClient::ExecuteInternal(std::move(raw_request), + metadata, std::move(on_done), + timeout, request_config); + } + + // If chaffing is enabled, we encode requests according to the new request + // format. + PS_VLOG(6) << "Raw request:\n" << raw_request->DebugString(); + std::string encoded_req_payload = + EncodeGetBidsPayload(*raw_request, request_config.chaff_request_size); + + return EncryptPayloadAndSendRpc(encoded_req_payload, metadata, + std::move(on_done), timeout, request_config); +} + +void BuyerFrontEndAsyncGrpcClient::OnGetBidsDoneChaffingDisabled( + absl::string_view decrypted_payload, const grpc::Status& status, + BuyerFrontendRawClientParams* params) const { + std::unique_ptr raw_response = + std::make_unique(); + if (!raw_response->ParseFromString(decrypted_payload)) { + PS_LOG(ERROR) << "Failed to parse proto from decrypted response"; + params->OnDone(grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, + raw_response->DebugString())); + return; + } + + PS_VLOG(6) << "Decryption/decoding of response succeeded: " + << raw_response->DebugString(); + + params->SetRawResponse(std::move(raw_response)); + PS_VLOG(6) << "Returning the decrypted response via callback"; + params->OnDone(status); +} + +void BuyerFrontEndAsyncGrpcClient::OnGetBidsDoneChaffingEnabled( + absl::string_view decrypted_payload, const grpc::Status& status, + BuyerFrontendRawClientParams* params) const { + if (params->RequestConfig().chaff_request_size > 0) { + // If the request was chaff, return an empty response up to + // select_ad_reactor regardless of the actual response from the BFE. + params->SetRawResponse( + std::make_unique()); + params->OnDone(grpc::Status()); // Defaults to 'StatusCode::OK'. + return; + } + + std::unique_ptr raw_response = + std::make_unique(); + // If the request wasn't chaff, the response will have no padding. So + // everything past the metadata bytes are the actual payload. + absl::string_view payload( + &decrypted_payload[kTotalMetadataSizeBytes], + decrypted_payload.length() - kTotalMetadataSizeBytes); + if (!raw_response->ParseFromString(payload)) { + PS_LOG(ERROR) << "Failed to parse proto from decrypted response: " + << decrypted_payload; + params->OnDone( + grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, + "Failed to parse proto from decrypted response")); + return; + } + + params->SetRawResponse(std::move(raw_response)); + PS_VLOG(6) << "Returning the decrypted response via callback"; + params->OnDone(status); +} + void BuyerFrontEndAsyncGrpcClient::SendRpc( const std::string& hpke_secret, RawClientParamsResponseRef()); - if (!decrypted_response.ok()) { + // For metric purposes in the reactor, note the response size. + params->SetResponseMetadata( + {.response_size = params->ResponseRef()->ByteSizeLong()}); + + PS_VLOG(6) << "Decrypting the response ..."; + absl::StatusOr< + google::cmrt::sdk::crypto_service::v1::AeadDecryptResponse> + decrypt_response = crypto_client_->AeadDecrypt( + params->ResponseRef()->response_ciphertext(), hpke_secret); + if (!decrypt_response.ok()) { PS_LOG(ERROR) << "BuyerFrontEndAsyncGrpcClient Failed to decrypt response"; params->OnDone(grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, - decrypted_response.status().ToString())); - return; + decrypt_response.status().ToString())); } - params->SetRawResponse(*std::move(decrypted_response)); - PS_VLOG(6) << "Returning the decrypted response via callback"; - params->OnDone(status); + absl::string_view decrypted_payload = decrypt_response->payload(); + // Look at the first byte of the decrypted payload to determine whether + // the BFE responded in the new or old request format. In the old + // format, the decrypted payload is just a serialized proto message, + // which will never have the null terminator as the first character. In + // the new format, the decrypted payload's first byte is guaranteed to + // be zero for now (it will be non-zero when we change the version + // number or enable compression). + bool is_new_format_bfe_response = + !decrypted_payload.empty() && decrypted_payload.front() == '\0'; + + if (is_new_format_bfe_response && chaffing_enabled_) { + OnGetBidsDoneChaffingEnabled(decrypted_payload, status, params); + } else { + OnGetBidsDoneChaffingDisabled(decrypted_payload, status, params); + } }); } diff --git a/services/common/clients/buyer_frontend_server/buyer_frontend_async_client.h b/services/common/clients/buyer_frontend_server/buyer_frontend_async_client.h index e27d4669..aeccb13b 100644 --- a/services/common/clients/buyer_frontend_server/buyer_frontend_async_client.h +++ b/services/common/clients/buyer_frontend_server/buyer_frontend_async_client.h @@ -15,6 +15,7 @@ #ifndef FLEDGE_SERVICES_COMMON_CLIENTS_BUYER_FRONTEND_ASYNC_CLIENT_H_ #define FLEDGE_SERVICES_COMMON_CLIENTS_BUYER_FRONTEND_ASYNC_CLIENT_H_ +#include #include #include #include @@ -23,8 +24,10 @@ #include "absl/time/time.h" #include "api/bidding_auction_servers.grpc.pb.h" #include "api/bidding_auction_servers.pb.h" +#include "quiche/common/quiche_data_writer.h" #include "services/common/clients/async_client.h" #include "services/common/clients/async_grpc/default_async_grpc_client.h" +#include "services/common/clients/async_grpc/request_config.h" #include "src/encryption/key_fetcher/key_fetcher_manager.h" namespace privacy_sandbox::bidding_auction_servers { @@ -33,11 +36,16 @@ using BuyerFrontEndAsyncClient = GetBidsRequest::GetBidsRawRequest, GetBidsResponse::GetBidsRawResponse>; +using BuyerFrontendRawClientParams = + RawClientParams; + struct BuyerServiceClientConfig { std::string server_addr; bool compression = false; bool secure_client = true; server_common::CloudPlatform cloud_platform; + bool chaffing_enabled = false; }; // This class is an async grpc client for Fledge Buyer FrontEnd Service. @@ -53,6 +61,16 @@ class BuyerFrontEndAsyncGrpcClient const BuyerServiceClientConfig& client_config, std::unique_ptr stub = nullptr); + absl::Status ExecuteInternal( + std::unique_ptr raw_request, + const RequestMetadata& metadata, + absl::AnyInvocable>, + ResponseMetadata) &&> + on_done, + absl::Duration timeout = kMaxClientTimeout, + RequestConfig request_config = {}) const override; + protected: // Sends an asynchronous request via grpc to the Buyer FrontEnd Service. // @@ -63,7 +81,22 @@ class BuyerFrontEndAsyncGrpcClient GetBidsResponse::GetBidsRawResponse>* params) const override; + // Decodes/Parses GetBidsResponses as per the old response format. + void OnGetBidsDoneChaffingDisabled( + absl::string_view decrypted_payload, const grpc::Status& status, + BuyerFrontendRawClientParams* params) const; + // Decodes/Parses GetBidsResponses as per the new response format. + // See the documentation on EncodeGetBidsPayload() in transcoding_utils.h + // for the format. + void OnGetBidsDoneChaffingEnabled(absl::string_view decrypted_payload, + const grpc::Status& status, + BuyerFrontendRawClientParams* params) const; + std::unique_ptr stub_; + + // This flag is overloaded in this class and decides whether chaffing as well + // as encoding/decoding requests in the new SFE <> BFE format is enabled. + bool chaffing_enabled_; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/clients/buyer_frontend_server/buyer_frontend_async_client_factory.cc b/services/common/clients/buyer_frontend_server/buyer_frontend_async_client_factory.cc index 100ee92a..04898036 100644 --- a/services/common/clients/buyer_frontend_server/buyer_frontend_async_client_factory.cc +++ b/services/common/clients/buyer_frontend_server/buyer_frontend_async_client_factory.cc @@ -20,7 +20,6 @@ #include "api/bidding_auction_servers.grpc.pb.h" #include "services/common/clients/async_client.h" #include "services/common/clients/buyer_frontend_server/buyer_frontend_async_client.h" -#include "services/common/concurrent/static_local_cache.h" namespace privacy_sandbox::bidding_auction_servers { @@ -56,15 +55,27 @@ BuyerFrontEndAsyncClientFactory::BuyerFrontEndAsyncClientFactory( static_client_map->try_emplace(ig_owner, bfe_client_ptr); } } - client_cache_ = std::make_unique< - StaticLocalCache>( - std::move(static_client_map)); + client_cache_ = std::move(static_client_map); + + for (const auto& [ig_owner, client] : *client_cache_) { + entries_.push_back({ig_owner, client}); + } } std::shared_ptr BuyerFrontEndAsyncClientFactory::Get(absl::string_view ig_owner) const { std::string ig_owner_str = absl::StrCat(ig_owner); - return client_cache_->LookUp(ig_owner_str); + if (auto it = client_cache_->find(ig_owner_str); it != client_cache_->end()) { + return it->second; + } + + return nullptr; +} + +std::vector>> +BuyerFrontEndAsyncClientFactory::Entries() const { + return entries_; } } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/clients/buyer_frontend_server/buyer_frontend_async_client_factory.h b/services/common/clients/buyer_frontend_server/buyer_frontend_async_client_factory.h index 7d5d7d81..2a9bfe03 100644 --- a/services/common/clients/buyer_frontend_server/buyer_frontend_async_client_factory.h +++ b/services/common/clients/buyer_frontend_server/buyer_frontend_async_client_factory.h @@ -17,6 +17,7 @@ #include #include +#include #include #include "absl/container/flat_hash_map.h" @@ -60,10 +61,20 @@ class BuyerFrontEndAsyncClientFactory std::shared_ptr Get( absl::string_view ig_owner) const override; + // Returns a list of all pairs contained by the + // factory. + std::vector>> + Entries() const override; + private: - std::unique_ptr< - LocalCache>> + std::unique_ptr>> client_cache_; + + std::vector>> + entries_; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/clients/buyer_frontend_server/buyer_frontend_async_grpc_client_stub_test.cc b/services/common/clients/buyer_frontend_server/buyer_frontend_async_grpc_client_stub_test.cc index 6b65599a..d08b753b 100644 --- a/services/common/clients/buyer_frontend_server/buyer_frontend_async_grpc_client_stub_test.cc +++ b/services/common/clients/buyer_frontend_server/buyer_frontend_async_grpc_client_stub_test.cc @@ -14,10 +14,13 @@ #include "absl/synchronization/notification.h" #include "gtest/gtest.h" +#include "quiche/common/quiche_data_reader.h" +#include "services/common/chaffing/transcoding_utils.h" #include "services/common/clients/async_grpc/default_async_grpc_client_stub_test.h" #include "services/common/clients/buyer_frontend_server/buyer_frontend_async_client.h" #include "services/common/test/mocks.h" #include "services/common/test/random.h" +#include "services/seller_frontend_service/runtime_flags.h" namespace privacy_sandbox::bidding_auction_servers { namespace { @@ -34,5 +37,195 @@ using BuyerFrontEndImplementationType = INSTANTIATE_TYPED_TEST_SUITE_P(BuyerFrontEndAsyncGrpcClientStubTest, AsyncGrpcClientStubTest, BuyerFrontEndImplementationType); + +TEST(BuyerFrontEndAsyncGrpcClientStubTest, + ChaffingEnabled_VerifyChaffRequestFormat) { + GetBidsResponse mock_bfe_response; + mock_bfe_response.set_response_ciphertext("sample_ciphertext"); + + // Hold onto a copy of the ciphertext that the BFE receives; we validate its + // encoding below. + std::string bfe_received_request_ciphertext; + + // Set up fake BFE on local thread. + auto dummy_service_thread_ = std::make_unique( + [&bfe_received_request_ciphertext, &mock_bfe_response]( + grpc::CallbackServerContext* context, const GetBidsRequest* request, + GetBidsResponse* response) { + bfe_received_request_ciphertext = request->request_ciphertext(); + response->CopyFrom(mock_bfe_response); + auto reactor = context->DefaultReactor(); + reactor->Finish(grpc::Status::OK); + return reactor; + }); + BuyerServiceClientConfig client_config = { + .server_addr = dummy_service_thread_->GetServerAddr(), + .chaffing_enabled = true}; + + TrustedServersConfigClient config_client({}); + config_client.SetFlagForTest(kTrue, TEST_MODE); + + GetBidsRequest::GetBidsRawRequest raw_request; + raw_request.set_is_chaff(true); + + RequestConfig request_config = {.chaff_request_size = 100}; + + MockCryptoClientWrapper crypto_client; + // Mock the HpkeEncrypt() call on the crypto client. + std::string encoded_payload = + EncodeGetBidsPayload(raw_request, request_config.chaff_request_size); + MockHpkeEncryptCall(crypto_client, encoded_payload); + // Mock the AeadDecrypt() call on the crypto client. + GetBidsResponse::GetBidsRawResponse mock_bfe_raw_response; + MockAeadDecryptCall(crypto_client, + EncodeGetBidsPayload(mock_bfe_raw_response, 50)); + + // Set up the BuyerFrontEndAsyncGrpcClient for the test. + auto key_fetcher_manager = + CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr); + BuyerFrontEndAsyncGrpcClient client(key_fetcher_manager.get(), &crypto_client, + client_config); + + // Run the test. + absl::Notification notification; + auto status = client.ExecuteInternal( + std::make_unique(raw_request), {}, + [¬ification, &mock_bfe_response]( + absl::StatusOr> + response, + ResponseMetadata response_metadata) { + // Verify that response_size in the ResponseMetadata correctly is + // populated correctly. It should match the mock_bfe_response we had the + // dummy thread/reactor return. + EXPECT_EQ(response_metadata.response_size, + mock_bfe_response.ByteSizeLong()); + // We pass an empty GetBidsRawResponse up to the reactor for chaff + // requests. + EXPECT_EQ((*response)->ByteSizeLong(), 0); + + notification.Notify(); + }, + kMaxClientTimeout, request_config); + CHECK_OK(status); + notification.WaitForNotification(); + + // Decode the GetBidsRequest received by our mock BFE and verify it matches + // the request we sent. + absl::StatusOr> + received_request = + DecodeGetBidsPayload( + bfe_received_request_ciphertext); + // Version and compression byte is 0 for now. + EXPECT_EQ(received_request->version_and_compression_num, 0); + EXPECT_EQ(received_request->payload_length, raw_request.ByteSizeLong()); + + std::string get_bids_raw_req_diff; + google::protobuf::util::MessageDifferencer get_bids_raw_req_differencer; + get_bids_raw_req_differencer.ReportDifferencesToString( + &get_bids_raw_req_diff); + EXPECT_TRUE(get_bids_raw_req_differencer.Compare( + received_request->get_bids_proto, raw_request)) + << "\nActual:\n" + << received_request->get_bids_proto.DebugString() << "\n\nExpected:\n" + << raw_request.DebugString() << "\n\nDifference:\n" + << get_bids_raw_req_diff; +} + +TEST(BuyerFrontEndAsyncGrpcClientStubTest, + ChaffingEnabled_VerifyNonChaffRequestFormat) { + // Create an encoded GetBidsResponse for our mock BFE to return. + AdWithBid ad_with_bid; + ad_with_bid.set_render("foo"); + GetBidsResponse::GetBidsRawResponse mock_bfe_raw_response; + mock_bfe_raw_response.mutable_bids()->Add(std::move(ad_with_bid)); + GetBidsResponse mock_bfe_response; + mock_bfe_response.set_response_ciphertext( + EncodeGetBidsPayload(mock_bfe_raw_response)); + + // Hold onto a copy of the ciphertext that the BFE receives; we validate its + // encoding below. + std::string bfe_received_request_ciphertext; + + // Set up fake BFE on local thread. + auto dummy_service_thread_ = std::make_unique( + [&bfe_received_request_ciphertext, &mock_bfe_response]( + grpc::CallbackServerContext* context, const GetBidsRequest* request, + GetBidsResponse* response) { + bfe_received_request_ciphertext = request->request_ciphertext(); + response->CopyFrom(mock_bfe_response); + auto reactor = context->DefaultReactor(); + reactor->Finish(grpc::Status::OK); + return reactor; + }); + BuyerServiceClientConfig client_config = { + .server_addr = dummy_service_thread_->GetServerAddr(), + .chaffing_enabled = true}; + + TrustedServersConfigClient config_client({}); + config_client.SetFlagForTest(kTrue, TEST_MODE); + + GetBidsRequest::GetBidsRawRequest raw_request; + raw_request.set_is_chaff(false); + + RequestConfig request_config = {.chaff_request_size = 0}; + + MockCryptoClientWrapper crypto_client; + // Mock the HpkeEncrypt() call on the crypto client. + MockHpkeEncryptCall(crypto_client, EncodeGetBidsPayload(raw_request)); + // Mock the AeadDecrypt() call on the crypto client. + MockAeadDecryptCall(crypto_client, + EncodeGetBidsPayload(mock_bfe_raw_response)); + + // Set up the BuyerFrontEndAsyncGrpcClient for the test. + auto key_fetcher_manager = + CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr); + BuyerFrontEndAsyncGrpcClient client(key_fetcher_manager.get(), &crypto_client, + client_config); + + // Run the test. + absl::Notification notification; + auto status = client.ExecuteInternal( + std::make_unique(raw_request), {}, + [¬ification, &mock_bfe_response, &mock_bfe_raw_response]( + absl::StatusOr> + response, + ResponseMetadata response_metadata) { + // Verify that response_size in the ResponseMetadata correctly is + // populated correctly. It should match the mock_bfe_response we had the + // dummy thread/reactor return. + EXPECT_EQ(response_metadata.response_size, + mock_bfe_response.ByteSizeLong()); + // We pass an empty proto up to the reactor for chaff requests. + EXPECT_EQ((*response)->ByteSizeLong(), + mock_bfe_raw_response.ByteSizeLong()); + + notification.Notify(); + }, + kMaxClientTimeout, request_config); + CHECK_OK(status); + notification.WaitForNotification(); + + // Decode the GetBidsRequest received by our mock BFE and verify it matches + // the request we sent. + absl::StatusOr> + received_request = + DecodeGetBidsPayload( + bfe_received_request_ciphertext); + // Version and compression byte is 0 for now. + EXPECT_EQ(received_request->version_and_compression_num, 0); + EXPECT_EQ(received_request->payload_length, raw_request.ByteSizeLong()); + + std::string get_bids_raw_req_diff; + google::protobuf::util::MessageDifferencer get_bids_raw_req_differencer; + get_bids_raw_req_differencer.ReportDifferencesToString( + &get_bids_raw_req_diff); + EXPECT_TRUE(get_bids_raw_req_differencer.Compare( + received_request->get_bids_proto, raw_request)) + << "\nActual:\n" + << received_request->get_bids_proto.DebugString() << "\n\nExpected:\n" + << raw_request.DebugString() << "\n\nDifference:\n" + << get_bids_raw_req_diff; +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/clients/client_factory.h b/services/common/clients/client_factory.h index f95e839c..f855d3fc 100644 --- a/services/common/clients/client_factory.h +++ b/services/common/clients/client_factory.h @@ -16,6 +16,8 @@ #define FLEDGE_SERVICES_COMMON_CLIENTS_CLIENT_FACTORY_H_ #include +#include +#include namespace privacy_sandbox::bidding_auction_servers { @@ -32,6 +34,10 @@ class ClientFactory { // client_key: This is a value for the key based on which the different // client objects are differentiated. virtual std::shared_ptr Get(ClientKey client_key) const = 0; + + // Returns a pair of all entries contained by the factory. + virtual std::vector>> + Entries() const = 0; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/clients/client_params.h b/services/common/clients/client_params.h index 04b73635..21c1b388 100644 --- a/services/common/clients/client_params.h +++ b/services/common/clients/client_params.h @@ -25,6 +25,7 @@ #include "absl/functional/any_invocable.h" #include "absl/status/statusor.h" #include "absl/time/time.h" +#include "services/common/clients/async_grpc/request_config.h" #include "src/logger/request_context_logger.h" namespace privacy_sandbox::bidding_auction_servers { @@ -52,6 +53,7 @@ class ClientParams { request_ = std::move(request); callback_ = std::move(callback); response_ = std::make_unique(); + for (const auto& it : metadata) { context_.AddMetadata(it.first, it.second); } @@ -113,12 +115,15 @@ class RawClientParams { // callback - Final callback executed with the response when RPC is finished. explicit RawClientParams( std::unique_ptr request, - absl::AnyInvocable>) &&> + absl::AnyInvocable>, + ResponseMetadata) &&> callback, - const absl::flat_hash_map& metadata = {}) { + const absl::flat_hash_map& metadata = {}, + RequestConfig request_config = {}) { request_ = std::move(request); raw_callback_ = std::move(callback); response_ = std::make_unique(); + request_config_ = request_config; for (const auto& it : metadata) { context_.AddMetadata(it.first, it.second); } @@ -128,6 +133,9 @@ class RawClientParams { RawClientParams(const RawClientParams&) = delete; RawClientParams& operator=(const RawClientParams&) = delete; + // Allows access to request config param by gRPC + const RequestConfig RequestConfig() { return request_config_; } + // Allows access to request param by gRPC Request* RequestRef() { return request_.get(); } @@ -137,6 +145,10 @@ class RawClientParams { // Allows access to response param by gRPC Response* ResponseRef() { return response_.get(); } + void SetResponseMetadata(ResponseMetadata response_metadata) { + response_metadata_ = response_metadata; + } + // Sets a deadline for the gRPC request. void SetDeadline(absl::Duration timeout) { context_.set_deadline( @@ -146,11 +158,12 @@ class RawClientParams { void OnDone(const grpc::Status& status) { if (status.ok()) { - std::move(raw_callback_)(std::move(raw_response_)); + std::move(raw_callback_)(std::move(raw_response_), response_metadata_); } else { std::move(raw_callback_)( absl::Status(static_cast(status.error_code()), - status.error_message())); + status.error_message()), + response_metadata_); } delete this; } @@ -170,8 +183,12 @@ class RawClientParams { std::unique_ptr raw_response_; // callback will only run once for a single gRPC call - absl::AnyInvocable>) &&> + absl::AnyInvocable>, + ResponseMetadata) &&> raw_callback_ = nullptr; + + struct RequestConfig request_config_; + ResponseMetadata response_metadata_; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/clients/code_dispatcher/BUILD b/services/common/clients/code_dispatcher/BUILD index 087bf1a9..62cc85d5 100644 --- a/services/common/clients/code_dispatcher/BUILD +++ b/services/common/clients/code_dispatcher/BUILD @@ -79,6 +79,9 @@ cc_library( "request_context.h", ], deps = [ - "@google_privacysandbox_servers_common//src/logger:request_context_impl", + "//services/common/loggers:request_log_context", + "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/status:statusor", + "@google_privacysandbox_servers_common//src/logger:request_context_logger", ], ) diff --git a/services/common/clients/code_dispatcher/request_context.cc b/services/common/clients/code_dispatcher/request_context.cc index 5968c258..14c5bd0f 100644 --- a/services/common/clients/code_dispatcher/request_context.cc +++ b/services/common/clients/code_dispatcher/request_context.cc @@ -19,11 +19,12 @@ namespace privacy_sandbox::bidding_auction_servers { RomaRequestContext::RomaRequestContext( const absl::btree_map& context_map, const privacy_sandbox::server_common::ConsentedDebugConfiguration& - debug_config) - : request_logging_context_(context_map, debug_config) {} + debug_config, + absl::AnyInvocable debug_info) + : request_logging_context_(context_map, debug_config, + std::move(debug_info)) {} -const privacy_sandbox::server_common::log::ContextImpl& -RomaRequestContext::GetLogContext() const { +RequestLogContext& RomaRequestContext::GetLogContext() { return request_logging_context_; } @@ -45,10 +46,10 @@ RomaRequestSharedContext::GetRomaRequestContext() const { RomaRequestContextFactory::RomaRequestContextFactory( const absl::btree_map& context_map, const privacy_sandbox::server_common::ConsentedDebugConfiguration& - debug_config) { - roma_request_context_ = - std::make_shared(context_map, debug_config); -} + debug_config, + absl::AnyInvocable debug_info) + : roma_request_context_(std::make_shared( + context_map, debug_config, std::move(debug_info))) {} RomaRequestSharedContext RomaRequestContextFactory::Create() { return RomaRequestSharedContext(roma_request_context_); diff --git a/services/common/clients/code_dispatcher/request_context.h b/services/common/clients/code_dispatcher/request_context.h index 708b9a86..3cea5856 100644 --- a/services/common/clients/code_dispatcher/request_context.h +++ b/services/common/clients/code_dispatcher/request_context.h @@ -23,6 +23,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "services/common/loggers/request_log_context.h" #include "src/logger/request_context_impl.h" namespace privacy_sandbox::bidding_auction_servers { @@ -34,12 +35,26 @@ class RomaRequestContext { RomaRequestContext( const absl::btree_map& context_map, const privacy_sandbox::server_common::ConsentedDebugConfiguration& - debug_config); + debug_config, + absl::AnyInvocable + debug_info); - const privacy_sandbox::server_common::log::ContextImpl& GetLogContext() const; + RequestLogContext& GetLogContext(); + + // Set if it is a protected audience request. + void SetIsProtectedAudienceRequest(bool is_protected_audience_request) { + is_protected_audience_request_ = is_protected_audience_request; + } + // Returns true if this is a Protected Audience request. + bool IsProtectedAudienceRequest() const { + return is_protected_audience_request_; + } + + bool IsConsented() { return request_logging_context_.is_consented(); } private: - privacy_sandbox::server_common::log::ContextImpl request_logging_context_; + RequestLogContext request_logging_context_; + bool is_protected_audience_request_ = false; }; class RomaRequestContextFactory; @@ -50,6 +65,9 @@ class RomaRequestSharedContext { public: RomaRequestSharedContext() {} + // The returned status indicates if RomaRequestContext has gone out of the + // scope. This can happen during Roma request processing timeout during which + // the caller owning the context could have returned. absl::StatusOr> GetRomaRequestContext() const; @@ -69,7 +87,9 @@ class RomaRequestContextFactory { RomaRequestContextFactory( const absl::btree_map& context_map, const privacy_sandbox::server_common::ConsentedDebugConfiguration& - debug_config); + debug_config, + absl::AnyInvocable + debug_info); RomaRequestSharedContext Create(); RomaRequestContextFactory(RomaRequestContextFactory&& other) = delete; diff --git a/services/common/clients/http/multi_curl_http_fetcher_async.cc b/services/common/clients/http/multi_curl_http_fetcher_async.cc index f71230cc..b3298aed 100644 --- a/services/common/clients/http/multi_curl_http_fetcher_async.cc +++ b/services/common/clients/http/multi_curl_http_fetcher_async.cc @@ -273,7 +273,12 @@ void MultiCurlHttpFetcherAsync::FetchUrls( shared_lifetime->all_done_callback = std::move(done_callback); shared_lifetime->results = std::vector>(requests.size()); - + if (requests.empty()) { + // Execute callback immediately if there are no requests. + std::move(shared_lifetime->all_done_callback)( + std::move(shared_lifetime->results)); + return; + } for (int i = 0; i < requests.size(); i++) { FetchUrl(requests.at(i), absl::ToInt64Milliseconds(timeout), [i, shared_lifetime](absl::StatusOr result) { diff --git a/services/common/clients/http/multi_curl_http_fetcher_async_test.cc b/services/common/clients/http/multi_curl_http_fetcher_async_test.cc index 4571bcd9..a7c74360 100644 --- a/services/common/clients/http/multi_curl_http_fetcher_async_test.cc +++ b/services/common/clients/http/multi_curl_http_fetcher_async_test.cc @@ -175,6 +175,20 @@ TEST_F(MultiCurlHttpFetcherAsyncTest, CanFetchMultipleUrlsInParallel) { done.Wait(); } +TEST_F(MultiCurlHttpFetcherAsyncTest, InvokesCallbackForEmptyRequestVector) { + absl::BlockingCounter done(1); + std::vector test_requests = {}; + auto done_cb = [&done, &test_requests]( + const std::vector>& results) { + EXPECT_EQ(results.size(), test_requests.size()); + done.DecrementCount(); + }; + + fetcher_->FetchUrls(test_requests, absl::Milliseconds(kNormalTimeoutMs), + std::move(done_cb)); + done.Wait(); +} + TEST_F(MultiCurlHttpFetcherAsyncTest, PutsUrlSuccessfully) { std::string msg; absl::Notification notification; diff --git a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client_test.cc b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client_test.cc index 6ac5be25..24dfa920 100644 --- a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client_test.cc +++ b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client_test.cc @@ -28,8 +28,7 @@ constexpr char kEgId[] = "1776"; class KeyValueAsyncHttpClientTest : public testing::Test { public: - static constexpr char hostname_[] = - "https://googleads.g.doubleclick.net/td/bts"; + static constexpr char hostname_[] = "https://trustedhost.com"; std::unique_ptr mock_http_fetcher_async_ = std::make_unique(); const absl::flat_hash_set expected_urls_1 = { diff --git a/services/common/clients/http_kv_server/seller/seller_key_value_async_http_client_test.cc b/services/common/clients/http_kv_server/seller/seller_key_value_async_http_client_test.cc index a73791e0..a271df1e 100644 --- a/services/common/clients/http_kv_server/seller/seller_key_value_async_http_client_test.cc +++ b/services/common/clients/http_kv_server/seller/seller_key_value_async_http_client_test.cc @@ -29,7 +29,7 @@ using test_timeout = absl::Duration; class KeyValueAsyncHttpClientTest : public testing::Test { public: - static constexpr char hostname_[] = "https://pubads.g.doubleclick.net/td/sts"; + static constexpr char hostname_[] = "https://trustedhost.com"; std::unique_ptr mock_http_fetcher_async_ = std::make_unique(); const absl::flat_hash_set expected_urls_ = { diff --git a/services/common/clients/http_kv_server/util/generate_url.h b/services/common/clients/http_kv_server/util/generate_url.h index 2c2b76c7..8ff724c9 100644 --- a/services/common/clients/http_kv_server/util/generate_url.h +++ b/services/common/clients/http_kv_server/util/generate_url.h @@ -51,9 +51,8 @@ void AddListItemsAsQueryParamsToUrl(std::string* url, absl::string_view key, /** * Clears the string when creating a new url, adds the host domain, and adds * the ? to signify the start of the query parameters section. - * @param kv_server_host_domain The full URL you want before the '?' - * Could be as simple as "example.com", or as much as - * "https://googleads.g.doubleclick.net/td/bts" + * @param kv_server_host_domain The full URL you want before the '?' for example + * "https://kvserver.com/trusted-signals"". * @param url Output variable, url will be stored in this string. Note the * string will be cleared. */ diff --git a/services/common/clients/kv_server/kv_async_client.cc b/services/common/clients/kv_server/kv_async_client.cc index 7f1a1d28..bf097dd3 100644 --- a/services/common/clients/kv_server/kv_async_client.cc +++ b/services/common/clients/kv_server/kv_async_client.cc @@ -94,10 +94,10 @@ void KVAsyncGrpcClient::SendRpc( absl::Status KVAsyncGrpcClient::ExecuteInternal( std::unique_ptr raw_request, const RequestMetadata& metadata, - absl::AnyInvocable< - void(absl::StatusOr>) &&> + absl::AnyInvocable>, + ResponseMetadata) &&> on_done, - absl::Duration timeout) const { + absl::Duration timeout, RequestConfig request_config) const { PS_VLOG(6) << "Raw request:\n" << raw_request->DebugString(); PS_ASSIGN_OR_RETURN(std::string binary_http_msg, ToBinaryHTTP(*raw_request, /*to_json=*/false)); diff --git a/services/common/clients/kv_server/kv_async_client.h b/services/common/clients/kv_server/kv_async_client.h index e2519834..d30ecdaa 100644 --- a/services/common/clients/kv_server/kv_async_client.h +++ b/services/common/clients/kv_server/kv_async_client.h @@ -74,9 +74,11 @@ class KVAsyncGrpcClient std::unique_ptr raw_request, const RequestMetadata& metadata, absl::AnyInvocable< - void(absl::StatusOr>) &&> + void(absl::StatusOr>, + ResponseMetadata) &&> on_done, - absl::Duration timeout = kMaxTimeout) const override; + absl::Duration timeout = kMaxTimeout, + RequestConfig request_config = {}) const override; protected: void SendRpc(ObliviousHttpRequestUptr oblivious_http_context, diff --git a/services/common/clients/kv_server/kv_async_client_test.cc b/services/common/clients/kv_server/kv_async_client_test.cc index b922de61..4092cc33 100644 --- a/services/common/clients/kv_server/kv_async_client_test.cc +++ b/services/common/clients/kv_server/kv_async_client_test.cc @@ -73,9 +73,8 @@ TEST_F(KVAsyncGrpcTest, CallsServerWithRequest) { absl::Status status = class_under_test.ExecuteInternal( std::make_unique(raw_request), {}, [¬ification]( - absl::StatusOr> get_values_response) { - notification.Notify(); - }); + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { notification.Notify(); }); notification.WaitForNotification(); EXPECT_TRUE(status.ok()) << status; } @@ -109,9 +108,8 @@ TEST_F(KVAsyncGrpcTest, CallsServerWithMetadata) { absl::Status status = class_under_test.ExecuteInternal( std::make_unique(raw_request), sent_metadata, [¬ification]( - absl::StatusOr> get_values_response) { - notification.Notify(); - }); + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { notification.Notify(); }); notification.WaitForNotification(); ASSERT_TRUE(status.ok()) << status; @@ -142,7 +140,8 @@ TEST_F(KVAsyncGrpcTest, PassesStatusToCallback) { absl::Status status = class_under_test.ExecuteInternal( std::make_unique(raw_request), {}, [¬ification]( - absl::StatusOr> get_values_response) { + absl::StatusOr> get_values_response, + ResponseMetadata response_metadata) { EXPECT_EQ(get_values_response.status().code(), absl::StatusCode::kInvalidArgument); notification.Notify(); diff --git a/services/common/code_fetch/code_fetcher_interface.h b/services/common/code_fetch/code_fetcher_interface.h index 7daf9ffe..0860002c 100644 --- a/services/common/code_fetch/code_fetcher_interface.h +++ b/services/common/code_fetch/code_fetcher_interface.h @@ -28,6 +28,9 @@ namespace privacy_sandbox::bidding_auction_servers { using WrapCodeForDispatch = absl::AnyInvocable&)>; +using WrapSingleCodeBlobForDispatch = + absl::AnyInvocable; + // Interface for different versions of code fetchers. class CodeFetcherInterface { public: diff --git a/services/common/constants/BUILD b/services/common/constants/BUILD index c85d284d..136bdb83 100644 --- a/services/common/constants/BUILD +++ b/services/common/constants/BUILD @@ -46,3 +46,13 @@ cc_library( "user_error_strings.h", ], ) + +cc_library( + name = "common_constants", + hdrs = [ + "common_constants.h", + ], + deps = [ + "@com_google_absl//absl/strings", + ], +) diff --git a/services/common/constants/common_constants.h b/services/common/constants/common_constants.h new file mode 100644 index 00000000..a15e3622 --- /dev/null +++ b/services/common/constants/common_constants.h @@ -0,0 +1,30 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_COMMON_CONSTANTS_COMMON_CONSTANTS_H_ +#define SERVICES_COMMON_CONSTANTS_COMMON_CONSTANTS_H_ + +#include + +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers { + +inline constexpr absl::string_view kIgnoredPlaceholderValue = "PLACEHOLDER"; + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_COMMON_CONSTANTS_COMMON_CONSTANTS_H_ diff --git a/services/common/feature_flags.cc b/services/common/feature_flags.cc index a8ceabce..87825b74 100644 --- a/services/common/feature_flags.cc +++ b/services/common/feature_flags.cc @@ -21,3 +21,8 @@ ABSL_FLAG(bool, enable_temporary_unlimited_egress, false, "If true, temporary unlimited egress is allowed (if " "client had also set enable_unlimited_egress flag in request)"); + +ABSL_FLAG(bool, enable_chaffing, false, + "If true, chaff requests are sent out from the SFE. Chaff requests " + "are requests sent to buyers not participating in an auction to mask " + "the buyers associated with a client request."); diff --git a/services/common/feature_flags.h b/services/common/feature_flags.h index 36a7d163..0aa3b828 100644 --- a/services/common/feature_flags.h +++ b/services/common/feature_flags.h @@ -20,5 +20,6 @@ #include "absl/flags/declare.h" ABSL_DECLARE_FLAG(bool, enable_temporary_unlimited_egress); +ABSL_DECLARE_FLAG(bool, enable_chaffing); #endif // SERVICES_COMMON_FEATURE_FLAGS_H_ diff --git a/services/common/loggers/BUILD b/services/common/loggers/BUILD index e2d289e3..9e56b0ba 100644 --- a/services/common/loggers/BUILD +++ b/services/common/loggers/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") package(default_visibility = ["//visibility:public"]) @@ -25,6 +25,25 @@ cc_library( ], ) +cc_library( + name = "request_log_context", + hdrs = ["request_log_context.h"], + deps = [ + "//api:bidding_auction_servers_cc_proto", + "@google_privacysandbox_servers_common//src/logger:request_context_impl", + ], +) + +cc_test( + name = "request_log_context_test", + timeout = "short", + srcs = ["request_log_context_test.cc"], + deps = [ + ":request_log_context", + "@com_google_googletest//:gtest_main", + ], +) + cc_library( name = "benchmarking_logger", srcs = ["benchmarking_logger.cc"], diff --git a/services/common/loggers/request_log_context.h b/services/common/loggers/request_log_context.h new file mode 100644 index 00000000..184918be --- /dev/null +++ b/services/common/loggers/request_log_context.h @@ -0,0 +1,83 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_COMMON_LOGGERS_REQUEST_LOG_CONTEXT_H_ +#define SERVICES_COMMON_LOGGERS_REQUEST_LOG_CONTEXT_H_ + +#include + +#include "api/bidding_auction_servers.pb.h" +#include "src/logger/request_context_impl.h" + +#define EVENT_MESSAGE_PROVIDER_SET(T, field) \ + void Set(const T& _##field) { *event_message_.mutable_##field() = _##field; } + +#define EVENT_MESSAGE_PROVIDER_SET_RESPONSE(T, field) \ + void Set(T _##field) { \ + _##field.clear_debug_info(); \ + *event_message_.mutable_##field() = std::move(_##field); \ + } + +namespace privacy_sandbox::bidding_auction_servers { + +class EventMessageProvider { + public: + EventMessage Get() { return event_message_; } + + EVENT_MESSAGE_PROVIDER_SET(SelectAdRequest, select_ad_request); + EVENT_MESSAGE_PROVIDER_SET(ProtectedAuctionInput, protected_auction); + EVENT_MESSAGE_PROVIDER_SET(ProtectedAudienceInput, protected_audience); + EVENT_MESSAGE_PROVIDER_SET(AuctionResult, auction_result); + + EVENT_MESSAGE_PROVIDER_SET(GetBidsRequest, get_bid_request); + EVENT_MESSAGE_PROVIDER_SET(GetBidsRequest::GetBidsRawRequest, + get_bid_raw_request); + EVENT_MESSAGE_PROVIDER_SET_RESPONSE(GetBidsResponse::GetBidsRawResponse, + get_bid_raw_response); + + EVENT_MESSAGE_PROVIDER_SET(GenerateBidsRequest, generate_bid_request); + EVENT_MESSAGE_PROVIDER_SET(GenerateBidsRequest::GenerateBidsRawRequest, + generate_bid_raw_request); + EVENT_MESSAGE_PROVIDER_SET_RESPONSE( + GenerateBidsResponse::GenerateBidsRawResponse, generate_bid_raw_response); + + EVENT_MESSAGE_PROVIDER_SET(GenerateProtectedAppSignalsBidsRequest, + generate_app_signal_request); + EVENT_MESSAGE_PROVIDER_SET(GenerateProtectedAppSignalsBidsRequest:: + GenerateProtectedAppSignalsBidsRawRequest, + generate_app_signal_raw_request); + EVENT_MESSAGE_PROVIDER_SET_RESPONSE( + GenerateProtectedAppSignalsBidsResponse :: + GenerateProtectedAppSignalsBidsRawResponse, + generate_app_signal_raw_response); + + EVENT_MESSAGE_PROVIDER_SET(ScoreAdsRequest, score_ad_request); + EVENT_MESSAGE_PROVIDER_SET(ScoreAdsRequest::ScoreAdsRawRequest, + score_ad_raw_request); + EVENT_MESSAGE_PROVIDER_SET_RESPONSE(ScoreAdsResponse::ScoreAdsRawResponse, + score_ad_raw_response); + + void Set(absl::string_view udf_log) { event_message_.add_udf_log(udf_log); } + + private: + EventMessage event_message_; +}; + +using RequestLogContext = server_common::log::ContextImpl; + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_COMMON_LOGGERS_REQUEST_LOG_CONTEXT_H_ diff --git a/services/common/loggers/request_log_context_test.cc b/services/common/loggers/request_log_context_test.cc new file mode 100644 index 00000000..327ad337 --- /dev/null +++ b/services/common/loggers/request_log_context_test.cc @@ -0,0 +1,27 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "services/common/loggers/request_log_context.h" + +#include "gtest/gtest.h" + +namespace privacy_sandbox::bidding_auction_servers { + +TEST(RequestLogContext, Initialization) { + RequestLogContext context({}, server_common::ConsentedDebugConfiguration()); +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/metric/error_code.h b/services/common/metric/error_code.h index 9d647758..08adbca0 100644 --- a/services/common/metric/error_code.h +++ b/services/common/metric/error_code.h @@ -25,6 +25,9 @@ inline constexpr absl::string_view kAuctionScoreAdsDispatchResponseError = "ScoreAds dispatch response error"; inline constexpr absl::string_view kAuctionScoreAdsFailedToDispatchCode = "ScoreAds failed to dispatch code"; +inline constexpr absl::string_view + kAuctionScoreAdsFailedToInsertDispatchRequest = + "ScoreAds failed to create dispatch code"; inline constexpr absl::string_view kAuctionScoreAdsNoAdSelected = "ScoreAds no ad selected"; diff --git a/services/common/telemetry/configure_telemetry.h b/services/common/telemetry/configure_telemetry.h index 4c33f44d..05fb2971 100644 --- a/services/common/telemetry/configure_telemetry.h +++ b/services/common/telemetry/configure_telemetry.h @@ -28,9 +28,9 @@ #include "services/common/clients/config/trusted_server_config_client.h" #include "services/common/clients/config/trusted_server_config_client_util.h" #include "services/common/constants/common_service_flags.h" +#include "services/common/loggers/request_log_context.h" #include "services/common/metric/server_definition.h" #include "services/common/util/build_info.h" -#include "src/logger/request_context_impl.h" #include "src/telemetry/flag/telemetry_flag.h" #include "src/telemetry/telemetry.h" diff --git a/services/common/test/mocks.h b/services/common/test/mocks.h index 422f9aec..5220c37c 100644 --- a/services/common/test/mocks.h +++ b/services/common/test/mocks.h @@ -129,13 +129,14 @@ class AsyncClientMock absl::Duration timeout), (const, override)); - MOCK_METHOD( - absl::Status, ExecuteInternal, - (std::unique_ptr raw_request, const RequestMetadata& metadata, - absl::AnyInvocable>) &&> - on_done, - absl::Duration timeout), - (const, override)); + using OnDoneCallbackType = + absl::AnyInvocable>, + ResponseMetadata) &&>; + MOCK_METHOD(absl::Status, ExecuteInternal, + (std::unique_ptr raw_request, + const RequestMetadata& metadata, OnDoneCallbackType on_done, + absl::Duration timeout, RequestConfig request_config), + (const, override)); }; using BuyerFrontEndAsyncClientMock = @@ -306,6 +307,11 @@ class BuyerFrontEndAsyncClientFactoryMock public: MOCK_METHOD(std::shared_ptr, Get, (absl::string_view), (const, override)); + + MOCK_METHOD( + (std::vector>>), + Entries, (), (const, override)); }; // Dummy server in lieu of no support for mocking async stubs. diff --git a/services/common/test/random.cc b/services/common/test/random.cc index 5c0fc81f..553cf1e4 100644 --- a/services/common/test/random.cc +++ b/services/common/test/random.cc @@ -22,7 +22,6 @@ namespace privacy_sandbox::bidding_auction_servers { constexpr char kTestIgWithTwoAds[] = R"json({"ad_render_ids":["adg_id=134256827445&cr_id=594352291621&cv_id=4", "adg_id=134256827445&cr_id=605048329089&cv_id=2"],"browser_signals":{"joinCount":1,"bidCount":100,"prevWins":"[[1472425.0,{\"metadata\":[134256827485.0,594352291615.0,null,16996677067.0]}],[1475389.0,{\"metadata\":[134256827445.0,594352291621.0,null,16996677067.0]}],[1487572.0,{\"metadata\":[134256827445.0,605490292974.0,null,16996677067.0]}],[1451707.0,{\"metadata\":[134256827445.0,605048329092.0,null,16996677067.0]}],[1485996.0,{\"metadata\":[134256827445.0,605048329089.0,null,16996677067.0]}],[1450931.0,{\"metadata\":[134256827485.0,605490292980.0,null,16996677067.0]}],[1473069.0,{\"metadata\":[134256827485.0,605048328957.0,null,16996677067.0]}],[1461197.0,{\"metadata\":[134256827485.0,605048329080.0,null,16996677067.0]}]]"},"name":"1j1043317685"})json"; - std::string MakeARandomString() { return std::to_string(ToUnixNanos(absl::Now())); } @@ -629,6 +628,43 @@ AuctionResult MakeARandomSingleSellerAuctionResult( return result; } +AuctionResult MakeARandomComponentAuctionResultWithReportingUrls( + std::string generation_id, std::string top_level_seller, + std::string test_component_event, + std::string test_component_win_reporting_url, + std::string test_component_interaction_reporting_url, + std::string test_component_seller, std::vector buyer_list) { + AuctionResult result = + MakeARandomSingleSellerAuctionResult(std::move(buyer_list)); + result.mutable_win_reporting_urls() + ->mutable_component_seller_reporting_urls() + ->set_reporting_url(test_component_win_reporting_url); + result.mutable_win_reporting_urls() + ->mutable_component_seller_reporting_urls() + ->mutable_interaction_reporting_urls() + ->try_emplace(test_component_event, + test_component_interaction_reporting_url); + result.mutable_win_reporting_urls() + ->mutable_buyer_reporting_urls() + ->set_reporting_url(std::move(test_component_win_reporting_url)); + result.mutable_win_reporting_urls() + ->mutable_buyer_reporting_urls() + ->mutable_interaction_reporting_urls() + ->try_emplace(std::move(test_component_event), + std::move(test_component_interaction_reporting_url)); + result.set_top_level_seller(std::move(top_level_seller)); + result.set_ad_type(AdType::AD_TYPE_PROTECTED_AUDIENCE_AD); + result.mutable_auction_params()->set_ciphertext_generation_id( + std::move(generation_id)); + result.mutable_auction_params()->set_component_seller( + std::move(test_component_seller)); + result.mutable_ad_component_render_urls()->Add(MakeARandomString()); + result.mutable_ad_component_render_urls()->Add(MakeARandomString()); + result.set_bid(1); + result.set_bid_currency(MakeARandomString()); + return result; +} + AuctionResult MakeARandomComponentAuctionResult( std::string generation_id, std::string top_level_seller, std::vector buyer_list) { @@ -645,5 +681,4 @@ AuctionResult MakeARandomComponentAuctionResult( result.set_bid_currency(MakeARandomString()); return result; } - } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/test/random.h b/services/common/test/random.h index 237d2d83..2f2a877d 100644 --- a/services/common/test/random.h +++ b/services/common/test/random.h @@ -229,6 +229,16 @@ AuctionResult MakeARandomSingleSellerAuctionResult( std::vector buyer_list = {}); // Populates fields for a auction result object for a component seller auction. +AuctionResult MakeARandomComponentAuctionResultWithReportingUrls( + std::string generation_id, std::string top_level_seller, + std::string test_component_event, + std::string test_component_win_reporting_url, + std::string test_component_interaction_reporting_url, + std::string test_component_seller, + std::vector buyer_list = {}); + +// Populates fields for a auction result object for a component seller auction +// without reporting urls AuctionResult MakeARandomComponentAuctionResult( std::string generation_id, std::string top_level_seller, std::vector buyer_list = {}); diff --git a/services/common/util/BUILD b/services/common/util/BUILD index 219b5872..2881354c 100644 --- a/services/common/util/BUILD +++ b/services/common/util/BUILD @@ -158,10 +158,10 @@ cc_library( deps = [ ":error_categories", ":error_reporter", + "//services/common/loggers:request_log_context", "//services/common/loggers:source_location_context", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/strings", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", "@google_privacysandbox_servers_common//src/util/status_macro:source_location", ], ) @@ -200,6 +200,7 @@ cc_library( ":post_auction_signals", "//api:bidding_auction_servers_cc_proto", "//services/common/clients/http:http_fetcher_async", + "//services/common/loggers:request_log_context", "//services/common/util:json_util", "//services/common/util:request_response_constants", "@com_google_absl//absl/container:flat_hash_map", @@ -207,7 +208,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", "@rapidjson", ], @@ -262,12 +262,12 @@ cc_library( srcs = ["async_task_tracker.cc"], hdrs = ["async_task_tracker.h"], deps = [ + "//services/common/loggers:request_log_context", "//services/common/util:request_response_constants", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/log:check", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/synchronization", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", "@google_privacysandbox_servers_common//src/util/status_macro:source_location", ], ) @@ -306,12 +306,12 @@ cc_library( "binary_http_utils.h", ], deps = [ + "//services/common/loggers:request_log_context", "//services/common/util:request_response_constants", "@com_github_google_quiche//quiche:binary_http_unstable_api", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", ], ) @@ -371,11 +371,11 @@ cc_library( ], deps = [ "//services/common/encryption:mock_crypto_client_wrapper", + "//services/common/loggers:request_log_context", "@com_github_google_quiche//quiche:oblivious_http_unstable_api", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", ], ) @@ -454,7 +454,30 @@ cc_library( "tcmalloc_utils.h", ], deps = [ + "//services/common/loggers:request_log_context", "@com_google_tcmalloc//tcmalloc:malloc_extension", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", + ], +) + +cc_library( + name = "data_util", + hdrs = [ + "data_util.h", + ], + deps = [ + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_test( + name = "data_util_test", + size = "small", + srcs = [ + "data_util_test.cc", + ], + deps = [ + ":data_util", + "@com_google_absl//absl/strings:string_view", + "@com_google_googletest//:gtest_main", ], ) diff --git a/services/common/util/async_task_tracker.cc b/services/common/util/async_task_tracker.cc index ba4667e7..33145821 100644 --- a/services/common/util/async_task_tracker.cc +++ b/services/common/util/async_task_tracker.cc @@ -23,7 +23,7 @@ namespace privacy_sandbox::bidding_auction_servers { AsyncTaskTracker::AsyncTaskTracker( - int num_tasks_to_track, server_common::log::ContextImpl& log_context, + int num_tasks_to_track, RequestLogContext& log_context, absl::AnyInvocable on_all_tasks_done) : num_tasks_to_track_(num_tasks_to_track), pending_tasks_count_(num_tasks_to_track), diff --git a/services/common/util/async_task_tracker.h b/services/common/util/async_task_tracker.h index cd6ad975..a3c3e5f6 100644 --- a/services/common/util/async_task_tracker.h +++ b/services/common/util/async_task_tracker.h @@ -23,7 +23,7 @@ #include "absl/base/thread_annotations.h" #include "absl/functional/any_invocable.h" #include "absl/synchronization/mutex.h" -#include "src/logger/request_context_impl.h" +#include "services/common/loggers/request_log_context.h" namespace privacy_sandbox::bidding_auction_servers { @@ -52,7 +52,7 @@ enum class TaskStatus { class AsyncTaskTracker { public: explicit AsyncTaskTracker( - int num_tasks_to_track, server_common::log::ContextImpl& log_context, + int num_tasks_to_track, RequestLogContext& log_context, absl::AnyInvocable on_all_tasks_done); // Updates the stats. If all bids have been completed, then the registered @@ -86,7 +86,7 @@ class AsyncTaskTracker { int skipped_tasks_count_ ABSL_GUARDED_BY(mu_); int error_tasks_count_ ABSL_GUARDED_BY(mu_); absl::AnyInvocable on_all_tasks_done_; - server_common::log::ContextImpl& log_context_; + RequestLogContext& log_context_; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/util/async_task_tracker_test.cc b/services/common/util/async_task_tracker_test.cc index 560dcb18..0e974a9b 100644 --- a/services/common/util/async_task_tracker_test.cc +++ b/services/common/util/async_task_tracker_test.cc @@ -32,9 +32,8 @@ constexpr int kNumMaxThreads = 10; class AsyncTasksTrackerTest : public testing::Test { protected: - server_common::log::ContextImpl log_context_{ - absl::btree_map{}, - server_common::ConsentedDebugConfiguration()}; + RequestLogContext log_context_{absl::btree_map{}, + server_common::ConsentedDebugConfiguration()}; absl::Notification notification_; }; diff --git a/services/common/util/data_util.h b/services/common/util/data_util.h new file mode 100644 index 00000000..5c02833a --- /dev/null +++ b/services/common/util/data_util.h @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_COMMON_UTIL_DATA_UTIL_H_ +#define SERVICES_COMMON_UTIL_DATA_UTIL_H_ + +#include + +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers { + +// Finds the provided string needle index in the haystack array. +template +int FindItemIndex(const std::array& haystack, + absl::string_view needle) { + auto it = std::find(haystack.begin(), haystack.end(), needle); + if (it == haystack.end()) { + return -1; + } + + return std::distance(haystack.begin(), it); +} + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_COMMON_UTIL_DATA_UTIL_H_ diff --git a/services/common/util/data_util_test.cc b/services/common/util/data_util_test.cc new file mode 100644 index 00000000..a7ae22c5 --- /dev/null +++ b/services/common/util/data_util_test.cc @@ -0,0 +1,29 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "services/common/util/data_util.h" + +#include "gtest/gtest.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { + +TEST(DataUtil, FindIndexInArray) { + std::array test_array = {"item1", "item2"}; + EXPECT_EQ(FindItemIndex(test_array, "item2"), 1); +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/util/error_accumulator.cc b/services/common/util/error_accumulator.cc index 5c6e1fd3..1fc566e6 100644 --- a/services/common/util/error_accumulator.cc +++ b/services/common/util/error_accumulator.cc @@ -20,7 +20,7 @@ namespace privacy_sandbox::bidding_auction_servers { -ErrorAccumulator::ErrorAccumulator(server_common::log::ContextImpl* log_context) +ErrorAccumulator::ErrorAccumulator(RequestLogContext* log_context) : log_context_(log_context) {} void ErrorAccumulator::ReportError( diff --git a/services/common/util/error_accumulator.h b/services/common/util/error_accumulator.h index c8b55cd8..e73fcd7c 100644 --- a/services/common/util/error_accumulator.h +++ b/services/common/util/error_accumulator.h @@ -24,9 +24,9 @@ #include "absl/container/flat_hash_map.h" #include "absl/strings/string_view.h" +#include "services/common/loggers/request_log_context.h" #include "services/common/loggers/source_location_context.h" #include "services/common/util/error_reporter.h" -#include "src/logger/request_context_impl.h" #include "src/util/status_macro/source_location.h" namespace privacy_sandbox::bidding_auction_servers { @@ -41,7 +41,7 @@ class ErrorAccumulator : public ErrorReporter { using ErrorMap = std::map>; ErrorAccumulator() = default; - explicit ErrorAccumulator(server_common::log::ContextImpl* log_context); + explicit ErrorAccumulator(RequestLogContext* log_context); virtual ~ErrorAccumulator() = default; // ErrorAccumulator is neither copyable nor movable. @@ -72,7 +72,7 @@ class ErrorAccumulator : public ErrorReporter { const ErrorMap empty_error_map_ = {}; // Optional log_context to be used for error reporting. - server_common::log::ContextImpl* log_context_ = nullptr; + RequestLogContext* log_context_ = nullptr; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/util/error_categories.h b/services/common/util/error_categories.h index cf233e9b..7212d1d3 100644 --- a/services/common/util/error_categories.h +++ b/services/common/util/error_categories.h @@ -53,6 +53,8 @@ inline constexpr char kBadBuyerInputProto[] = inline constexpr char kPASSignalsForComponentAuction[] = "Unsupported component auction input (protected signals) for buyer: %s"; inline constexpr char kMalformedBuyerInput[] = "Malformed buyer input."; +inline constexpr char kEmptySelectAdRequest[] = + "Empty SelectAdRequest received."; inline constexpr char kEmptyProtectedAuctionCiphertextError[] = "protected_auction_ciphertext must be non-null."; inline constexpr char kUnsupportedClientType[] = "Unsupported client type."; @@ -92,8 +94,6 @@ inline constexpr char kEmptySeller[] = "Seller origin missing in auction config"; inline constexpr char kEmptyBuyerSignals[] = "Buyer signals missing in auction config for buyer: %s"; -inline constexpr char kUnknownClientType[] = - "Unknown client type in SelectAdRequest"; inline constexpr char kWrongSellerDomain[] = "Seller domain passed in request does not match this server's domain"; inline constexpr char kEmptyBuyerInPerBuyerConfig[] = diff --git a/services/common/util/json_util.h b/services/common/util/json_util.h index 264dafba..3e7a6ea8 100644 --- a/services/common/util/json_util.h +++ b/services/common/util/json_util.h @@ -126,9 +126,9 @@ inline absl::StatusOr SerializeJsonDoc( // Retrieves the string value of the specified member in the document. template inline absl::StatusOr GetStringMember( - const T& document, const std::string& member_name, + const T& document, absl::string_view member_name, bool is_empty_ok = false) { - auto it = document.FindMember(member_name.c_str()); + auto it = document.FindMember(member_name.data()); if (it == document.MemberEnd()) { return absl::InvalidArgumentError( absl::StrFormat(kMissingMember, member_name)); @@ -168,6 +168,45 @@ GetArrayMember(const T& document, const std::string& member_name) { return it->value.GetArray(); } +// Retrieves the array value of the specified member in the document as a +// non-const array. +template +inline absl::StatusOr>::Array> +GetArrayMember(T& document, const std::string& member_name) { + auto it = document.FindMember(member_name.c_str()); + if (it == document.MemberEnd()) { + return absl::InvalidArgumentError( + absl::StrFormat(kMissingMember, member_name)); + } + + if (!it->value.IsArray()) { + return absl::InvalidArgumentError( + absl::StrFormat(kUnexpectedMemberType, member_name, + rapidjson::kArrayType, it->value.GetType())); + } + + return it->value.GetArray(); +} + +// Retrieves the number value of the specified member in the document. +template +inline absl::StatusOr GetIntMember(const T& document, + absl::string_view member_name) { + auto it = document.FindMember(member_name.data()); + if (it == document.MemberEnd()) { + return absl::InvalidArgumentError( + absl::StrFormat(kMissingMember, member_name)); + } + + if (!it->value.IsInt()) { + return absl::InvalidArgumentError( + absl::StrFormat(kUnexpectedMemberType, member_name, + rapidjson::kNumberType, it->value.GetType())); + } + + return it->value.GetInt(); +} + } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_COMMON_UTIL_JSON_UTIL_H_ diff --git a/services/common/util/json_util_test.cc b/services/common/util/json_util_test.cc index 048b353f..87fd97ae 100644 --- a/services/common/util/json_util_test.cc +++ b/services/common/util/json_util_test.cc @@ -123,9 +123,10 @@ TEST(SerializeJsonDoc, GetStringMember_FailsOnNonStringVal) { } // Function to compare a rapidjson::GenericArray with a std::vector -bool areArraysEqual(const rapidjson::GenericArray< - true, rapidjson::GenericValue>>& arr1, - const std::vector& arr2) { +bool are_arrays_equal( + const rapidjson::GenericArray< + true, rapidjson::GenericValue>>& arr1, + const std::vector& arr2) { if (arr1.Size() != arr2.size()) { return false; } @@ -141,13 +142,13 @@ bool areArraysEqual(const rapidjson::GenericArray< TEST(SerializeJsonDoc, GetArrayMember_WorksForKeyPresentInDocument) { std::string json_str = R"json({"key": ["0.32", "0.12", "0.98"]})json"; - auto document = ParseJsonString(json_str); + const auto document = ParseJsonString(json_str); ASSERT_TRUE(document.ok()) << document.status(); auto actual_value = GetArrayMember(*document, "key"); ASSERT_TRUE(actual_value.ok()) << actual_value.status(); const std::vector expectedArray = {"0.32", "0.12", "0.98"}; - ASSERT_TRUE(areArraysEqual(actual_value.value(), expectedArray)); + ASSERT_TRUE(are_arrays_equal(*actual_value, expectedArray)); } TEST(SerializeJsonDoc, GetArrayMember_FailsOnMissingKey) { @@ -167,5 +168,33 @@ TEST(SerializeJsonDoc, GetArrayMember_FailsOnNonArrayVal) { EXPECT_FALSE(actual_value.ok()); } +TEST(SerializeJsonDoc, GetNumberMember_ParsesTheNumberSuccessfully) { + std::string json_str = R"json({"key": 123})json"; + auto document = ParseJsonString(json_str); + ASSERT_TRUE(document.ok()) << document.status(); + + auto actual_value = GetIntMember(*document, "key"); + ASSERT_TRUE(actual_value.ok()); + EXPECT_EQ(*actual_value, 123); +} + +TEST(SerializeJsonDoc, GetNumberMember_ComplainsOnMissingKey) { + std::string json_str = R"json({"key": 123})json"; + auto document = ParseJsonString(json_str); + ASSERT_TRUE(document.ok()) << document.status(); + + auto actual_value = GetIntMember(*document, "MissingKey"); + EXPECT_FALSE(actual_value.ok()) << actual_value.status(); +} + +TEST(SerializeJsonDoc, GetNumberMember_ComplainsOnNonIntType) { + std::string json_str = R"json({"key": 123.67})json"; + auto document = ParseJsonString(json_str); + ASSERT_TRUE(document.ok()) << document.status(); + + auto actual_value = GetIntMember(*document, "Key"); + EXPECT_FALSE(actual_value.ok()) << actual_value.status(); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/util/post_auction_signals.h b/services/common/util/post_auction_signals.h index a76151b2..bed4ca99 100644 --- a/services/common/util/post_auction_signals.h +++ b/services/common/util/post_auction_signals.h @@ -33,6 +33,10 @@ struct PostAuctionSignals { float winning_bid; // Currency for said winning bid. std::string winning_bid_currency; + // The winning bid from the auction, in the seller's currency. + float winning_bid_in_seller_currency; + // The seller's currency. + std::string seller_currency; // The bid which was scored second highest in the auction. float highest_scoring_other_bid; diff --git a/services/common/util/reporting_util.cc b/services/common/util/reporting_util.cc index 9b3d6049..95be134e 100644 --- a/services/common/util/reporting_util.cc +++ b/services/common/util/reporting_util.cc @@ -41,11 +41,13 @@ inline constexpr absl::string_view kFeatureEnabled = "true"; void MayVlogAdTechCodeLogs(const rapidjson::Document& document, const std::string& log_type, - server_common::log::ContextImpl& log_context) { + RequestLogContext& log_context) { auto logs_it = document.FindMember(log_type.c_str()); if (logs_it != document.MemberEnd()) { for (const auto& log : logs_it->value.GetArray()) { - PS_VLOG(kUdfLog, log_context) << log_type << ": " << log.GetString(); + std::string log_str = absl::StrCat(log_type, ": ", log.GetString()); + PS_VLOG(kUdfLog, log_context) << log_str; + log_context.SetEventMessageField(log_str); } } } @@ -63,17 +65,55 @@ void AppendFeatureFlagValue(std::string& feature_flags, } // namespace +PostAuctionSignals GeneratePostAuctionSignalsForTopLevelSeller( + const std::optional& winning_ad_score) { + // If there is no winning ad, return with default signals values. + if (!winning_ad_score.has_value()) { + return {kDefaultWinningInterestGroupName, + kDefaultWinningInterestGroupOwner, + kDefaultWinningBid, + /*DefaultWinningBidCurrency=*/kUnknownBidCurrencyCode, + /*winning_bid_in_seller_currency=*/kDefaultWinningBid, + kUnknownBidCurrencyCode, + /*DefaultHighestScoringOtherBid=*/0.0, + /*DefaultHighestScoringOtherBidCurrency=*/kUnknownBidCurrencyCode, + kDefaultHighestScoringOtherBidInterestGroupOwner, + /*DefaultHasHighestScoringOtherBid=*/false, + kDefaultWinningScore, + kDefaultWinningAdRenderUrl, + {}}; + } + float winning_bid = winning_ad_score->buyer_bid(); + float winning_score = winning_ad_score->desirability(); + + return {winning_ad_score->interest_group_name(), + winning_ad_score->interest_group_owner(), + winning_bid, + /*DefaultWinningBidCurrency=*/kUnknownBidCurrencyCode, + /*winning_bid_in_seller_currency=*/kDefaultWinningBid, + kUnknownBidCurrencyCode, + /*DefaultHighestScoringOtherBid=*/0.0, + /*DefaultHighestScoringOtherBidCurrency=*/kUnknownBidCurrencyCode, + kDefaultHighestScoringOtherBidInterestGroupOwner, + /*DefaultHasHighestScoringOtherBid=*/false, + winning_score, + winning_ad_score->render(), + {}}; +} + PostAuctionSignals GeneratePostAuctionSignals( const std::optional& winning_ad_score, - absl::string_view seller_currency) { + std::string seller_currency) { // If there is no winning ad, return with default signals values. if (!winning_ad_score.has_value()) { return {kDefaultWinningInterestGroupName, kDefaultWinningInterestGroupOwner, kDefaultWinningBid, - /*DefaultWinningBidCurrency=*/kEmptyBidCurrencyCode, + /*DefaultWinningBidCurrency=*/kUnknownBidCurrencyCode, + /*winning_bid_in_seller_currency=*/kDefaultWinningBid, + std::move(seller_currency), /*DefaultHighestScoringOtherBid=*/0.0, - /*DefaultHighestScoringOtherBidCurrency=*/kEmptyBidCurrencyCode, + /*DefaultHighestScoringOtherBidCurrency=*/kUnknownBidCurrencyCode, kDefaultHighestScoringOtherBidInterestGroupOwner, /*DefaultHasHighestScoringOtherBid=*/false, kDefaultWinningScore, @@ -81,6 +121,8 @@ PostAuctionSignals GeneratePostAuctionSignals( {}}; } float winning_bid = winning_ad_score->buyer_bid(); + float winning_bid_in_seller_currency = + winning_ad_score->incoming_bid_in_seller_currency(); float winning_score = winning_ad_score->desirability(); // Set second highest other bid information in signals if available. std::string highest_scoring_other_bid_ig_owner = @@ -99,11 +141,10 @@ PostAuctionSignals GeneratePostAuctionSignals( } std::string bid_currency = (winning_ad_score->buyer_bid_currency().empty()) - ? kEmptyBidCurrencyCode + ? kUnknownBidCurrencyCode : winning_ad_score->buyer_bid_currency(); std::string highest_scoring_other_bid_currency = - (seller_currency.empty()) ? kEmptyBidCurrencyCode - : std::string(seller_currency); + (seller_currency.empty()) ? kUnknownBidCurrencyCode : seller_currency; bool made_highest_scoring_other_bid = false; if (winning_ad_score->ig_owner_highest_scoring_other_bids_map().size() == 1 && @@ -137,6 +178,8 @@ PostAuctionSignals GeneratePostAuctionSignals( winning_ad_score->interest_group_owner(), winning_bid, bid_currency, + winning_bid_in_seller_currency, + std::move(seller_currency), highest_scoring_other_bid, highest_scoring_other_bid_currency, std::move(highest_scoring_other_bid_ig_owner), @@ -202,12 +245,34 @@ DebugReportingPlaceholder GetPlaceholderDataForInterestGroup( rejection_reason = ig_name_itr->second; } } - return {.winning_bid = post_auction_signals.winning_bid, - .made_winning_bid = made_winning_bid, - .highest_scoring_other_bid = - post_auction_signals.highest_scoring_other_bid, - .made_highest_scoring_other_bid = made_highest_scoring_other_bid, - .rejection_reason = rejection_reason}; + float winning_bid; + absl::string_view both_currency_codes; + if (!post_auction_signals.seller_currency.empty()) { + winning_bid = post_auction_signals.winning_bid_in_seller_currency; + // In DebugReporting, when seller currency is set, all currency markers are + // set to it. + both_currency_codes = post_auction_signals.seller_currency; + } else { + // In the PostAuctionSignals, the winning bid is always the original + // buyer_bid. + winning_bid = post_auction_signals.winning_bid; + // In DebugReporting, when seller currency is un-set, all currency markers + // are set to Unknown. + both_currency_codes = kUnknownBidCurrencyCode; + } + return { + .winning_bid = winning_bid, + .winning_bid_currency = std::string(both_currency_codes), + .made_winning_bid = made_winning_bid, + // The PostAuctionSignals just retrieves and sets the + // highest_scoring_other_bid. highest_scoring_other_bid was already set to + // the original value if no seller_currency, and the modified value if + // some seller currency. + .highest_scoring_other_bid = + post_auction_signals.highest_scoring_other_bid, + .highest_scoring_other_bid_currency = std::string(both_currency_codes), + .made_highest_scoring_other_bid = made_highest_scoring_other_bid, + .rejection_reason = rejection_reason}; } SellerRejectionReason ToSellerRejectionReason( @@ -270,7 +335,7 @@ absl::string_view ToSellerRejectionReasonString( void MayVlogAdTechCodeLogs(bool enable_ad_tech_code_logging, const rapidjson::Document& document, - server_common::log::ContextImpl& log_context) { + RequestLogContext& log_context) { if (!enable_ad_tech_code_logging) { return; } @@ -282,7 +347,7 @@ void MayVlogAdTechCodeLogs(bool enable_ad_tech_code_logging, absl::StatusOr ParseAndGetResponseJson( bool enable_ad_tech_code_logging, const std::string& response, - server_common::log::ContextImpl& log_context) { + RequestLogContext& log_context) { PS_ASSIGN_OR_RETURN(rapidjson::Document document, ParseJsonString(response)); MayVlogAdTechCodeLogs(enable_ad_tech_code_logging, document, log_context); return SerializeJsonDoc(document["response"]); diff --git a/services/common/util/reporting_util.h b/services/common/util/reporting_util.h index 578dcc72..408143d3 100644 --- a/services/common/util/reporting_util.h +++ b/services/common/util/reporting_util.h @@ -23,15 +23,19 @@ #include "absl/strings/string_view.h" #include "api/bidding_auction_servers.pb.h" #include "services/common/clients/http/http_fetcher_async.h" +#include "services/common/loggers/request_log_context.h" #include "services/common/util/post_auction_signals.h" -#include "src/logger/request_context_impl.h" namespace privacy_sandbox::bidding_auction_servers { constexpr absl::string_view kWinningBidPlaceholder = "${winningBid}"; +constexpr absl::string_view kWinningBidCurrencyPlaceholder = + "${winningBidCurrency}"; constexpr absl::string_view kMadeWinningBidPlaceholder = "${madeWinningBid}"; constexpr absl::string_view kHighestScoringOtherBidPlaceholder = "${highestScoringOtherBid}"; +constexpr absl::string_view kHighestScoringOtherBidCurrencyPlaceholder = + "${highestScoringOtherBidCurrency}"; constexpr absl::string_view kMadeHighestScoringOtherBidPlaceholder = "${madeHighestScoringOtherBid}"; constexpr absl::string_view kRejectReasonPlaceholder = "${rejectReason}"; @@ -58,7 +62,7 @@ constexpr absl::string_view kRejectionReasonBidFromScoreAdFailedCurrencyCheck = "bid-from-score-ad-failed-currency-check"; constexpr float kDefaultWinningBid = 0.0; -constexpr char kEmptyBidCurrencyCode[] = "???"; +constexpr char kUnknownBidCurrencyCode[] = "???"; constexpr float kDefaultWinningScore = 0.0; inline constexpr char kDefaultWinningAdRenderUrl[] = ""; inline constexpr char kDefaultWinningInterestGroupName[] = ""; @@ -74,10 +78,14 @@ inline constexpr char kFeatureDebugUrlGeneration[] = struct DebugReportingPlaceholder { // The winning bid from the auction. float winning_bid; + // Currency for said bid. + std::string winning_bid_currency; // If the interest group made the winning bid. bool made_winning_bid; // The bid which was scored second highest in the auction. float highest_scoring_other_bid; + // Currency for said second-highest-scoring bid in the auction. + std::string highest_scoring_other_bid_currency; // If the interest group made the highest scoring other bid. bool made_highest_scoring_other_bid; // Reason provided by the seller if rejected. @@ -89,7 +97,12 @@ struct DebugReportingPlaceholder { // If there is no winning ad, default values are returned. PostAuctionSignals GeneratePostAuctionSignals( const std::optional& winning_ad_score, - absl::string_view seller_currency); + std::string seller_currency); + +// Returns post auction signals from winning ad score for the +// top level seller. +PostAuctionSignals GeneratePostAuctionSignalsForTopLevelSeller( + const std::optional& winning_ad_score); // Returns a http request object for debug reporting after replacing placeholder // data in the url. @@ -118,13 +131,13 @@ absl::string_view ToSellerRejectionReasonString( // `enable_ad_tech_code_logging` is enabled. void MayVlogAdTechCodeLogs(bool enable_ad_tech_code_logging, const rapidjson::Document& document, - server_common::log::ContextImpl& log_context); + RequestLogContext& log_context); // Parses the JSON string, conditionally prints the logs from the response and // returns a serialized response string retrieved from the underlying UDF. absl::StatusOr ParseAndGetResponseJson( bool enable_ad_tech_code_logging, const std::string& response, - server_common::log::ContextImpl& log_context); + RequestLogContext& log_context); // Returns a JSON string for feature flags to be used by the wrapper script. std::string GetFeatureFlagJson(bool enable_logging, diff --git a/services/common/util/reporting_util_test.cc b/services/common/util/reporting_util_test.cc index 28e543cc..0cb25cf7 100644 --- a/services/common/util/reporting_util_test.cc +++ b/services/common/util/reporting_util_test.cc @@ -32,7 +32,7 @@ inline constexpr char kTestIgName[] = "test_ig_name"; inline constexpr char kWinningIgName[] = "winning_ig_name"; inline constexpr char kWinningIgOwner[] = "winning_ig_owner"; inline constexpr char kSellerCurrency[] = "GBP"; -inline constexpr char kDefaultHighestScoringOtherBidCurrency[] = "???"; +inline constexpr float kIncomingBidInSellerCurrency = 1.776f; TEST(PostAuctionSignalsTest, HasAllSignals) { std::optional ad_score = @@ -61,7 +61,7 @@ TEST(PostAuctionSignalsTest, HasWinningBidSignals) { EXPECT_EQ(ad_score->desirability(), signals.winning_score); EXPECT_EQ(ad_score->render(), signals.winning_ad_render_url); EXPECT_EQ(signals.highest_scoring_other_bid_currency, - kDefaultHighestScoringOtherBidCurrency); + kUnknownBidCurrencyCode); } TEST(PostAuctionSignalsTest, HasHighestOtherBidSignals) { @@ -92,7 +92,7 @@ TEST(PostAuctionSignalsTest, DoesNotHaveAnySignal) { EXPECT_FALSE(signals.has_highest_scoring_other_bid); EXPECT_EQ(signals.highest_scoring_other_bid_ig_owner, ""); EXPECT_EQ(signals.highest_scoring_other_bid_currency, - kDefaultHighestScoringOtherBidCurrency); + kUnknownBidCurrencyCode); EXPECT_EQ(signals.highest_scoring_other_bid, 0.0); EXPECT_EQ(signals.rejection_reason_map.size(), 0); } @@ -298,45 +298,74 @@ TEST(CreateDebugReportingHttpRequestTest, } TEST(GetPlaceholderDataForInterestGroupOwnerTest, IgOwnerIsNone) { - absl::flat_hash_map> - rejection_reason_map; PostAuctionSignals signals = { .winning_ig_name = kEmptyString, .winning_ig_owner = kEmptyString, .winning_bid = 0.0, + .winning_bid_currency = "YEN", + .winning_bid_in_seller_currency = 0.0, + .seller_currency = "", .highest_scoring_other_bid = 0.0, .highest_scoring_other_bid_ig_owner = kEmptyString, .has_highest_scoring_other_bid = false, .winning_score = 0.0, .winning_ad_render_url = kEmptyString, - .rejection_reason_map = std::move(rejection_reason_map)}; + .rejection_reason_map = {}}; DebugReportingPlaceholder placeholder = GetPlaceholderDataForInterestGroup(kTestIgOwner, kTestIgName, signals); EXPECT_FALSE(placeholder.made_winning_bid); EXPECT_FALSE(placeholder.made_highest_scoring_other_bid); EXPECT_EQ(placeholder.winning_bid, signals.winning_bid); + // No Seller Currency set, so currency is expected to be the unknown marker. + EXPECT_EQ(placeholder.winning_bid_currency, kUnknownBidCurrencyCode); + EXPECT_EQ(placeholder.highest_scoring_other_bid, + signals.highest_scoring_other_bid); + // No Seller Currency set, so currency is expected to be the unknown marker. + EXPECT_EQ(placeholder.highest_scoring_other_bid_currency, + kUnknownBidCurrencyCode); + EXPECT_EQ(placeholder.rejection_reason, + SellerRejectionReason::SELLER_REJECTION_REASON_NOT_AVAILABLE); +} + +TEST(GetPlaceholderDataForInterestGroupOwnerTest, SellerCurrencySet) { + PostAuctionSignals signals = { + .winning_ig_name = kEmptyString, + .winning_ig_owner = kEmptyString, + .winning_bid = 0.0, + .winning_bid_in_seller_currency = kIncomingBidInSellerCurrency, + .seller_currency = kSellerCurrency, + .highest_scoring_other_bid = 0.1776, + .highest_scoring_other_bid_ig_owner = kEmptyString, + .has_highest_scoring_other_bid = false, + .winning_score = 0.0, + .winning_ad_render_url = kEmptyString, + .rejection_reason_map = {}}; + DebugReportingPlaceholder placeholder = + GetPlaceholderDataForInterestGroup(kTestIgOwner, kTestIgName, signals); + EXPECT_FALSE(placeholder.made_winning_bid); + EXPECT_FALSE(placeholder.made_highest_scoring_other_bid); + EXPECT_EQ(placeholder.winning_bid, kIncomingBidInSellerCurrency); EXPECT_EQ(placeholder.highest_scoring_other_bid, signals.highest_scoring_other_bid); + EXPECT_EQ(placeholder.highest_scoring_other_bid_currency, kSellerCurrency); EXPECT_EQ(placeholder.rejection_reason, SellerRejectionReason::SELLER_REJECTION_REASON_NOT_AVAILABLE); } TEST(GetPlaceholderDataForInterestGroupOwnerTest, IgOwnerIsWinner) { float winning_bid = 1.9; - absl::flat_hash_map> - rejection_reason_map; PostAuctionSignals signals = { .winning_ig_name = kTestIgName, .winning_ig_owner = kTestIgOwner, .winning_bid = winning_bid, + .winning_bid_in_seller_currency = 0.0, + .seller_currency = "", .highest_scoring_other_bid = 0.0, .highest_scoring_other_bid_ig_owner = kEmptyString, .has_highest_scoring_other_bid = false, .winning_score = 0.0, .winning_ad_render_url = kEmptyString, - .rejection_reason_map = std::move(rejection_reason_map)}; + .rejection_reason_map = {}}; DebugReportingPlaceholder placeholder = GetPlaceholderDataForInterestGroup(kTestIgOwner, kTestIgName, signals); @@ -347,19 +376,18 @@ TEST(GetPlaceholderDataForInterestGroupOwnerTest, IgOwnerIsWinner) { TEST(GetPlaceholderDataForInterestGroupOwnerTest, IgOwnerMadeHighestOtherBid) { float winning_bid = 1.9; float highest_scoring_other_bid = 2.18; - absl::flat_hash_map> - rejection_reason_map; PostAuctionSignals signals = { .winning_ig_name = kWinningIgName, .winning_ig_owner = kWinningIgOwner, .winning_bid = winning_bid, + .winning_bid_in_seller_currency = 0.0, + .seller_currency = "", .highest_scoring_other_bid = highest_scoring_other_bid, .highest_scoring_other_bid_ig_owner = kTestIgOwner, .has_highest_scoring_other_bid = true, .winning_score = 0.0, .winning_ad_render_url = kEmptyString, - .rejection_reason_map = std::move(rejection_reason_map)}; + .rejection_reason_map = {}}; DebugReportingPlaceholder placeholder = GetPlaceholderDataForInterestGroup(kTestIgOwner, kWinningIgName, signals); @@ -371,19 +399,18 @@ TEST(GetPlaceholderDataForInterestGroupOwnerTest, IgOwnerMadeHighestOtherBid) { TEST(GetPlaceholderDataForInterestGroupOwnerTest, IgOwnerIsBoth) { float winning_bid = 1.9; float highest_scoring_other_bid = 2.18; - absl::flat_hash_map> - rejection_reason_map; PostAuctionSignals signals = { .winning_ig_name = kWinningIgName, .winning_ig_owner = kWinningIgOwner, .winning_bid = winning_bid, + .winning_bid_in_seller_currency = 0.0, + .seller_currency = "", .highest_scoring_other_bid = highest_scoring_other_bid, .highest_scoring_other_bid_ig_owner = kWinningIgOwner, .has_highest_scoring_other_bid = true, .winning_score = 0.0, .winning_ad_render_url = kEmptyString, - .rejection_reason_map = std::move(rejection_reason_map)}; + .rejection_reason_map = {}}; absl::string_view test_ig_name = "test_ig_name"; DebugReportingPlaceholder placeholder = GetPlaceholderDataForInterestGroup( @@ -410,6 +437,8 @@ TEST(GetPlaceholderDataForInterestGroupOwnerTest, IgHasRejectionReason) { .winning_ig_name = kWinningIgName, .winning_ig_owner = kWinningIgOwner, .winning_bid = winning_bid, + .winning_bid_in_seller_currency = 0.0, + .seller_currency = "", .highest_scoring_other_bid = 0.0, .highest_scoring_other_bid_ig_owner = kEmptyString, .has_highest_scoring_other_bid = false, diff --git a/services/inference_sidecar/README.md b/services/inference_sidecar/README.md index 3527c084..c3179dcb 100644 --- a/services/inference_sidecar/README.md +++ b/services/inference_sidecar/README.md @@ -42,14 +42,33 @@ variable with `--//:inference_runtime=pytorch`. ## Start the B&A servers in GCP - Create a GCS bucket and store ML models into it. -- Set runtime flags: `INFERENCE_SIDECAR_BINARY_PATH`, `INFERENCE_MODEL_BUCKET_NAME`, - `INFERENCE_MODEL_BUCKET_PATHS`, and `INFERENCE_SIDECAR_RUNTIME_CONFIG`. +- The GCS bucket needs to give the service account associated with the GCE instance both storage + legacy bucket reader and storage legacy object reader permissions: + [guide on setting bucket permissions](https://cloud.google.com/storage/docs/access-control/using-iam-permissions#bucket-add). +- Now you need to deploy a B&A server stack: + [deployment guide](https://github.com/privacysandbox/bidding-auction-servers/tree/main/production/deploy/gcp/terraform/environment/demo/README.md). +- In the B&A deployment terraform configuration, set runtime flags: + - Set `INFERENCE_SIDECAR_BINARY_PATH` to `/server/bin/inference_sidecar`. - - `INFERENCE_SIDECAR_RUNTIME_CONFIG` has the following format: { "num_interop_threads": - , "num_intraop_threads": , "module_name": , } - Currently "module_name" can be one of "test", "tensorflow_v2_14_0", "pytorch_v2_1_1". -- Refer to - [README.md](https://github.com/privacysandbox/bidding-auction-servers/tree/main/production/deploy/gcp/terraform/environment/demo/README.md). + - Set `INFERENCE_MODEL_BUCKET_NAME` to the name of the GCS bucket that you have created. Note + that this _not_ a url. For example, the bucket name can be "test_models". + - Set `INFERENCE_MODEL_BUCKET_PATHS` to a comma separated list of model paths under the + bucket. For example, within the "test_models" bucket there are saved models + "tensorflow/model1" and "tensorflow/model1". To load these two models, this flag should be + set to "tensorflow/model1,tensorflow/model2". + - Set `INFERENCE_SIDECAR_RUNTIME_CONFIG` which has the following format: + + ```javascript + { + "num_interop_threads": , + "num_intraop_threads": , + "module_name": , + "cpuset": + } + ``` + + `module_name` flag is required and should be one of "test", "tensorflow_v2_14_0", or + "pytorch_v2_1_1". All other flags are optional. ## Start the B&A servers locally for testing/debugging diff --git a/services/inference_sidecar/common/BUILD b/services/inference_sidecar/common/BUILD index 95c8911e..8e3d8b9b 100644 --- a/services/inference_sidecar/common/BUILD +++ b/services/inference_sidecar/common/BUILD @@ -48,6 +48,7 @@ cc_library( "//proto:inference_sidecar_cc_proto", "//sandbox:sandbox_worker", "//utils:cpu", + "//utils:log", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/log", "@com_google_absl//absl/log:absl_log", @@ -113,9 +114,34 @@ cc_test( ], ) +cc_test( + name = "consented_logging_test", + srcs = ["consented_logging_test.cc"], + data = [ + ":gen_test_model", + ":inference_sidecar_test_target", + ], + deps = [ + ":test_constants", + "//proto:inference_sidecar_cc_grpc_proto", + "//proto:inference_sidecar_cc_proto", + "//sandbox:sandbox_executor", + "//utils:file_util", + "@com_github_grpc_grpc//:grpc++", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:reflection", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +# Used to faciliate setting common constants in both inside the (test module) +# inference sidecar and the test that invokes the sidecar. As a result, this +# library cannot be marked as testonly. cc_library( name = "test_constants", - testonly = True, hdrs = ["test_constants.h"], deps = [ "@com_google_absl//absl/strings", diff --git a/services/inference_sidecar/common/consented_logging_test.cc b/services/inference_sidecar/common/consented_logging_test.cc new file mode 100644 index 00000000..10ad2bcc --- /dev/null +++ b/services/inference_sidecar/common/consented_logging_test.cc @@ -0,0 +1,130 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/flags/reflection.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "proto/inference_sidecar.grpc.pb.h" +#include "proto/inference_sidecar.pb.h" +#include "sandbox/sandbox_executor.h" +#include "utils/file_util.h" + +#include "test_constants.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { +namespace { + +using ::testing::HasSubstr; +using ::testing::IsEmpty; +using ::testing::Not; + +constexpr absl::string_view kInferenceSidecarBinary = "inference_sidecar"; +constexpr absl::string_view kTestModelPath = "test_model"; +constexpr char kJsonString[] = R"json({ + "request" : [{ + "model_path" : "test_model", + "tensors" : [ + { + "tensor_name": "serving_default_double1:0", + "data_type": "DOUBLE", + "tensor_shape": [ + 1, + 1 + ], + "tensor_content": ["3.14"] + } + ] +}] + })json"; + +class ConsentedLoggingTest : public ::testing::Test { + protected: + void SetUp() override { + flag_saver_ = std::make_unique(); + absl::SetFlag(&FLAGS_testonly_allow_policies_for_bazel, true); + + executor_ = std::make_unique( + kInferenceSidecarBinary, + std::vector{std::string(kRuntimeConfig)}); + ASSERT_EQ(executor_->StartSandboxee().code(), absl::StatusCode::kOk); + + client_channel_ = grpc::CreateInsecureChannelFromFd( + "GrpcChannel", executor_->FileDescriptor()); + stub_ = InferenceService::NewStub(client_channel_); + + RegisterModelRequest register_model_request; + ASSERT_TRUE( + PopulateRegisterModelRequest(kTestModelPath, register_model_request) + .ok()); + + grpc::ClientContext context; + RegisterModelResponse register_model_response; + grpc::Status status = stub_->RegisterModel(&context, register_model_request, + ®ister_model_response); + ASSERT_TRUE(status.ok()) << status.error_message(); + } + + void TearDown() override { + absl::StatusOr result = executor_->StopSandboxee(); + ASSERT_TRUE(result.ok()); + ASSERT_EQ(result->final_status(), sandbox2::Result::EXTERNAL_KILL); + ASSERT_EQ(result->reason_code(), 0); + } + + std::unique_ptr flag_saver_; + std::unique_ptr executor_; + std::shared_ptr client_channel_; + std::unique_ptr stub_; +}; + +TEST_F(ConsentedLoggingTest, LogIfConsented) { + PredictRequest predict_request; + predict_request.set_input(kJsonString); + predict_request.set_is_consented(true); + PredictResponse predict_response; + + grpc::ClientContext context; + grpc::Status status = + stub_->Predict(&context, predict_request, &predict_response); + ASSERT_TRUE(status.ok()) << status.error_message(); + + ASSERT_THAT(predict_response.debug_info().logs(), Not(IsEmpty())); + EXPECT_THAT(predict_response.debug_info().logs(0), + HasSubstr(kConsentedLogMsg)); +} + +TEST_F(ConsentedLoggingTest, NoConsentSkipLogging) { + PredictRequest predict_request; + predict_request.set_input(kJsonString); + predict_request.set_is_consented(false); + PredictResponse predict_response; + + grpc::ClientContext context; + grpc::Status status = + stub_->Predict(&context, predict_request, &predict_response); + ASSERT_TRUE(status.ok()) << status.error_message(); + + EXPECT_THAT(predict_response.debug_info().logs(), IsEmpty()); +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/grpc_sidecar.cc b/services/inference_sidecar/common/grpc_sidecar.cc index 083f600a..0a8b4c98 100644 --- a/services/inference_sidecar/common/grpc_sidecar.cc +++ b/services/inference_sidecar/common/grpc_sidecar.cc @@ -26,10 +26,12 @@ #include #include +#include "absl/container/flat_hash_set.h" #include "absl/log/absl_log.h" #include "absl/log/check.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" +#include "absl/synchronization/mutex.h" #include "absl/time/clock.h" #include "modules/module_interface.h" #include "proto/inference_sidecar.grpc.pb.h" @@ -38,6 +40,7 @@ #include "sandboxed_api/sandbox2/comms.h" #include "src/util/status_macro/status_util.h" #include "utils/cpu.h" +#include "utils/log.h" namespace privacy_sandbox::bidding_auction_servers::inference { namespace { @@ -56,24 +59,51 @@ class InferenceServiceImpl final : public InferenceService::Service { if (!register_model_response.ok()) { return server_common::FromAbslStatus(register_model_response.status()); } + + // save the model path since it was registered successfully + absl::WriterMutexLock write_model_paths_lock(&model_paths_mutex_); + model_paths_.insert(request->model_spec().model_path()); + *response = register_model_response.value(); + return grpc::Status::OK; } grpc::Status Predict(grpc::ServerContext* context, const PredictRequest* request, PredictResponse* response) override { + RequestContext request_context( + [response] { return response->mutable_debug_info(); }, + request->is_consented()); absl::StatusOr predict_response = - inference_module_->Predict(*request); + inference_module_->Predict(*request, request_context); if (!predict_response.ok()) { + ABSL_LOG(ERROR) << predict_response.status().message(); return server_common::FromAbslStatus(predict_response.status()); } - *response = predict_response.value(); + *(response->mutable_output()) = predict_response->output(); + return grpc::Status::OK; + } + + // TODO: b/348968123) - Relook at API implementation + grpc::Status GetModelPaths(grpc::ServerContext* context, + const GetModelPathsRequest* request, + GetModelPathsResponse* response) override { + absl::ReaderMutexLock read_model_paths_lock(&model_paths_mutex_); + + for (std::string model_path : model_paths_) { + ModelSpec* model_spec = response->add_model_specs(); + model_spec->set_model_path(model_path); + } + return grpc::Status::OK; } private: std::unique_ptr inference_module_; + mutable absl::Mutex model_paths_mutex_; + absl::flat_hash_set model_paths_ + ABSL_GUARDED_BY(model_paths_mutex_); }; } // namespace @@ -86,6 +116,16 @@ absl::Status SetCpuAffinity(const InferenceSidecarRuntimeConfig& config) { return SetCpuAffinity(cpuset); } +absl::Status EnforceModelResetProbability( + InferenceSidecarRuntimeConfig& config) { + if (config.model_reset_probability() != 0.0) { + return absl::InvalidArgumentError( + "model_reset_probability should not be set"); + } + config.set_model_reset_probability(kMinResetProbability); + return absl::OkStatus(); +} + absl::Status Run(const InferenceSidecarRuntimeConfig& config) { SandboxWorker worker; grpc::ServerBuilder builder; diff --git a/services/inference_sidecar/common/grpc_sidecar.h b/services/inference_sidecar/common/grpc_sidecar.h index af9faee4..b7b11aca 100644 --- a/services/inference_sidecar/common/grpc_sidecar.h +++ b/services/inference_sidecar/common/grpc_sidecar.h @@ -20,8 +20,15 @@ namespace privacy_sandbox::bidding_auction_servers::inference { +// ML models are reset at the probability of 0.1%. +constexpr double kMinResetProbability = 0.001; + absl::Status SetCpuAffinity(const InferenceSidecarRuntimeConfig& config); +// Makes sure model_reset_probability must be set to kMinResetProbability. +absl::Status EnforceModelResetProbability( + InferenceSidecarRuntimeConfig& config); + // Runs a simple gRPC server. It is thread safe. absl::Status Run(const InferenceSidecarRuntimeConfig& config); diff --git a/services/inference_sidecar/common/inference_sidecar_main.cc b/services/inference_sidecar/common/inference_sidecar_main.cc index d193bd36..49bfae45 100644 --- a/services/inference_sidecar/common/inference_sidecar_main.cc +++ b/services/inference_sidecar/common/inference_sidecar_main.cc @@ -33,6 +33,11 @@ int main(int argc, char** argv) { config) .ok()) << "Could not set CPU affinity."; + CHECK(privacy_sandbox::bidding_auction_servers::inference:: + EnforceModelResetProbability(config) + .ok()) + << "Could not set the model reset probability"; + if (absl::Status run_status = privacy_sandbox::bidding_auction_servers::inference::Run(config); !run_status.ok()) { diff --git a/services/inference_sidecar/common/model/BUILD b/services/inference_sidecar/common/model/BUILD new file mode 100644 index 00000000..51a757d4 --- /dev/null +++ b/services/inference_sidecar/common/model/BUILD @@ -0,0 +1,44 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "model_store", + hdrs = ["model_store.h"], + deps = [ + "//proto:inference_sidecar_cc_proto", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/functional:any_invocable", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + ], +) + +cc_test( + name = "model_store_test", + srcs = ["model_store_test.cc"], + deps = [ + ":model_store", + "//proto:inference_sidecar_cc_proto", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/services/inference_sidecar/common/model/model_store.h b/services/inference_sidecar/common/model/model_store.h new file mode 100644 index 00000000..82180262 --- /dev/null +++ b/services/inference_sidecar/common/model/model_store.h @@ -0,0 +1,156 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_INFERENCE_SIDECAR_COMMON_MODEL_STORE_MODEL_STORE_H_ +#define SERVICES_INFERENCE_SIDECAR_COMMON_MODEL_STORE_MODEL_STORE_H_ + +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/functional/any_invocable.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/synchronization/mutex.h" +#include "proto/inference_sidecar.pb.h" +#include "src/util/status_macro/status_macros.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { + +class ModelStoreTest; + +// Class that manages models for the inference sidecar. It keeps shared +// ownership of the models. The accepted models can either be ones that accept +// consented traffic and ones that accept production traffic. The interface is +// thread-safe, assuming the ModelType also exposes a thread-safe interface. +template +class ModelStore { + public: + using ModelConstructor = + absl::AnyInvocable>( + const InferenceSidecarRuntimeConfig&, const RegisterModelRequest&) + const>; + + explicit ModelStore(const InferenceSidecarRuntimeConfig& config, + ModelConstructor model_constructor) + : config_(config), model_constructor_(std::move(model_constructor)) {} + + // Puts model into the model store. Creates both a copy for consented traffic + // and a copy for prod traffic. Future reset calls use the saved RegisterModel + // Request. Overwrites a model entry if the key already exists. + // This method is thread-safe. + absl::Status PutModel(absl::string_view key, + const RegisterModelRequest& request) { + PS_ASSIGN_OR_RETURN(std::unique_ptr prod_model, + model_constructor_(config_, request)); + PS_ASSIGN_OR_RETURN(std::unique_ptr consented_model, + model_constructor_(config_, request)); + + absl::MutexLock model_data_lock(&model_data_mutex_); + model_data_map_[key] = request; + + absl::MutexLock prod_model_lock(&prod_model_mutex_); + prod_model_map_[key] = std::move(prod_model); + + absl::MutexLock consented_model_lock(&consented_model_mutex_); + consented_model_map_[key] = std::move(consented_model); + + return absl::OkStatus(); + } + + // Gets a model for serving. Returns an error status if a given key is not + // found. + // This method is thread-safe. + absl::StatusOr> GetModel( + absl::string_view key, bool is_consented = false) const { + const absl::flat_hash_map>& + model_map = is_consented ? consented_model_map_ : prod_model_map_; + absl::MutexLock lock(is_consented ? &consented_model_mutex_ + : &prod_model_mutex_); + + auto it = model_map.find(key); + if (it == model_map.end()) { + return absl::NotFoundError( + absl::StrCat("Requested model '", key, "' has not been registered")); + } + return it->second; + } + + // Reset a model entry using the model constructor. + // This method is thread-safe. + absl::Status ResetModel(absl::string_view key, bool is_consented = false) { + absl::MutexLock model_data_lock(&model_data_mutex_); + auto it = model_data_map_.find(key); + if (it == model_data_map_.end()) { + return absl::NotFoundError( + absl::StrCat("Resetting model '", key, + "' fails because it has not been registered")); + } + const RegisterModelRequest& request = it->second; + PS_ASSIGN_OR_RETURN(std::unique_ptr model, + model_constructor_(config_, request)); + + absl::flat_hash_map>& model_map = + is_consented ? consented_model_map_ : prod_model_map_; + absl::MutexLock model_lock(is_consented ? &consented_model_mutex_ + : &prod_model_mutex_); + model_map[key] = std::move(model); + return absl::OkStatus(); + } + + std::vector ListModels() const { + std::vector model_keys; + absl::MutexLock model_data_lock(&model_data_mutex_); + for (auto& [key, value] : model_data_map_) { + model_keys.push_back(key); + } + return model_keys; + } + + ModelStore(const ModelStore&) = delete; + ModelStore& operator=(const ModelStore&) = delete; + + ModelStore(ModelStore&&) = delete; + ModelStore& operator=(ModelStore&&) = delete; + + protected: + // Used for test only, the caller needs to ensure thread-safety. + void SetModelConstructorForTestOnly(ModelConstructor model_constructor) { + model_constructor_ = std::move(model_constructor); + } + + private: + // Always lock on `model_data_mutex_` before `prod_model_mutex_` and + // then `consented_model_mutex_` to avoid deadlock. + mutable absl::Mutex model_data_mutex_ ABSL_ACQUIRED_BEFORE(prod_model_mutex_); + mutable absl::Mutex prod_model_mutex_ + ABSL_ACQUIRED_BEFORE(consented_model_mutex_); + mutable absl::Mutex consented_model_mutex_; + absl::flat_hash_map model_data_map_ + ABSL_GUARDED_BY(model_data_mutex_); + absl::flat_hash_map> prod_model_map_ + ABSL_GUARDED_BY(prod_model_mutex_); + absl::flat_hash_map> + consented_model_map_ ABSL_GUARDED_BY(consented_model_mutex_); + + const InferenceSidecarRuntimeConfig config_; + ModelConstructor model_constructor_; +}; + +} // namespace privacy_sandbox::bidding_auction_servers::inference + +#endif // SERVICES_INFERENCE_SIDECAR_COMMON_MODEL_STORE_MODEL_STORE_H_ diff --git a/services/inference_sidecar/common/model/model_store_test.cc b/services/inference_sidecar/common/model/model_store_test.cc new file mode 100644 index 00000000..371267c0 --- /dev/null +++ b/services/inference_sidecar/common/model/model_store_test.cc @@ -0,0 +1,407 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "model/model_store.h" + +#include +#include + +#include +#include +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "proto/inference_sidecar.pb.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { +namespace { + +constexpr absl::string_view kTestModelName = "test_model"; +constexpr absl::string_view kNonExistModelName = "non_exist_model"; +constexpr int kNumThreads = 100; + +class MockModel { + public: + MockModel() : counter_(0) {} + explicit MockModel(int init_counter) : counter_(init_counter) {} + + void Increment() { + absl::MutexLock lock(&mutex_); + counter_++; + } + + int GetCounter() { + absl::MutexLock lock(&mutex_); + return counter_; + } + + private: + int counter_; + mutable absl::Mutex mutex_; +}; + +absl::StatusOr> MockModelConstructor( + const InferenceSidecarRuntimeConfig& config, + const RegisterModelRequest& request) { + return std::make_unique(); +} + +absl::StatusOr> FailureModelConstructor( + const InferenceSidecarRuntimeConfig& config, + const RegisterModelRequest& request) { + return absl::FailedPreconditionError("Error"); +} + +class MockModelStore : public ModelStore { + public: + using ModelStore::ModelStore; + using ModelStore::PutModel; + using ModelStore::GetModel; + using ModelStore::ResetModel; + using ModelStore::SetModelConstructorForTestOnly; +}; + +class ModelStoreTest : public ::testing::Test { + protected: + ModelStoreTest() + : store_(InferenceSidecarRuntimeConfig(), MockModelConstructor) {} + + MockModelStore store_; +}; + +TEST_F(ModelStoreTest, PutModelAddsModel) { + RegisterModelRequest request; + EXPECT_TRUE(store_.PutModel(kTestModelName, request).ok()); + EXPECT_TRUE(store_.GetModel(kTestModelName, /*is_consented=*/false).ok()); + EXPECT_TRUE(store_.GetModel(kTestModelName, /*is_consented=*/true).ok()); +} + +TEST_F(ModelStoreTest, PutModelFailureDoesNotWrite) { + store_.SetModelConstructorForTestOnly(FailureModelConstructor); + RegisterModelRequest request; + EXPECT_FALSE(store_.PutModel(kTestModelName, request).ok()); + EXPECT_FALSE(store_.GetModel(kTestModelName, /*is_consented=*/false).ok()); + EXPECT_FALSE(store_.GetModel(kTestModelName, /*is_consented=*/true).ok()); +} + +TEST_F(ModelStoreTest, GetNonExistentModelReturnsNotFound) { + absl::StatusOr> model = + store_.GetModel(kNonExistModelName); + ASSERT_FALSE(model.ok()); + EXPECT_EQ(model.status().code(), absl::StatusCode::kNotFound); +} + +TEST_F(ModelStoreTest, ResetModelSuccess) { + RegisterModelRequest request; + ASSERT_TRUE(store_.PutModel(kTestModelName, request).ok()); + + absl::StatusOr> result1 = + store_.GetModel(kTestModelName); + ASSERT_TRUE(result1.ok()); + (*result1)->Increment(); + + ASSERT_TRUE(store_.ResetModel(kTestModelName).ok()); + + absl::StatusOr> result2 = + store_.GetModel(kTestModelName); + ASSERT_TRUE(result2.ok()); + EXPECT_EQ((*result2)->GetCounter(), 0); +} + +TEST_F(ModelStoreTest, ResetModelFailureDoesNotOverwrite) { + RegisterModelRequest request; + ASSERT_TRUE(store_.PutModel(kTestModelName, request).ok()); + + absl::StatusOr> result1 = + store_.GetModel(kTestModelName); + ASSERT_TRUE(result1.ok()); + (*result1)->Increment(); + + store_.SetModelConstructorForTestOnly(FailureModelConstructor); + ASSERT_FALSE(store_.ResetModel(kTestModelName).ok()); + + absl::StatusOr> result2 = + store_.GetModel(kTestModelName); + ASSERT_TRUE(result2.ok()); + EXPECT_EQ((*result2)->GetCounter(), 1); +} + +TEST_F(ModelStoreTest, ResetNonExistentModelReturnsNotFound) { + absl::Status status = store_.ResetModel(kNonExistModelName); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), absl::StatusCode::kNotFound); +} + +TEST_F(ModelStoreTest, ListModels) { + EXPECT_TRUE(store_.ListModels().empty()); + + RegisterModelRequest request; + EXPECT_TRUE(store_.PutModel(kTestModelName, request).ok()); + std::vector models = store_.ListModels(); + EXPECT_EQ(models.size(), 1); + EXPECT_EQ(models.front(), kTestModelName); +} + +absl::StatusOr> MockModelConstructorWithInitCounter( + const InferenceSidecarRuntimeConfig& config, + const RegisterModelRequest& request) { + int init_counter; + auto it = request.model_files().find(kTestModelName); + if (it == request.model_files().end() || + !absl::SimpleAtoi(it->second, &init_counter)) { + return absl::FailedPreconditionError("Error"); + } + return std::make_unique(init_counter); +} + +class ModelStoreConcurrencyTest : public ::testing::Test { + protected: + ModelStoreConcurrencyTest() + : store_(InferenceSidecarRuntimeConfig(), + MockModelConstructorWithInitCounter) {} + + MockModelStore store_; +}; + +RegisterModelRequest BuildMockModelRegisterModelRequest(int init_counter) { + RegisterModelRequest request; + (*request.mutable_model_files())[kTestModelName] = absl::StrCat(init_counter); + return request; +} + +TEST_F(ModelStoreConcurrencyTest, + ConcurrentPutModelEnsuresConsistentModelAcrossProdAndNonProd) { + std::vector threads; + threads.reserve(kNumThreads * 2); + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back(std::thread([this]() { + EXPECT_TRUE( + this->store_ + .PutModel(kTestModelName, BuildMockModelRegisterModelRequest(1)) + .ok()); + })); + threads.push_back(std::thread([this]() { + EXPECT_TRUE( + this->store_ + .PutModel(kTestModelName, BuildMockModelRegisterModelRequest(2)) + .ok()); + })); + } + + for (auto& thread : threads) { + thread.join(); + } + + absl::StatusOr> prod_model = + store_.GetModel(kTestModelName, /*is_consented=*/false); + ASSERT_TRUE(prod_model.ok()); + int prod_model_counter = (*prod_model)->GetCounter(); + + absl::StatusOr> consented_model = + store_.GetModel(kTestModelName, /*is_consented=*/true); + ASSERT_TRUE(consented_model.ok()); + int consented_model_counter = (*consented_model)->GetCounter(); + + EXPECT_EQ(prod_model_counter, consented_model_counter); +} + +TEST_F(ModelStoreConcurrencyTest, ConcurrentGetModelSuccess) { + EXPECT_TRUE( + store_.PutModel(kTestModelName, BuildMockModelRegisterModelRequest(1)) + .ok()); + + std::vector threads; + threads.reserve(kNumThreads * 2); + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back(std::thread([this]() { + absl::StatusOr> prod_model = + store_.GetModel(kTestModelName, /*is_consented=*/false); + ASSERT_TRUE(prod_model.ok()); + EXPECT_EQ((*prod_model)->GetCounter(), 1); + })); + threads.push_back(std::thread([this]() { + absl::StatusOr> consented_model = + store_.GetModel(kTestModelName, /*is_consented=*/true); + ASSERT_TRUE(consented_model.ok()); + EXPECT_EQ((*consented_model)->GetCounter(), 1); + })); + } + + for (auto& thread : threads) { + thread.join(); + } +} + +TEST_F(ModelStoreConcurrencyTest, ConcurrentResetModelSuccess) { + EXPECT_TRUE( + store_.PutModel(kTestModelName, BuildMockModelRegisterModelRequest(0)) + .ok()); + + absl::StatusOr> prod_model_result_1 = + store_.GetModel(kTestModelName, /*is_consented=*/false); + ASSERT_TRUE(prod_model_result_1.ok()); + (*prod_model_result_1)->Increment(); + + absl::StatusOr> consented_model_result_1 = + store_.GetModel(kTestModelName, /*is_consented=*/true); + ASSERT_TRUE(consented_model_result_1.ok()); + (*consented_model_result_1)->Increment(); + + std::vector threads; + threads.reserve(kNumThreads * 2); + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back(std::thread([this]() { + EXPECT_TRUE( + this->store_.ResetModel(kTestModelName, /*is_consented=*/false).ok()); + })); + threads.push_back(std::thread([this]() { + EXPECT_TRUE( + this->store_.ResetModel(kTestModelName, /*is_consented=*/true).ok()); + })); + } + + for (auto& thread : threads) { + thread.join(); + } + + absl::StatusOr> prod_model_result_2 = + store_.GetModel(kTestModelName, /*is_consented=*/false); + ASSERT_TRUE(prod_model_result_2.ok()); + EXPECT_EQ((*prod_model_result_2)->GetCounter(), 0); + + absl::StatusOr> consented_model_result_2 = + store_.GetModel(kTestModelName, /*is_consented=*/true); + ASSERT_TRUE(consented_model_result_2.ok()); + EXPECT_EQ((*consented_model_result_2)->GetCounter(), 0); +} + +TEST_F(ModelStoreConcurrencyTest, + GetModelReturnsModelDuringConcurrentResetModel) { + EXPECT_TRUE( + store_.PutModel(kTestModelName, BuildMockModelRegisterModelRequest(1)) + .ok()); + + std::vector threads; + threads.reserve(kNumThreads * 4); + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back(std::thread([this]() { + EXPECT_TRUE( + this->store_.ResetModel(kTestModelName, /*is_consented=*/true).ok()); + })); + threads.push_back(std::thread([this]() { + EXPECT_TRUE( + this->store_.ResetModel(kTestModelName, /*is_consented=*/false).ok()); + })); + threads.push_back(std::thread([this]() { + absl::StatusOr> model = + store_.GetModel(kTestModelName, /*is_consented=*/true); + ASSERT_TRUE(model.ok()); + (*model)->Increment(); + })); + threads.push_back(std::thread([this]() { + absl::StatusOr> model = + store_.GetModel(kTestModelName, /*is_consented=*/false); + ASSERT_TRUE(model.ok()); + (*model)->Increment(); + })); + } + + for (auto& thread : threads) { + thread.join(); + } +} + +TEST_F(ModelStoreConcurrencyTest, + GetModelReturnsModelDuringConcurrentPutModel) { + EXPECT_TRUE( + store_.PutModel(kTestModelName, BuildMockModelRegisterModelRequest(1)) + .ok()); + + std::vector threads; + threads.reserve(kNumThreads * 3); + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back(std::thread([this]() { + EXPECT_TRUE( + this->store_ + .PutModel(kTestModelName, BuildMockModelRegisterModelRequest(1)) + .ok()); + })); + threads.push_back(std::thread([this]() { + absl::StatusOr> model = + store_.GetModel(kTestModelName, /*is_consented=*/true); + ASSERT_TRUE(model.ok()); + (*model)->Increment(); + })); + threads.push_back(std::thread([this]() { + absl::StatusOr> model = + store_.GetModel(kTestModelName, /*is_consented=*/false); + ASSERT_TRUE(model.ok()); + (*model)->Increment(); + })); + } + + for (auto& thread : threads) { + thread.join(); + } +} + +TEST_F(ModelStoreConcurrencyTest, + GetModelReturnsModelDuringConcurrentPutModelAndResetModel) { + EXPECT_TRUE( + store_.PutModel(kTestModelName, BuildMockModelRegisterModelRequest(1)) + .ok()); + + std::vector threads; + threads.reserve(kNumThreads * 5); + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back(std::thread([this]() { + EXPECT_TRUE( + this->store_ + .PutModel(kTestModelName, BuildMockModelRegisterModelRequest(1)) + .ok()); + })); + threads.push_back(std::thread([this]() { + EXPECT_TRUE( + this->store_.ResetModel(kTestModelName, /*is_consented=*/true).ok()); + })); + threads.push_back(std::thread([this]() { + EXPECT_TRUE( + this->store_.ResetModel(kTestModelName, /*is_consented=*/false).ok()); + })); + threads.push_back(std::thread([this]() { + absl::StatusOr> model = + store_.GetModel(kTestModelName, /*is_consented=*/true); + ASSERT_TRUE(model.ok()); + (*model)->Increment(); + })); + threads.push_back(std::thread([this]() { + absl::StatusOr> model = + store_.GetModel(kTestModelName, /*is_consented=*/false); + ASSERT_TRUE(model.ok()); + (*model)->Increment(); + })); + } + + for (auto& thread : threads) { + thread.join(); + } +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/modules/BUILD b/services/inference_sidecar/common/modules/BUILD index d36eda89..583028b7 100644 --- a/services/inference_sidecar/common/modules/BUILD +++ b/services/inference_sidecar/common/modules/BUILD @@ -27,6 +27,7 @@ cc_library( hdrs = ["test_module.h"], deps = [ ":module_interface", + "//:test_constants", "//proto:inference_sidecar_cc_proto", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/log:check", @@ -59,6 +60,7 @@ cc_library( ], deps = [ "//proto:inference_sidecar_cc_proto", + "//utils:log", "@com_google_absl//absl/status:statusor", ], ) diff --git a/services/inference_sidecar/common/modules/module_interface.h b/services/inference_sidecar/common/modules/module_interface.h index 65210e05..d54d015c 100644 --- a/services/inference_sidecar/common/modules/module_interface.h +++ b/services/inference_sidecar/common/modules/module_interface.h @@ -21,6 +21,7 @@ #include "absl/status/statusor.h" #include "proto/inference_sidecar.pb.h" +#include "utils/log.h" namespace privacy_sandbox::bidding_auction_servers::inference { @@ -36,10 +37,13 @@ class ModuleInterface { virtual ~ModuleInterface() = default; // Executes inference on a registered ML model. virtual absl::StatusOr Predict( - const PredictRequest& request) = 0; + const PredictRequest& request, + const RequestContext& request_context = RequestContext()) = 0; // Registers a new model. virtual absl::StatusOr RegisterModel( const RegisterModelRequest& request) = 0; + // Resets models. + virtual void ResetModels() = 0; }; } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/modules/test_module.cc b/services/inference_sidecar/common/modules/test_module.cc index b4ef9ba3..73f30783 100644 --- a/services/inference_sidecar/common/modules/test_module.cc +++ b/services/inference_sidecar/common/modules/test_module.cc @@ -30,6 +30,8 @@ #include "proto/inference_sidecar.pb.h" #include "sandboxed_api/util/runfiles.h" +#include "test_constants.h" + namespace privacy_sandbox::bidding_auction_servers::inference { TestModule::~TestModule() { @@ -42,7 +44,9 @@ TestModule::~TestModule() { } absl::StatusOr TestModule::Predict( - const PredictRequest& request) { + const PredictRequest& request, const RequestContext& request_context) { + // For testing consented debugging. + INFERENCE_LOG(INFO, request_context) << kConsentedLogMsg; // Returns a placeholder value. PredictResponse response; response.set_output("0.57721"); @@ -75,6 +79,12 @@ absl::StatusOr TestModule::RegisterModel( return response; } +void TestModule::ResetModels() { + if (config_.model_reset_probability() == 0.0) { + ABSL_LOG(INFO) << "Model reset is disabled."; + } +} + std::unique_ptr ModuleInterface::Create( const InferenceSidecarRuntimeConfig& config) { return std::make_unique(config); diff --git a/services/inference_sidecar/common/modules/test_module.h b/services/inference_sidecar/common/modules/test_module.h index 5512b244..c0aedc9f 100644 --- a/services/inference_sidecar/common/modules/test_module.h +++ b/services/inference_sidecar/common/modules/test_module.h @@ -39,9 +39,11 @@ class TestModule final : public ModuleInterface { ~TestModule() override; absl::StatusOr Predict( - const PredictRequest& request) override; + const PredictRequest& request, + const RequestContext& request_context) override; absl::StatusOr RegisterModel( const RegisterModelRequest& request) override; + void ResetModels() override; void set_model_path(absl::string_view path) { model_path_ = path; } diff --git a/services/inference_sidecar/common/modules/test_module_test.cc b/services/inference_sidecar/common/modules/test_module_test.cc index 5fb9b87d..2c1f360c 100644 --- a/services/inference_sidecar/common/modules/test_module_test.cc +++ b/services/inference_sidecar/common/modules/test_module_test.cc @@ -53,5 +53,11 @@ TEST(TestModule, Success_ReadModel) { EXPECT_GT(module->model_size(), 0); } +TEST(TestModule, Success_ResetModels) { + InferenceSidecarRuntimeConfig config; + std::unique_ptr module = std::make_unique(config); + module->ResetModels(); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/proto/inference_sidecar.proto b/services/inference_sidecar/common/proto/inference_sidecar.proto index 823b15a4..89e6537a 100644 --- a/services/inference_sidecar/common/proto/inference_sidecar.proto +++ b/services/inference_sidecar/common/proto/inference_sidecar.proto @@ -23,6 +23,9 @@ service InferenceService { // Registers model. rpc RegisterModel(RegisterModelRequest) returns (RegisterModelResponse) { } + // Gets model paths for successfully registered models. + rpc GetModelPaths(GetModelPathsRequest) returns (GetModelPathsResponse) { + } } message PredictRequest { @@ -57,6 +60,13 @@ message RegisterModelRequest { message RegisterModelResponse { } +message GetModelPathsRequest { +} + +message GetModelPathsResponse { + repeated ModelSpec model_specs = 1; +} + message InferenceSidecarRuntimeConfig { // The following two parameters control the threading behavior of the // inference backend. @@ -77,6 +87,18 @@ message InferenceSidecarRuntimeConfig { // The sidecar process will run only on the specified CPUs. // If `cpuset` is empty, the CPU affinity is not used. repeated int32 cpuset = 4; + + // Internal use only. Please note that it is outside the scope of Adtech's control. + // + // Represents the probability of resetting ML models in the inference sidecar. + // It's the privacy preserving feature to prevent data leak between users. + // Range: 0.0 (never reset) to 1.0 (always reset per inference); and + // 0.5 means the reset probability of 50%. Applied to each model independently. + // + // TODO(b/330362159): Support different reset probability for consented + // traffic. + // TODO(b/348985774): Remove this field. + double model_reset_probability = 5; } // Proto to store consented debugging logs. It's passed back with diff --git a/services/inference_sidecar/common/test_constants.h b/services/inference_sidecar/common/test_constants.h index 23a462fa..fc4cd6d6 100644 --- a/services/inference_sidecar/common/test_constants.h +++ b/services/inference_sidecar/common/test_constants.h @@ -28,6 +28,8 @@ constexpr absl::string_view kRuntimeConfig = R"json({ "cpuset": [0, 1] })json"; +constexpr absl::string_view kConsentedLogMsg = "Consented Log"; + } // namespace privacy_sandbox::bidding_auction_servers::inference #endif // SERVICES_BIDDING_SERVICE_CONSTANTS_H_ diff --git a/services/inference_sidecar/common/testdata/BUILD b/services/inference_sidecar/common/testdata/BUILD index 38871392..53435e74 100644 --- a/services/inference_sidecar/common/testdata/BUILD +++ b/services/inference_sidecar/common/testdata/BUILD @@ -25,4 +25,6 @@ exports_files([ # This pytorch model takes one int64 value and three floats as input. It # returns two tensors: a float embedding of size 4 and one int64 value. "models/pytorch_mixed_inputs_mixed_outputs_model.pt", + # This PyTorch model returns the incrementing counter. + "models/pytorch_stateful_model.pt", ]) diff --git a/services/inference_sidecar/common/testdata/models/pytorch_stateful_model.pt b/services/inference_sidecar/common/testdata/models/pytorch_stateful_model.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6fe88a442059a11d1a59cb307341167721ed52d GIT binary patch literal 2351 zcmWIWW@cev;NW1u0HO@M48kPrW=@&3W-U<9YfgA=3@5eYK1 z&)@rqfxywp?|!8he^#mzC|a|k_3G^=r`FKv5v@^M#W}NQU&_dtUDEaDZ1lcuzDHYI z8y}v1e|GlIkC&KRE}Tu4Y2~bsRVn|Y*(uJy-G+1h`{zk=cFk z$-mbI9Mzl}dYO?uaO>8Yfh&B&*iJ6!dwKKfu>`j*)8Fx1PTBQVVfAjVmWArmOT4~h z{5a~UyukQsTlQ?pct|j zn6A+b!6;JEO+(HTS}3Mn0{WhiX^652-K)r{T?WP6FkmLYFc&F*p__!9azV8s0$jl~ z37S{X%|K3h5(oo;tRPkl*T7Rex+%!1Llwo8YnY~BPqP8uY-~DEt#Zt|a6@4ABa8;; kRiJDID6aqkP#>tyXE+O00E}s%7%NzogMotqq#mLc0CCNlz5oCK literal 0 HcmV?d00001 diff --git a/services/inference_sidecar/common/tools/debug/start_inference b/services/inference_sidecar/common/tools/debug/start_inference index c7cbf445..1811c019 100755 --- a/services/inference_sidecar/common/tools/debug/start_inference +++ b/services/inference_sidecar/common/tools/debug/start_inference @@ -1,10 +1,11 @@ #!/usr/bin/env bash - +# Copyright 2024 Google LLC +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -30,8 +31,8 @@ export PROJECT_ROOT=$(git rev-parse --show-toplevel) # Default mount variables: BIDDING_SERVER_MOUNT=${PROJECT_ROOT}/bazel-bin/services/bidding_service/server:/server -INFERENCE_SIDECAR_MOUNT=${PROJECT_ROOT}/services/inference_sidecar/modules/pytorch_v2_1_1/artifacts/inference_sidecar:/inference_sidecar -INFERENCE_MODEL_MOUNT=${PROJECT_ROOT}/services/inference_sidecar/common/testdata/models/pytorch_e2e_model1.pt:/test_model +INFERENCE_SIDECAR_MOUNT=${PROJECT_ROOT}/services/inference_sidecar/modules/pytorch_v2_1_1/artifacts/inference_sidecar_pytorch_v2_1_1:/inference_sidecar_pytorch_v2_1_1 +INFERENCE_MODEL_MOUNT=${PROJECT_ROOT}/services/inference_sidecar/common/testdata/models/pytorch_e2e_model1.pt:/generate_bid_model # Default client JS code: BIDDING_JS_PATH="/src/workspace/testing/functional/suts/inference_pytorch/data/buyer-bidding-service/generateBidRunInference.js" @@ -110,8 +111,8 @@ echo $EXTRA_DOCKER_RUN_ARGS export SERVER_START_CMD=$(cat << END /server \ ---inference_sidecar_binary_path="/inference_sidecar" \ ---inference_model_local_paths="/test_model" \ +--inference_sidecar_binary_path="/inference_sidecar_pytorch_v2_1_1" \ +--inference_model_local_paths="/generate_bid_model" \ --inference_sidecar_runtime_config='{ "num_interop_threads": 4, "num_intraop_threads": 4, @@ -127,13 +128,12 @@ export SERVER_START_CMD=$(cat << END --buyer_code_fetch_config='{ "fetchMode": 2, "biddingJsPath": "${BIDDING_JS_PATH}", - "protectedAppSignalsBiddingJsUrl": "https://td.doubleclick.net/td/bjs", + "protectedAppSignalsBiddingJsUrl": "${PROTECTED_APP_SIGNALS_BIDDING_JS_URL}", "biddingWasmHelperUrl": "", "protectedAppSignalsBiddingWasmHelperUrl": "", "urlFetchPeriodMs": 13000000, "urlFetchTimeoutMs": 30000, "enableBuyerDebugUrlGeneration": true, - "enableAdtechCodeLogging": false, "prepareDataForAdsRetrievalJsUrl": "", "prepareDataForAdsRetrievalWasmHelperUrl": "" }' \ diff --git a/services/inference_sidecar/common/utils/log.h b/services/inference_sidecar/common/utils/log.h index 695ed836..894c5dfb 100644 --- a/services/inference_sidecar/common/utils/log.h +++ b/services/inference_sidecar/common/utils/log.h @@ -26,22 +26,22 @@ namespace privacy_sandbox::bidding_auction_servers::inference { // The INFERENCE_LOG marcro is used to log to an InferenceDebugInfo proto. -#define INFERENCE_LOG(level, log_context) \ - ABSL_LOG(level).ToSinkOnly(log_context.LogSink()) +#define INFERENCE_LOG(level, request_context) \ + ABSL_LOG(level).ToSinkOnly(request_context.LogSink()) // This class wraps a consented debugging log sink which redirect logs to an // InferenceDebugInfo proto. When ABSL_LOG is invoked on the log sink, logging // can be redirected to it based on user consent. We currently return the // InferenceDebugInfo proto with a PredictResponse to the caller of // InferenceService -class LogContext { +class RequestContext { public: - LogContext(absl::AnyInvocable debug_info, - bool is_consented) + RequestContext(absl::AnyInvocable debug_info, + bool is_consented) : log_sink_(std::make_unique(std::move(debug_info), is_consented)) {} - LogContext() : log_sink_(std::make_unique()) {} + RequestContext() : log_sink_(std::make_unique()) {} absl::LogSink* LogSink() const { return log_sink_.get(); } diff --git a/services/inference_sidecar/common/utils/log_test.cc b/services/inference_sidecar/common/utils/log_test.cc index 039e8db5..dc57fc02 100644 --- a/services/inference_sidecar/common/utils/log_test.cc +++ b/services/inference_sidecar/common/utils/log_test.cc @@ -29,33 +29,33 @@ constexpr absl::string_view kLogContent = "log content"; TEST(LogTest, LogIfConsented) { PredictResponse predict_response; - LogContext log_context( + RequestContext request_context( [&predict_response]() { return predict_response.mutable_debug_info(); }, true); - INFERENCE_LOG(INFO, log_context) << kLogContent; + INFERENCE_LOG(INFO, request_context) << kLogContent; ASSERT_EQ(predict_response.debug_info().logs_size(), 1); EXPECT_THAT(predict_response.debug_info().logs(0), HasSubstr(kLogContent)); } TEST(LogTest, LogNothingIfNoConsent) { PredictResponse predict_response; - LogContext log_context( + RequestContext request_context( [&predict_response]() { return predict_response.mutable_debug_info(); }, false); - INFERENCE_LOG(INFO, log_context) << kLogContent; + INFERENCE_LOG(INFO, request_context) << kLogContent; EXPECT_EQ(predict_response.debug_info().logs_size(), 0); } TEST(LogTest, CanLogMultipleTimes) { PredictResponse predict_response; - LogContext log_context( + RequestContext request_context( [&predict_response]() { return predict_response.mutable_debug_info(); }, true); for (int i = 0; i < 10; ++i) { - INFERENCE_LOG(INFO, log_context) << kLogContent; + INFERENCE_LOG(INFO, request_context) << kLogContent; } EXPECT_EQ(predict_response.debug_info().logs_size(), 10); } diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/BUILD b/services/inference_sidecar/modules/pytorch_v2_1_1/BUILD index ea6daf4a..d0cdbfd0 100644 --- a/services/inference_sidecar/modules/pytorch_v2_1_1/BUILD +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/BUILD @@ -86,12 +86,15 @@ cc_library( ":pytorch_parser", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/random", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", + "@inference_common//model:model_store", "@inference_common//modules:module_interface", "@inference_common//proto:inference_sidecar_cc_proto", + "@inference_common//utils:log", "@inference_common//utils:request_parser", "@pytorch_v2_1_1//:torch", ], @@ -118,6 +121,13 @@ genrule( cmd = "cp $< $@", ) +genrule( + name = "test_stateful_model_target", + srcs = ["@inference_common//testdata:models/pytorch_stateful_model.pt"], + outs = ["stateful_model"], + cmd = "cp $< $@", +) + genrule( name = "test_mixed_inputs_mixed_outputs_target", srcs = ["@inference_common//testdata:models/pytorch_mixed_inputs_mixed_outputs_model.pt"], @@ -134,6 +144,7 @@ cc_test( ":test_e2e_model2_target", ":test_mixed_inputs_mixed_outputs_target", ":test_simple_model_target", + ":test_stateful_model_target", ], deps = [ ":pytorch", @@ -141,6 +152,7 @@ cc_test( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", "@inference_common//proto:inference_sidecar_cc_proto", @@ -207,7 +219,7 @@ genrule( outs = ["inference_sidecar_binary"], cmd_bash = """cat << EOF > '$@' mkdir -p artifacts -cp $(execpath :inference_sidecar_bin) artifacts/inference_sidecar +cp $(execpath :inference_sidecar_bin) artifacts/inference_sidecar_pytorch_v2_1_1 EOF""", executable = True, local = True, @@ -217,7 +229,7 @@ EOF""", # Exports PyTorch inference sidecar binary to be packaged in the B&A workspace. exports_files( [ - "artifacts/inference_sidecar", + "artifacts/inference_sidecar_pytorch_v2_1_1", ], ) diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc index 4c85870c..7f620bc5 100644 --- a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc @@ -14,21 +14,29 @@ * limitations under the License. */ +#include #include #include +#include +#include +#include #include #include "absl/base/thread_annotations.h" #include "absl/container/flat_hash_map.h" +#include "absl/random/random.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" #include "absl/synchronization/mutex.h" +#include "absl/synchronization/notification.h" +#include "model/model_store.h" #include "modules/module_interface.h" #include "proto/inference_sidecar.pb.h" #include "src/util/status_macro/status_macros.h" +#include "utils/log.h" #include "utils/request_parser.h" #include "pytorch_parser.h" @@ -40,8 +48,9 @@ namespace { // the inference result for a single request in a batched predict request. // The forward method of a torch module is non-const although we disallow // mutable models. -absl::StatusOr PredictInternal(torch::jit::script::Module* model, - const InferenceRequest& request) { +absl::StatusOr PredictInternal( + std::shared_ptr model, + const InferenceRequest& request) { absl::string_view model_key = request.model_path; std::vector inputs; for (Tensor tensor : request.inputs) { @@ -94,30 +103,86 @@ absl::Status InitRuntimeThreadConfig( return absl::OkStatus(); } +absl::StatusOr> +PyTorchModelConstructor(const InferenceSidecarRuntimeConfig& config, + const RegisterModelRequest& request) { + absl::string_view model_key = request.model_spec().model_path(); + // Converts PyTorch exception to absl status. + try { + const std::string& model_payload = request.model_files().begin()->second; + std::istringstream is(model_payload); + auto model = + std::make_unique(torch::jit::load(is)); + // Turn on eval model for layers that behave differently during train and + // eval times, for example, dropout and batch norm layers. + model->eval(); + return model; + } catch (...) { + return absl::InternalError("Error loading model"); + } +} + class PyTorchModule final : public ModuleInterface { public: - explicit PyTorchModule(const InferenceSidecarRuntimeConfig& config) { + explicit PyTorchModule(const InferenceSidecarRuntimeConfig& config) + : runtime_config_(config), + store_(config, PyTorchModelConstructor), + background_thread_running_(true) { absl::Status init_result = InitRuntimeThreadConfig(config); CHECK(init_result.ok()) << "Could not initialize runtime flags: " << init_result; + + // TODO(b/332328206): Move reset model logic into the ModelStore. + background_thread_ = std::thread([this]() { + while (background_thread_running_) { + // ResetModels() is continually called in the background thread. + ResetModels(); + // Waits for the completed inference execution. + // + // Moves on after 1 second and check if the background thread should + // terminate. The destructor terminates this thread by setting + // |background_thread_running_|. + inference_notification_.WaitForNotificationWithTimeout( + absl::Seconds(1)); + } + }); + } + + ~PyTorchModule() override { + background_thread_running_ = false; + background_thread_.join(); } absl::StatusOr Predict( - const PredictRequest& request) override ABSL_LOCKS_EXCLUDED(mu_); + const PredictRequest& request, + const RequestContext& request_context) override; absl::StatusOr RegisterModel( - const RegisterModelRequest& request) override ABSL_LOCKS_EXCLUDED(mu_); + const RegisterModelRequest& request) override; + void ResetModels() override; private: - // The key to `model map` is the `model_path` field in an inference request. - absl::flat_hash_map> - model_map_ ABSL_GUARDED_BY(mu_); - absl::Mutex mu_; + const InferenceSidecarRuntimeConfig runtime_config_; + + // Stores a set of models. It's thread safe. + ModelStore store_; + + // Counts the number of inferences per model. Used for model reset. + absl::flat_hash_map per_model_inference_count_ + ABSL_GUARDED_BY(per_model_inference_count_mu_); + absl::Mutex per_model_inference_count_mu_; + // Notification to trigger model reset. + absl::Notification inference_notification_; + + // The background thread continuously running ResetModels(). + std::thread background_thread_; + // The background thread is shutdown if set to false. + std::atomic background_thread_running_; + // Exclusively used in a single backgrond thread. No need of a mutex. + absl::BitGen bitgen_; }; absl::StatusOr PyTorchModule::Predict( - const PredictRequest& request) { - absl::ReaderMutexLock lock(&mu_); - + const PredictRequest& request, const RequestContext& request_context) { absl::StatusOr> parsed_requests = ParseJsonInferenceRequest(request.input()); if (!parsed_requests.ok()) { @@ -129,13 +194,12 @@ absl::StatusOr PyTorchModule::Predict( std::vector>> tasks; for (const InferenceRequest& inference_request : (*parsed_requests)) { absl::string_view model_key = inference_request.model_path; - auto it = model_map_.find(model_key); - if (it == model_map_.end()) { - return absl::NotFoundError( - absl::StrCat("Model ", model_key, " has not been registered")); - } - tasks.push_back(std::async(std::launch::async, &PredictInternal, - it->second.get(), inference_request)); + INFERENCE_LOG(INFO, request_context) + << "Received inference request to model: " << model_key; + PS_ASSIGN_OR_RETURN(std::shared_ptr model, + store_.GetModel(model_key, request.is_consented())); + tasks.push_back(std::async(std::launch::async, &PredictInternal, model, + inference_request)); } std::vector batch_result_outputs; @@ -156,6 +220,16 @@ absl::StatusOr PyTorchModule::Predict( } PS_ASSIGN_OR_RETURN(std::string output_json, ConvertBatchOutputsToJson(batch_result_outputs)); + + { + absl::MutexLock lock(&per_model_inference_count_mu_); + for (const InferenceRequest& inference_request : *parsed_requests) { + const std::string& model_key = inference_request.model_path; + ++per_model_inference_count_[model_key]; + } + } + inference_notification_.Notify(); + PredictResponse response; response.set_output(output_json); return response; @@ -173,26 +247,44 @@ absl::StatusOr PyTorchModule::RegisterModel( request.model_files().size())); } - absl::WriterMutexLock lock(&mu_); - if (model_map_.find(model_key) != model_map_.end()) { + if (store_.GetModel(model_key).ok()) { return absl::AlreadyExistsError( absl::StrCat("Model ", model_key, " has already been registered")); } + PS_RETURN_IF_ERROR(store_.PutModel(model_key, request)); + return RegisterModelResponse(); +} - // Convert PyTorch exception to absl status. - try { - const std::string& model_payload = request.model_files().begin()->second; - std::istringstream is(model_payload); - model_map_[model_key] = - std::make_unique(torch::jit::load(is)); - // Turn on eval model for layers that behave differently during train and - // eval times, for example, dropout and batch norm layers. - model_map_[model_key]->eval(); - } catch (...) { - return absl::InternalError("Error loading model"); +// TODO(b/346813356): Refactor duplicate logic into a shared library/class. +void PyTorchModule::ResetModels() { + const double reset_probability = runtime_config_.model_reset_probability(); + if (reset_probability == 0.0) { + // Model reset is disabled. + return; } - return RegisterModelResponse(); + std::vector models = store_.ListModels(); + for (const auto& model_key : models) { + int count = 0; + { + absl::MutexLock lock(&per_model_inference_count_mu_); + count = per_model_inference_count_[model_key]; + // Sets the per-model counter to 0. + per_model_inference_count_[model_key] = 0; + } + if (count <= 0) continue; + double random = absl::Uniform(bitgen_, 0.0, 1.0); + // Boosts the chance of reset multiplied by the number of inferences as + // approximation. + if (reset_probability != 1.0 && random >= reset_probability * count) { + continue; + } + + // We should make sure the model reset is successfully done. + // Otherwise, we terminate the program to preserve user privacy. + CHECK(store_.ResetModel(model_key).ok()) + << "Failed to reset model: " << model_key; + } } } // namespace diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch_test.cc b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch_test.cc index 928d861b..8bd4fa59 100644 --- a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch_test.cc +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch_test.cc @@ -23,6 +23,8 @@ #include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/synchronization/blocking_counter.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" #include "gtest/gtest.h" #include "modules/module_interface.h" #include "proto/inference_sidecar.pb.h" @@ -37,6 +39,7 @@ constexpr absl::string_view kTestModelVariedInputs1 = "e2e_model1"; constexpr absl::string_view kTestModelVariedInputs2 = "e2e_model2"; constexpr absl::string_view kTestModelMixedInputsMixedOutputs = "mixed_inputs_mixed_outputs_model"; +constexpr absl::string_view kStatefulModelDir = "stateful_model"; constexpr int kNumThreads = 100; TEST(PyTorchModuleRuntimeConfigTest, @@ -181,6 +184,10 @@ constexpr char kSimpleRequest[] = R"json({ }] })json"; +constexpr absl::string_view kSimpleRequestResponse = + "{\"response\":[{\"model_path\":\"simple_model\",\"tensors\":[{\"tensor_" + "shape\":[1],\"data_type\":\"DOUBLE\",\"tensor_content\":[3.14]}]}]}"; + TEST(PyTorchModulePredictTest, PredictWithoutValidModelReturnsNotFound) { InferenceSidecarRuntimeConfig config; std::unique_ptr torch_module = @@ -213,10 +220,55 @@ TEST(PyTorchModulePredictTest, PredictSimpleSuccess) { const absl::StatusOr result = torch_module->Predict(predict_request); EXPECT_TRUE(result.ok()); - EXPECT_EQ( - result->output(), - "{\"response\":[{\"model_path\":\"simple_model\",\"tensors\":[{\"tensor_" - "shape\":[1],\"data_type\":\"DOUBLE\",\"tensor_content\":[3.14]}]}]}"); + EXPECT_EQ(result->output(), kSimpleRequestResponse); +} + +TEST(PyTorchModulePredictTest, PredictConsentedRequestSuccess) { + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + ASSERT_TRUE( + PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); + ASSERT_TRUE(torch_module->RegisterModel(register_request).ok()); + + PredictRequest predict_request; + predict_request.set_input(kSimpleRequest); + predict_request.set_is_consented(true); + + const absl::StatusOr result = + torch_module->Predict(predict_request); + EXPECT_TRUE(result.ok()); + EXPECT_EQ(result->output(), kSimpleRequestResponse); +} + +TEST(PyTorchModuleResetModelTest, NoModel) { + InferenceSidecarRuntimeConfig config; + config.set_model_reset_probability(1.0); + std::unique_ptr torch_module = + ModuleInterface::Create(config); + torch_module->ResetModels(); +} + +TEST(PyTorchModuleResetModelTest, ResetModelOk) { + InferenceSidecarRuntimeConfig config; + config.set_model_reset_probability(1.0); + std::unique_ptr torch_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + ASSERT_TRUE( + PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); + ASSERT_TRUE(torch_module->RegisterModel(register_request).ok()); + + PredictRequest predict_request; + predict_request.set_input(kSimpleRequest); + + const absl::StatusOr result = + torch_module->Predict(predict_request); + EXPECT_TRUE(result.ok()); + EXPECT_EQ(result->output(), kSimpleRequestResponse); + + torch_module->ResetModels(); } constexpr char kNotRegisteredModelRequest[] = R"json({ @@ -251,7 +303,7 @@ TEST(PyTorchModulePredictTest, PredictReturnsNotFoundError) { EXPECT_FALSE(result.ok()); EXPECT_EQ(result.status().code(), absl::StatusCode::kNotFound); EXPECT_EQ(result.status().message(), - "Model not_registered has not been registered"); + "Requested model 'not_registered' has not been registered"); } constexpr char kMismatchedInput[] = R"json({ @@ -956,6 +1008,76 @@ TEST(PyTorchModulePredictTest, "content\":[0.3857767879962921,0.458008348941803]}]}]}"); } +constexpr char kStatefulModelRequest[] = R"json({ + "request" : [{ + "model_path" : "stateful_model", + "tensors" : [ + { + "data_type": "INT32", + "tensor_shape": [ + 1 + ], + "tensor_content": ["0"] + } + ] +}] + })json"; + +TEST(PyTorchModuleResetModelTest, NoResetWithStatefulModel) { + const int kIterations = 100; + InferenceSidecarRuntimeConfig config; + config.set_model_reset_probability(0.0); + std::unique_ptr torch_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + ASSERT_TRUE( + PopulateRegisterModelRequest(kStatefulModelDir, register_request).ok()); + ASSERT_TRUE(torch_module->RegisterModel(register_request).ok()); + + for (int count = 1; count < kIterations; count++) { + PredictRequest predict_request; + predict_request.set_input(kStatefulModelRequest); + absl::StatusOr predict_status = torch_module->Predict(predict_request); + ASSERT_TRUE(predict_status.ok()); + PredictResponse response = predict_status.value(); + ASSERT_FALSE(response.output().empty()); + + EXPECT_TRUE(absl::StrContains( + response.output(), + absl::StrCat( + "{\"tensor_shape\":[],\"data_type\":\"INT32\",\"tensor_content\":[", + count, "]}"))) + << response.output(); + } +} + +TEST(PyTorchModuleResetModelTest, ResetSuccessWithStatefulModel) { + const int kIterations = 10; + InferenceSidecarRuntimeConfig config; + config.set_model_reset_probability(1.0); + std::unique_ptr torch_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + ASSERT_TRUE( + PopulateRegisterModelRequest(kStatefulModelDir, register_request).ok()); + ASSERT_TRUE(torch_module->RegisterModel(register_request).ok()); + + for (int i = 0; i < kIterations; i++) { + PredictRequest predict_request; + predict_request.set_input(kStatefulModelRequest); + absl::StatusOr predict_status = torch_module->Predict(predict_request); + ASSERT_TRUE(predict_status.ok()); + PredictResponse response = predict_status.value(); + ASSERT_FALSE(response.output().empty()); + + EXPECT_TRUE(absl::StrContains( + response.output(), + "{\"tensor_shape\":[],\"data_type\":\"INT32\",\"tensor_content\":[1]}")) + << response.output(); + absl::SleepFor(absl::Seconds(1)); + } +} + TEST(PyTorchModuleConcurrencyTest, RegisterSameModelWithMultipleThreads) { InferenceSidecarRuntimeConfig config; std::unique_ptr torch_module = @@ -981,9 +1103,11 @@ TEST(PyTorchModuleConcurrencyTest, RegisterSameModelWithMultipleThreads) { num_returns_already_exists.DecrementCount(); } })); + if (i == 0) { + // The first call returns ok. + num_returns_ok.Wait(); + } } - - num_returns_ok.Wait(); num_returns_already_exists.Wait(); for (auto& thread : threads) { diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/test_constants.h b/services/inference_sidecar/modules/pytorch_v2_1_1/test_constants.h index 5a98cd58..fe77b50a 100644 --- a/services/inference_sidecar/modules/pytorch_v2_1_1/test_constants.h +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/test_constants.h @@ -21,8 +21,12 @@ namespace privacy_sandbox::bidding_auction_servers::inference { -constexpr absl::string_view kRuntimeConfig = - R"json({"num_interop_threads": 4, "num_intraop_threads": 5, "module_name": "pytorch_v2_1_1"})json"; +constexpr absl::string_view kRuntimeConfig = R"json({ + "num_interop_threads": 4, + "num_intraop_threads": 5, + "module_name": "pytorch_v2_1_1", + "cpuset": [0, 1, 2, 3] +})json"; } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/BUILD b/services/inference_sidecar/modules/tensorflow_v2_14_0/BUILD index 32e9d49c..29a30f77 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/BUILD +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/BUILD @@ -136,11 +136,14 @@ cc_library( ":tensorflow_parser", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/random", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + "@inference_common//model:model_store", "@inference_common//modules:module_interface", "@inference_common//proto:inference_sidecar_cc_proto", "@inference_common//utils:request_parser", @@ -169,12 +172,14 @@ cc_test( "//benchmark_models/embedding:embedding_model", "//benchmark_models/pctr:pctr_model", "//benchmark_models/pcvr:pcvr_model", + "//benchmark_models/stateful:stateful_model", ], deps = [ ":tensorflow", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@com_google_absl//absl/time", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", "@inference_common//proto:inference_sidecar_cc_proto", @@ -234,7 +239,7 @@ genrule( outs = ["inference_sidecar_binary"], cmd_bash = """cat << EOF > '$@' mkdir -p artifacts -cp $(execpath :inference_sidecar_bin) artifacts/inference_sidecar +cp $(execpath :inference_sidecar_bin) artifacts/inference_sidecar_tensorflow_v2_14_0 EOF""", executable = True, local = True, @@ -244,7 +249,7 @@ EOF""", # Exports TensorFlow inference sidecar binary to be packaged in the B&A workspace. exports_files( [ - "artifacts/inference_sidecar", + "artifacts/inference_sidecar_tensorflow_v2_14_0", ], ) diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark_models/stateful/BUILD b/services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark_models/stateful/BUILD new file mode 100644 index 00000000..fb16d57d --- /dev/null +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark_models/stateful/BUILD @@ -0,0 +1,23 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filegroup( + name = "stateful_model", + srcs = [ + "saved_model.pb", + "variables/variables.data-00000-of-00001", + "variables/variables.index", + ], + visibility = ["//visibility:public"], +) diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark_models/stateful/saved_model.pb b/services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark_models/stateful/saved_model.pb new file mode 100644 index 0000000000000000000000000000000000000000..43eb009d4fc5d53bd7fbf38d3369c1023b81ddcd GIT binary patch literal 18376 zcmeHP>u(%a72nzM?v9V&*G{~-&U2FHv9z(hN#$e(EnYWmlBP+pok)Q2FxeeD6R&5N znO!F-2#E^us8khNgir-vEQG3nDkvbt7Z4Q))K4J(0KUMdBKQCZ@d?ho^PZitcT*=# zOIjs6k309Cd+vGs&bjAC63AOG3$W2o{?!L(V06~9^c7>aTD_{8x_YgqU22eikW9_0 zx6F!`A0va1SyyW&46ANNh=W5g@|13|Y0cFew=Ws< znpzD{kM(~Xq;plxXzI<|1R$duYy@GE_Ibkzi*hdri}sY^O9GJRG;>9}TAaJCRaP5y z-Dp~597ZeNt+wX%v!mE}uiA@v2}Vt4}mAY3FO zQhHRF5(3kahoDEjrCaH2a74l+t+tJ_>`3 z+esopw-&29^v)yO>L&6T*ClIn6!|#}{q}MzhPuW%JO;y@1VuHfidEAq>?s|AoZZeU zX1VNIavq#30yW)g9%UkYj4_1CK@GAC>iS)dM!FsCNgNGVztSWF&|9e+Rh>yk9!M4h zCVuQ!+e{44!qCEX)vRjOr%^N*E0}0sQ!RdteCp*3V^;GbN(LCMHKtr<)yj($8!~xf zALVoKD*?tAny7utt=eVPL`6VKwCWrRAn8NtFjuUixTqVrXpRCFdlpd>%fCZGVyfhSo0GxUBzYWqEv=!NDw5{*#HIt^6rc}<|13%*GpdZ^zENeL z;bfbtZfK#;~=?)WJZJkIgP+jX@K-e|8@Tpf)f-9d3S=*X7IgqvPvcC z8ciZIFyR4P z3ih(BnVL~S*N#X-ZMIBJxoN761`_u8Y3R3o7_mLsFS3rWC$`I%2PSP{F5yt6sl*us z#4cC(PsW7>f6tWvnvrrORsKr`-WFg$5=ig5gLEtym(pakX`&V~h@ki{c1a@HVQMR_ z8n%7jcH6bOWoZ_6(L;JyHB+?|wV^}5lp&I$R8aL41qxD@9Ol=|5t$lJ)==0h&%UcQ z#p!4Q=^FP0(n81MGYI-5iR3EvmVp)u2BlsC9`me1T0+|b8Ehg<@tbrZ1?8tk5&B7PO+$lMv0$GxKqmMLBsy-^Bhnxlup7AFYy&UD%-fV9 z-Ej6Wu7Vt0h4vNTtlN!@7$Q0377FMT-VD1p)5uWvcEo+a-V*nQy=5$aiaY^>S7+zX z&6Y2mQLfHjxN^ofY*coZaPs)<;^O?d@|DFi3zOm2oC<`F(kK~lI0#4 zOLmA^vVHS=U?OPAn08gr(Q>UlH3Q$-gh|D_U=nhJ11oOka}hKD;4tS1V~#w(ANIyE z2mJ^G@5<@ZIF#T80_P2scOe?6^!BP27lt3>ik%#2W@qV z#RP0I){D&4T1`ZI2zhlUs#8wna|nGlgqW~xL)gh7M_Zh@A&l}D>@OdS(3=T3GZuZ) z;Do}ytdQu-cKFZ~LDG_SqBJ>qyg)y(nP*JyE#w(@x{h(6+{fKw<~U;4CRHIf$JmA% z9p`sD?0v%Ros@@p@1QibwZKP}-4h4cOE#)3P0>M4l7Bz!kmx;$GBn3R z^z>9NiL-P$gzy7sKeSGC$d1c48|NZP=hKPfU?ky3TMkE)8V3QTSRgJ7d&@Z{*Cjzf zs&_1eGR)@`rPmyh8Hi9Z2B2U z&`;8y+M!#Si|C+ZZHOOXGGj!X>bHdF9Cybs(_0<7KY@C`#i+Za z%C!G5lpQ5P}#yL2HkAatk*DrX=n!e*@}({hxAdQ5UW##VK4EV5Ad)k$hn)`V-D9vD~!J_yi+|?v023b^;)l z&O|#($7}-|hG_8;C?5#oLE@8q!XvqG6n5C~(QIJ?1%qfNNgEHDqu-hHwMUyJOW4z zSbupp2|IUm>W&-=+#cNg+yV>SF3w+o^PO`%kkLZ5uy-rGY?g9jiY~G_G15fK{ z*ia;+lMjasMSdxv;{^5JTW+JvZ zIaRuV%61ky$yCFEgpPWNE4J{XKGeFcg7>zt{2=U7`L?keh1VNi?$lGgVEA2h!tb_i zV)}-_Y@oBr_kKWzO`tx#_YEArE(G^N{cX{3+`7-FEQqmvO*`l}h?4F#ih_dOZBQIQ zVmrL-PYkU4WULiH59KU!$=QUVnKHv;Slrna1lv#OIwES?A}>S>saNjKi5 zM?T!%)ntsV+z<}LGz_P*annB^(}?Sslc?>;eZ{)+pWp^eOy@DA#+r57Ez4-!e7iRn zZwe6CQJ8UUd$nB?dt|9I-gLx&x)KW?j>QZSo=4Y+q8>sNolb(vakCAR5^u`yH}1IH zvCMWT+r_6l5vQ#s?mfdWKg+_})^)aFd-v|PN?!SY+A2BBwn`4n@7uzkJdEk@c})NK zz_n(Rxg^~EnmnxfRs-Y?_9v`J@-Nj#TT)ho@?JA2_zACMAzJfAb9G?x{- z&Yy2SlO9RHvkpg`x_fyq>|{m2tNi2uJ7?n_YH%9v--Z@%Kdk3xM=m<7=a<>e>57IY zs`QFo+fcuDgPpIlSMCq#8NdyRpW(pr2gu&&SyQc5Jd~Eh>>xXj$IkPhPsgLV=ITv;GY*n- zy2&Da8=?g11|CtsRwDqbH`&P^{+;-mm?BuCWk2%1U=0cMsfJN!r|7U4n22}8Q~{qq zA3Py}H~t9;__COMK%)5j^mgV;x3RC8_wh>jr%`Yv{L?7#JE6RF%21 z@ui3?I;TAeml8yYuC$>5a_JdyB@X3u^(KEb7?8W|5H}bu4YJbbqIb#<9*sR7BkzC# z?^G0)B{tp*6Wz6$XRb2hD*i2$|0#57PO$hXu$u2v%K~>Zb2OwMfSJFNF2B63_VFQ8 z+H2BgJ_tOE|@DdvLuWn}CBT(W#7<)>_izq&1O8aA)W+rLrlSAcfa_$tIQ6bTp$coZ>)9XOzNdggM@l!(_#vV;nj zavmmSc$5`tf4(}GPvpPMyFkP*nkGrv{0K+F0xNy%IDgg*`2e$ E1D?CjWdHyG literal 0 HcmV?d00001 diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark_models/stateful/variables/variables.index b/services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark_models/stateful/variables/variables.index new file mode 100644 index 0000000000000000000000000000000000000000..b855c8a9a70a6b377bc03700ccb2db1050f4e8af GIT binary patch literal 195 zcmZQzVB=tvV&Y(Akl~AW_HcFf4)FK%3vqPvagFzP@^WN$4h) literal 0 HcmV?d00001 diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc index f1ef5a49..cd4704c6 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc @@ -14,21 +14,22 @@ * limitations under the License. */ -#include +#include #include -#include -#include -#include -#include +#include #include #include "absl/base/thread_annotations.h" #include "absl/container/flat_hash_map.h" +#include "absl/log/absl_log.h" +#include "absl/random/random.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" #include "absl/synchronization/mutex.h" +#include "absl/synchronization/notification.h" +#include "model/model_store.h" #include "modules/module_interface.h" #include "proto/inference_sidecar.pb.h" #include "src/util/status_macro/status_macros.h" @@ -40,6 +41,7 @@ #include "tensorflow/core/public/session.h" #include "tensorflow/tsl/platform/env.h" #include "tensorflow/tsl/platform/file_system.h" +#include "utils/log.h" #include "utils/request_parser.h" #include "tensorflow_parser.h" @@ -62,7 +64,7 @@ absl::Status SaveToRamFileSystem(const RegisterModelRequest& request) { } absl::StatusOr>> -PredictPerModel(const tensorflow::SavedModelBundle* model, +PredictPerModel(std::shared_ptr model, const InferenceRequest& inference_request) { std::vector> inputs; for (const auto& tensor : inference_request.inputs) { @@ -111,30 +113,91 @@ PredictPerModel(const tensorflow::SavedModelBundle* model, return std::make_pair(std::string(model_key), zipped_vector); } +absl::StatusOr> +TensorFlowModelConstructor(const InferenceSidecarRuntimeConfig& config, + const RegisterModelRequest& request) { + tensorflow::SessionOptions session_options; + // TODO(b/332599154): Support runtime configuration on a per-session basis. + if (config.num_intraop_threads() != 0) { + session_options.config.set_intra_op_parallelism_threads( + config.num_intraop_threads()); + } + if (config.num_interop_threads() != 0) { + session_options.config.set_inter_op_parallelism_threads( + config.num_interop_threads()); + } + const std::unordered_set tags = {"serve"}; + const auto& model_path = request.model_spec().model_path(); + auto model_bundle = std::make_unique(); + if (auto status = tensorflow::LoadSavedModel( + session_options, {}, absl::StrCat(kRamFileSystemScheme, model_path), + tags, model_bundle.get()); + !status.ok()) { + return absl::InternalError( + absl::StrCat("Error loading model: ", model_path)); + } + return model_bundle; +} + class TensorflowModule final : public ModuleInterface { public: explicit TensorflowModule(const InferenceSidecarRuntimeConfig& config) - : runtime_config_(config) {} + : runtime_config_(config), + store_(config, TensorFlowModelConstructor), + background_thread_running_(true) { + // TODO(b/332328206): Move reset model logic into the ModelStore. + background_thread_ = std::thread([this]() { + while (background_thread_running_) { + // ResetModels() is continually called in the background thread. + ResetModels(); + // Waits for the completed inference execution. + // + // Moves on after 1 second and check if the background thread should + // terminate. The destructor terminates this thread by setting + // |background_thread_running_|. + inference_notification_.WaitForNotificationWithTimeout( + absl::Seconds(1)); + } + }); + } + + ~TensorflowModule() override { + background_thread_running_ = false; + background_thread_.join(); + } + absl::StatusOr Predict( - const PredictRequest& request) override ABSL_LOCKS_EXCLUDED(mu_); + const PredictRequest& request, + const RequestContext& request_context) override; absl::StatusOr RegisterModel( - const RegisterModelRequest& request) override ABSL_LOCKS_EXCLUDED(mu_); + const RegisterModelRequest& request) override; + void ResetModels() override; private: - // Maps each `model_path` from an inference request to its corresponding - // tensorflow::SavedModelBundle instance. - absl::flat_hash_map> - model_map_ ABSL_GUARDED_BY(mu_); - // TODO(b/327907675) : Add a test for concurrency - absl::Mutex mu_; const InferenceSidecarRuntimeConfig runtime_config_; + + // Stores a set of models. It's thread safe. + // TODO(b/327907675) : Add a test for concurrency + ModelStore store_; + + // Counts the number of inferences per model. Used for model reset. + absl::flat_hash_map per_model_inference_count_ + ABSL_GUARDED_BY(per_model_inference_count_mu_); + absl::Mutex per_model_inference_count_mu_; + + // Notification to trigger model reset. + absl::Notification inference_notification_; + + // The background thread continuously running ResetModels(). + std::thread background_thread_; + // The background thread is shutdown if set to false. + std::atomic background_thread_running_; + // Exclusively used in a single backgrond thread. No need of a mutex. + absl::BitGen bitgen_; }; absl::StatusOr TensorflowModule::Predict( - const PredictRequest& request) { - absl::ReaderMutexLock lock(&mu_); - + const PredictRequest& request, const RequestContext& request_context) { absl::StatusOr> parsed_requests = ParseJsonInferenceRequest(request.input()); if (!parsed_requests.ok()) { @@ -147,13 +210,12 @@ absl::StatusOr TensorflowModule::Predict( for (const InferenceRequest& inference_request : *parsed_requests) { absl::string_view model_key = inference_request.model_path; - auto it = model_map_.find(model_key); - if (it == model_map_.end()) { - return absl::NotFoundError( - absl::StrCat("Requested model '", model_key, "' is not registered")); - } - tasks.push_back(std::async(std::launch::async, &PredictPerModel, - it->second.get(), inference_request)); + INFERENCE_LOG(INFO, request_context) + << "Received inference request to model: " << model_key; + PS_ASSIGN_OR_RETURN(std::shared_ptr model, + store_.GetModel(model_key, request.is_consented())); + tasks.push_back(std::async(std::launch::async, &PredictPerModel, model, + inference_request)); } std::vector>> @@ -176,6 +238,15 @@ absl::StatusOr TensorflowModule::Predict( return absl::InternalError("Error during output parsing to json"); } + { + absl::MutexLock lock(&per_model_inference_count_mu_); + for (const InferenceRequest& inference_request : *parsed_requests) { + const std::string& model_key = inference_request.model_path; + ++per_model_inference_count_[model_key]; + } + } + inference_notification_.Notify(); + PredictResponse predict_response; predict_response.set_output(output_json.value()); return predict_response; @@ -183,44 +254,53 @@ absl::StatusOr TensorflowModule::Predict( absl::StatusOr TensorflowModule::RegisterModel( const RegisterModelRequest& request) { - tensorflow::SessionOptions session_options; - // TODO(b/332599154): Support runtime configuration on a per-session basis. - if (runtime_config_.num_intraop_threads() != 0) { - session_options.config.set_intra_op_parallelism_threads( - runtime_config_.num_intraop_threads()); - } - if (runtime_config_.num_interop_threads() != 0) { - session_options.config.set_inter_op_parallelism_threads( - runtime_config_.num_interop_threads()); - } - - const std::unordered_set tags = {"serve"}; const auto& model_path = request.model_spec().model_path(); - if (model_path.empty()) { return absl::InvalidArgumentError("Model path is empty"); } - absl::WriterMutexLock lock(&mu_); - auto it = model_map_.find(model_path); - if (it != model_map_.end()) { + if (store_.GetModel(model_path).ok()) { return absl::AlreadyExistsError( - absl::StrCat("Model '", model_path, "' already registered")); + absl::StrCat("Model ", model_path, " has already been registered")); } - PS_RETURN_IF_ERROR(SaveToRamFileSystem(request)); - auto model_bundle = std::make_unique(); - auto status = tensorflow::LoadSavedModel( - session_options, {}, absl::StrCat(kRamFileSystemScheme, model_path), tags, - model_bundle.get()); + RegisterModelRequest model_request; + *model_request.mutable_model_spec() = request.model_spec(); + PS_RETURN_IF_ERROR(store_.PutModel(model_path, model_request)); + return RegisterModelResponse(); +} - if (!status.ok()) { - return absl::InternalError( - absl::StrCat("Error loading model: ", model_path)); +// TODO(b/346813356): Refactor duplicate logic into a shared library/class. +void TensorflowModule::ResetModels() { + const double reset_probability = runtime_config_.model_reset_probability(); + if (reset_probability == 0.0) { + // Model reset is disabled. + return; + } + + std::vector models = store_.ListModels(); + for (const auto& model_key : models) { + int count = 0; + { + absl::MutexLock lock(&per_model_inference_count_mu_); + count = per_model_inference_count_[model_key]; + // Sets the per-model counter to 0. + per_model_inference_count_[model_key] = 0; + } + if (count <= 0) continue; + double random = absl::Uniform(bitgen_, 0.0, 1.0); + // Boosts the chance of reset multiplied by the number of inferences as + // approximation. + if (reset_probability != 1.0 && random >= reset_probability * count) { + continue; + } + + // We should make sure the model reset is successfully done. + // Otherwise, we terminate the program to preserve user privacy. + CHECK(store_.ResetModel(model_key).ok()) + << "Failed to reset model: " << model_key; } - model_map_[model_path] = std::move(model_bundle); - return RegisterModelResponse(); } } // namespace diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_test.cc b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_test.cc index a749a989..9a75d2b7 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_test.cc +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_test.cc @@ -18,6 +18,9 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" #include "gtest/gtest.h" #include "modules/module_interface.h" #include "proto/inference_sidecar.pb.h" @@ -30,6 +33,7 @@ using ::testing::HasSubstr; constexpr absl::string_view kModel1Dir = "./benchmark_models/pcvr"; constexpr absl::string_view kModel2Dir = "./benchmark_models/pctr"; constexpr absl::string_view kEmbeddingModelDir = "./benchmark_models/embedding"; +constexpr absl::string_view kStatefulModelDir = "./benchmark_models/stateful"; TEST(TensorflowModuleTest, Failure_RegisterModelWithEmptyPath) { InferenceSidecarRuntimeConfig config; @@ -68,8 +72,8 @@ TEST(TensorflowModuleTest, Failure_RegisterModelAlreadyRegistered) { ASSERT_FALSE(response_2.ok()); EXPECT_EQ(response_2.status().code(), absl::StatusCode::kAlreadyExists); - EXPECT_EQ(response_2.status().message(), - "Model '" + std::string(kModel1Dir) + "' already registered"); + EXPECT_EQ(response_2.status().message(), "Model " + std::string(kModel1Dir) + + " has already been registered"); } TEST(TensorflowModuleTest, Failure_RegisterModelLoadError) { @@ -117,7 +121,7 @@ TEST(TensorflowModuleTest, Failure_PredictInvalidJson) { absl::StatusCode::kInvalidArgument); } -constexpr char kJsonString[] = R"json({ +constexpr char kPcvrJsonRequest[] = R"json({ "request" : [{ "model_path" : "./benchmark_models/pcvr", "tensors" : [ @@ -181,13 +185,19 @@ constexpr char kJsonString[] = R"json({ }] })json"; +constexpr absl::string_view kPcvrResponse = + "{\"response\":[{\"model_path\":\"./benchmark_models/" + "pcvr\",\"tensors\":[{\"tensor_name\":\"StatefulPartitionedCall:" + "0\",\"tensor_shape\":[1,1],\"data_type\":\"FLOAT\",\"tensor_" + "content\":[0.019116628915071489]}]}]}"; + TEST(TensorflowModuleTest, Failure_PredictModelNotRegistered) { InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = ModuleInterface::Create(config); PredictRequest predict_request; - predict_request.set_input(kJsonString); + predict_request.set_input(kPcvrJsonRequest); absl::StatusOr predict_response = tensorflow_module->Predict(predict_request); @@ -205,19 +215,61 @@ TEST(TensorflowModuleTest, Success_Predict) { ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); PredictRequest predict_request; - predict_request.set_input(kJsonString); + predict_request.set_input(kPcvrJsonRequest); absl::StatusOr predict_status = tensorflow_module->Predict(predict_request); ASSERT_TRUE(predict_status.ok()); PredictResponse response = predict_status.value(); ASSERT_FALSE(response.output().empty()); - ASSERT_EQ(response.output(), - "{\"response\":[{\"model_path\":\"./benchmark_models/" - "pcvr\",\"tensors\":[{\"tensor_name\":\"StatefulPartitionedCall:" - "0\",\"tensor_shape\":[1,1],\"data_type\":\"FLOAT\",\"tensor_" - "content\":[0.019116628915071489]}]}]}"); + ASSERT_EQ(response.output(), kPcvrResponse); +} + +TEST(TensorflowModuleTest, Success_PredictWithConsentedRequest) { + InferenceSidecarRuntimeConfig config; + std::unique_ptr tensorflow_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + ASSERT_TRUE(PopulateRegisterModelRequest(kModel1Dir, register_request).ok()); + ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); + + PredictRequest predict_request; + predict_request.set_input(kPcvrJsonRequest); + predict_request.set_is_consented(true); + absl::StatusOr predict_status = tensorflow_module->Predict(predict_request); + ASSERT_TRUE(predict_status.ok()); + PredictResponse response = predict_status.value(); + ASSERT_FALSE(response.output().empty()); + ASSERT_EQ(response.output(), kPcvrResponse); +} + +TEST(TensorflowModuleTest, Success_ResetModels_NoModel) { + InferenceSidecarRuntimeConfig config; + config.set_model_reset_probability(1.0); + std::unique_ptr tensorflow_module = + ModuleInterface::Create(config); + tensorflow_module->ResetModels(); +} + +TEST(TensorflowModuleTest, Success_ResetModels) { + InferenceSidecarRuntimeConfig config; + config.set_model_reset_probability(1.0); + std::unique_ptr tensorflow_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + ASSERT_TRUE(PopulateRegisterModelRequest(kModel1Dir, register_request).ok()); + ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); + + PredictRequest predict_request; + predict_request.set_input(kPcvrJsonRequest); + absl::StatusOr predict_status = tensorflow_module->Predict(predict_request); + ASSERT_TRUE(predict_status.ok()); + PredictResponse response = predict_status.value(); + ASSERT_FALSE(response.output().empty()); + ASSERT_EQ(response.output(), kPcvrResponse); + + tensorflow_module->ResetModels(); } -constexpr char kJsonStringBatchSize2[] = R"json({ +constexpr char kPcvrJsonRequestBatchSize2[] = R"json({ "request" : [{ "model_path" : "./benchmark_models/pcvr", "tensors" : [ @@ -290,7 +342,7 @@ TEST(TensorflowModuleTest, Success_PredictBatchSize2) { ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); PredictRequest predict_request; - predict_request.set_input(kJsonStringBatchSize2); + predict_request.set_input(kPcvrJsonRequestBatchSize2); absl::StatusOr predict_status = tensorflow_module->Predict(predict_request); ASSERT_TRUE(predict_status.ok()); PredictResponse response = predict_status.value(); @@ -302,7 +354,7 @@ TEST(TensorflowModuleTest, Success_PredictBatchSize2) { "content\":[0.019116630777716638,0.1847093403339386]}]}]}"); } -constexpr char kJsonStringMissingTensorName[] = R"json({ +constexpr char kPcvrJsonRequestMissingTensorName[] = R"json({ "request" : [{ "model_path" : "./benchmark_models/pcvr", "tensors" : [ @@ -326,7 +378,7 @@ TEST(TensorflowModuleTest, Failure_PredictMissingTensorName) { ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); PredictRequest predict_request; - predict_request.set_input(kJsonStringMissingTensorName); + predict_request.set_input(kPcvrJsonRequestMissingTensorName); absl::StatusOr predict_status = tensorflow_module->Predict(predict_request); ASSERT_FALSE(predict_status.ok()); @@ -335,7 +387,7 @@ TEST(TensorflowModuleTest, Failure_PredictMissingTensorName) { HasSubstr("Name is required for each TensorFlow tensor input")); } -constexpr char kJsonStringWith2Model[] = R"json({ +constexpr char kPcvrJsonRequestWith2Model[] = R"json({ "request" : [{ "model_path" : "./benchmark_models/pcvr", "tensors" : [ @@ -475,7 +527,7 @@ TEST(TensorflowModuleTest, Success_PredictWith2Models) { ASSERT_TRUE(tensorflow_module->RegisterModel(register_request_2).ok()); PredictRequest predict_request; - predict_request.set_input(kJsonStringWith2Model); + predict_request.set_input(kPcvrJsonRequestWith2Model); absl::StatusOr predict_status = tensorflow_module->Predict(predict_request); ASSERT_TRUE(predict_status.ok()); PredictResponse response = predict_status.value(); @@ -491,7 +543,7 @@ TEST(TensorflowModuleTest, Success_PredictWith2Models) { "content\":[0.14649735391139985,0.2522672712802887]}]}]}"); } -constexpr char kJsonStringWith1ModelVariedSize[] = R"json({ +constexpr char kPcvrJsonRequestWith1ModelVariedSize[] = R"json({ "request" : [{ "model_path" : "./benchmark_models/pcvr", "tensors" : [ @@ -628,7 +680,7 @@ TEST(TensorflowModuleTest, ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); PredictRequest predict_request; - predict_request.set_input(kJsonStringWith1ModelVariedSize); + predict_request.set_input(kPcvrJsonRequestWith1ModelVariedSize); absl::StatusOr predict_status = tensorflow_module->Predict(predict_request); ASSERT_TRUE(predict_status.ok()); PredictResponse response = predict_status.value(); @@ -644,7 +696,7 @@ TEST(TensorflowModuleTest, "content\":[0.010360434651374817]}]}]}"); } -constexpr char kJsonStringEmbeddingModel[] = R"json({ +constexpr char kPcvrJsonRequestEmbeddingModel[] = R"json({ "request" : [{ "model_path" : "./benchmark_models/embedding", "tensors" : [ @@ -718,7 +770,7 @@ TEST(TensorflowModuleTest, Success_PredictEmbed) { ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); PredictRequest predict_request; - predict_request.set_input(kJsonStringEmbeddingModel); + predict_request.set_input(kPcvrJsonRequestEmbeddingModel); absl::StatusOr predict_status = tensorflow_module->Predict(predict_request); ASSERT_TRUE(predict_status.ok()); PredictResponse response = predict_status.value(); @@ -737,7 +789,7 @@ TEST(TensorflowModuleTest, Success_PredictEmbed) { "\"data_type\":\"INT32\",\"tensor_content\":[0]}")); } -constexpr char kJsonStringWithMultipleInvalidInputs[] = R"json({ +constexpr char kPcvrJsonRequestWithMultipleInvalidInputs[] = R"json({ "request" : [{ "model_path" : "./benchmark_models/pcvr", "tensors" : [ @@ -782,7 +834,7 @@ TEST(TensorflowModuleTest, PredictMultipleInvalidInputsReturnsFirstError) { ASSERT_TRUE(tensorflow_module->RegisterModel(register_request_2).ok()); PredictRequest predict_request; - predict_request.set_input(kJsonStringWithMultipleInvalidInputs); + predict_request.set_input(kPcvrJsonRequestWithMultipleInvalidInputs); absl::StatusOr predict_status = tensorflow_module->Predict(predict_request); ASSERT_FALSE(predict_status.ok()); EXPECT_EQ(predict_status.status().code(), absl::StatusCode::kInternal); @@ -791,5 +843,80 @@ TEST(TensorflowModuleTest, PredictMultipleInvalidInputsReturnsFirstError) { HasSubstr("Error during inference for model './benchmark_models/pcvr'")); } +constexpr char kJsonRequestStatefulModel[] = R"json({ + "request" : [{ + "model_path" : "./benchmark_models/stateful", + "tensors" : [ + { + "tensor_name": "serving_default_input_1:0", + "data_type": "FLOAT", + "tensor_shape": [ + 1, 1 + ], + "tensor_content": ["0.0"] + } + ] +}] + })json"; + +TEST(TensorflowModuleTest, Success_NoReset_StatefulModel) { + const int kIterations = 100; + InferenceSidecarRuntimeConfig config; + config.set_model_reset_probability(0.0); + + std::unique_ptr tensorflow_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + ASSERT_TRUE( + PopulateRegisterModelRequest(kStatefulModelDir, register_request).ok()); + ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); + + for (int count = 2; count < kIterations; count++) { + PredictRequest predict_request; + predict_request.set_input(kJsonRequestStatefulModel); + absl::StatusOr predict_status = tensorflow_module->Predict(predict_request); + ASSERT_TRUE(predict_status.ok()); + PredictResponse response = predict_status.value(); + ASSERT_FALSE(response.output().empty()); + + EXPECT_TRUE(absl::StrContains( + response.output(), + absl::StrCat("{\"tensor_name\":\"StatefulPartitionedCall:0\",\"tensor_" + "shape\":[]," + "\"data_type\":\"INT32\",\"tensor_content\":[", + count, "]}"))) + << response.output(); + } +} + +TEST(TensorflowModuleTest, Success_Reset_StatefulModel) { + const int kIterations = 10; + InferenceSidecarRuntimeConfig config; + config.set_model_reset_probability(1.0); + + std::unique_ptr tensorflow_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + ASSERT_TRUE( + PopulateRegisterModelRequest(kStatefulModelDir, register_request).ok()); + ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); + + for (int i = 0; i < kIterations; i++) { + PredictRequest predict_request; + predict_request.set_input(kJsonRequestStatefulModel); + absl::StatusOr predict_status = tensorflow_module->Predict(predict_request); + ASSERT_TRUE(predict_status.ok()); + PredictResponse response = predict_status.value(); + ASSERT_FALSE(response.output().empty()); + + EXPECT_TRUE(absl::StrContains( + response.output(), + "{\"tensor_name\":\"StatefulPartitionedCall:0\",\"tensor_shape\":[]," + "\"data_type\":\"INT32\",\"tensor_content\":[2]}")) + << response.output(); + absl::SleepFor(absl::Seconds(1)); + } +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/test_constants.h b/services/inference_sidecar/modules/tensorflow_v2_14_0/test_constants.h index e50168c2..c830b815 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/test_constants.h +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/test_constants.h @@ -21,8 +21,13 @@ namespace privacy_sandbox::bidding_auction_servers::inference { -constexpr absl::string_view kRuntimeConfig = - R"json({"num_interop_threads": 4, "num_intraop_threads": 5, "module_name": "tensorflow_v2_14_0"})json"; +constexpr absl::string_view kRuntimeConfig = R"json( + { + "num_interop_threads": 4, + "num_intraop_threads": 5, + "module_name": "tensorflow_v2_14_0", + "cpuset": [0, 1, 2, 3] + })json"; } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/seller_frontend_service/BUILD b/services/seller_frontend_service/BUILD index 9aac67d5..45edb5c0 100644 --- a/services/seller_frontend_service/BUILD +++ b/services/seller_frontend_service/BUILD @@ -33,7 +33,7 @@ cc_library( srcs = [ "select_ad_reactor.cc", "select_ad_reactor_app.cc", - "select_ad_reactor_invalid_client.cc", + "select_ad_reactor_invalid.cc", "select_ad_reactor_web.cc", "select_auction_result_reactor.cc", "seller_frontend_service.cc", @@ -41,7 +41,7 @@ cc_library( hdrs = [ "select_ad_reactor.h", "select_ad_reactor_app.h", - "select_ad_reactor_invalid_client.h", + "select_ad_reactor_invalid.h", "select_ad_reactor_web.h", "select_auction_result_reactor.h", "seller_frontend_service.h", @@ -51,6 +51,7 @@ cc_library( ":runtime_flags", "//api:bidding_auction_servers_cc_grpc_proto", "//api:bidding_auction_servers_cc_proto", + "//services/common:feature_flags", "//services/common/clients/auction_server:async_client", "//services/common/clients/buyer_frontend_server:buyer_frontend_async_client", "//services/common/clients/buyer_frontend_server:buyer_frontend_async_client_factory", @@ -137,6 +138,7 @@ cc_test( ":seller_frontend_service", "//api:bidding_auction_servers_cc_grpc_proto", "//api:bidding_auction_servers_cc_proto", + "//services/common:feature_flags", "//services/common/test:mocks", "//services/common/test:random", "//services/seller_frontend_service/util:select_ad_reactor_test_utils", @@ -154,6 +156,7 @@ cc_test( ], deps = [ ":seller_frontend_service", + "//services/common:feature_flags", "//services/common/test:mocks", "//services/common/test:random", "//services/common/test/utils:cbor_test_utils", @@ -195,6 +198,7 @@ cc_test( ], deps = [ ":seller_frontend_service", + "//services/common:feature_flags", "//services/common/test:mocks", "//services/common/test:random", "//services/common/test/utils:cbor_test_utils", @@ -217,6 +221,7 @@ cc_test( ], deps = [ ":seller_frontend_service", + "//services/common:feature_flags", "//services/common/compression:gzip", "//services/common/test:mocks", "//services/common/test:random", diff --git a/services/seller_frontend_service/benchmarking/select_ad_reactor_benchmarks.cc b/services/seller_frontend_service/benchmarking/select_ad_reactor_benchmarks.cc index 0d6dbe2c..e44c5854 100644 --- a/services/seller_frontend_service/benchmarking/select_ad_reactor_benchmarks.cc +++ b/services/seller_frontend_service/benchmarking/select_ad_reactor_benchmarks.cc @@ -73,9 +73,10 @@ class BuyerFrontEndAsyncClientStub : public BuyerFrontEndAsyncClient { std::unique_ptr request, const RequestMetadata& metadata, absl::AnyInvocable>) &&> + GetBidsResponse::GetBidsRawResponse>>, + ResponseMetadata) &&> on_done, - absl::Duration timeout) const override; + absl::Duration timeout, RequestConfig request_config) const override; private: const std::string ad_render_url_; @@ -99,9 +100,10 @@ absl::Status BuyerFrontEndAsyncClientStub::ExecuteInternal( std::unique_ptr request, const RequestMetadata& metadata, absl::AnyInvocable>) &&> + GetBidsResponse::GetBidsRawResponse>>, + ResponseMetadata) &&> on_done, - absl::Duration timeout) const { + absl::Duration timeout, RequestConfig request_config) const { auto response = std::make_unique(); switch (buyer_mock_type_) { case BuyerMockType::DEBUG_REPORTING: { @@ -129,7 +131,7 @@ absl::Status BuyerFrontEndAsyncClientStub::ExecuteInternal( break; } } - std::move(on_done)(std::move(response)); + std::move(on_done)(std::move(response), /* response_metadata= */ {}); return absl::OkStatus(); } @@ -149,9 +151,10 @@ class ScoringClientStub std::unique_ptr request, const RequestMetadata& metadata, absl::AnyInvocable>) &&> + ScoreAdsResponse::ScoreAdsRawResponse>>, + ResponseMetadata) &&> on_done, - absl::Duration timeout) const override; + absl::Duration timeout, RequestConfig request_config) const override; }; absl::Status ScoringClientStub::Execute( @@ -167,9 +170,10 @@ absl::Status ScoringClientStub::ExecuteInternal( std::unique_ptr request, const RequestMetadata& metadata, absl::AnyInvocable>) &&> + ScoreAdsResponse::ScoreAdsRawResponse>>, + ResponseMetadata) &&> on_done, - absl::Duration timeout) const { + absl::Duration timeout, RequestConfig request_config) const { ScoreAdsResponse::ScoreAdsRawResponse response; float i = 1; // Last bid wins. @@ -189,7 +193,8 @@ absl::Status ScoringClientStub::ExecuteInternal( std::string(kDebugUrlLength, 'B'); } std::move(on_done)( - std::make_unique(response)); + std::make_unique(response), + /* response_metadata= */ {}); return absl::OkStatus(); } @@ -200,9 +205,14 @@ class BuyerFrontEndAsyncClientFactoryStub const SelectAdRequest& request, const ProtectedAuctionInput& protected_auction_input, const BuyerMockType buyer_mock_type); + std::shared_ptr Get( absl::string_view client_key) const override; + std::vector>> + Entries() const override; + private: const SelectAdRequest& request_; absl::flat_hash_map>> +BuyerFrontEndAsyncClientFactoryStub::Entries() const { + std::vector>> + entries; + for (const auto& [key, value] : buyer_clients_) { + entries.emplace_back(key, value); + } + + return entries; +} + class KeyFetcherManagerStub : public server_common::KeyFetcherManagerInterface { public: // Fetches a public key to be used for encrypting outgoing requests. diff --git a/services/seller_frontend_service/get_component_auction_ciphertexts_reactor.h b/services/seller_frontend_service/get_component_auction_ciphertexts_reactor.h index 53981424..9ff37aa0 100644 --- a/services/seller_frontend_service/get_component_auction_ciphertexts_reactor.h +++ b/services/seller_frontend_service/get_component_auction_ciphertexts_reactor.h @@ -25,8 +25,8 @@ #include "api/bidding_auction_servers.grpc.pb.h" #include "api/bidding_auction_servers.pb.h" #include "include/grpcpp/impl/codegen/server_callback.h" +#include "services/common/loggers/request_log_context.h" #include "src/encryption/key_fetcher/interface/key_fetcher_manager_interface.h" -#include "src/logger/request_context_impl.h" namespace privacy_sandbox::bidding_auction_servers { @@ -60,7 +60,7 @@ class GetComponentAuctionCiphertextsReactor : public grpc::ServerUnaryReactor { server_common::KeyFetcherManagerInterface& key_fetcher_manager_; const absl::flat_hash_map& seller_cloud_platforms_map_; - server_common::log::ContextImpl log_context_; + RequestLogContext log_context_; // Cleans up and deletes the GetComponentAuctionCiphertextsReactor. Called by // the grpc library after the response has finished. diff --git a/services/seller_frontend_service/runtime_flags.h b/services/seller_frontend_service/runtime_flags.h index aad34287..c410a59f 100644 --- a/services/seller_frontend_service/runtime_flags.h +++ b/services/seller_frontend_service/runtime_flags.h @@ -35,6 +35,8 @@ inline constexpr absl::string_view SCORE_ADS_RPC_TIMEOUT_MS = inline constexpr absl::string_view SELLER_ORIGIN_DOMAIN = "SELLER_ORIGIN_DOMAIN"; inline constexpr absl::string_view AUCTION_SERVER_HOST = "AUCTION_SERVER_HOST"; +inline constexpr absl::string_view GRPC_ARG_DEFAULT_AUTHORITY_VAL = + "GRPC_ARG_DEFAULT_AUTHORITY"; inline constexpr absl::string_view KEY_VALUE_SIGNALS_HOST = "KEY_VALUE_SIGNALS_HOST"; inline constexpr absl::string_view BUYER_SERVER_HOSTS = "BUYER_SERVER_HOSTS"; @@ -61,7 +63,7 @@ inline constexpr absl::string_view inline constexpr absl::string_view SFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES = "SFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES"; -inline constexpr int kNumRuntimeFlags = 22; +inline constexpr int kNumRuntimeFlags = 23; inline constexpr std::array kFlags = { PORT, HEALTHCHECK_PORT, @@ -70,6 +72,7 @@ inline constexpr std::array kFlags = { SCORE_ADS_RPC_TIMEOUT_MS, SELLER_ORIGIN_DOMAIN, AUCTION_SERVER_HOST, + GRPC_ARG_DEFAULT_AUTHORITY_VAL, KEY_VALUE_SIGNALS_HOST, BUYER_SERVER_HOSTS, ENABLE_BUYER_COMPRESSION, diff --git a/services/seller_frontend_service/schemas/auction_request.json b/services/seller_frontend_service/schemas/auction_request.json index 299ea9e4..4f60ed94 100644 --- a/services/seller_frontend_service/schemas/auction_request.json +++ b/services/seller_frontend_service/schemas/auction_request.json @@ -16,9 +16,10 @@ }, "generationId": { "type": "string", - "format": "uri" + "format": "uuid" }, "enableDebugReporting": { "type": "boolean" } }, - "required": ["version", "interestGroups", "publisher", "generationId"] + "required": ["version", "interestGroups", "publisher", "generationId"], + "additionalProperties": false } diff --git a/services/seller_frontend_service/schemas/auction_response.json b/services/seller_frontend_service/schemas/auction_response.json index a3a5431b..10327cfc 100644 --- a/services/seller_frontend_service/schemas/auction_response.json +++ b/services/seller_frontend_service/schemas/auction_response.json @@ -2,6 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "/SellerFrontEnd/AuctionResponse", "type": "object", + "additionalProperties": false, "$defs": { "errorDef": { "type": "object", @@ -29,6 +30,109 @@ "buyerReportingUrls": { "$ref": "#/$defs/reportingUrlsDef" }, "componentSellerReportingUrls": { "$ref": "#/$defs/reportingUrlsDef" }, "topLevelSellerReportingUrls": { "$ref": "#/$defs/reportingUrlsDef" } + }, + "KAnonJoinCandidate": { + "properties": { + "adRenderUrlHash": { + "type": "string", + "description": "Protected Audience: - SHA-256 hash of the tuple: render_url, interest group owner, reportWin() UDF endpoint. Protected App Signals: - SHA-256 hash of render_url, reportWin UDF endpoint. NOTE: The buyer's reportWin() UDF url endpoint must match with what Chrome has. Refer to the spec to create key hashes- https://wicg.github.io/turtledove/#query-ad-k-anonymity-count" + }, + "adComponentRenderUrlsHash": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Protected Audience: - SHA-256 hash of an ad_component_render_url. Refer to the spec to create key hashes- https://wicg.github.io/turtledove/#query-component-ad-k-anonymity-count Note: There is a maximum limit of 40 ad component render urls per render url. Note: Currently component ads are not in scope of Protected App Signals for Android." + }, + "reportingIdHash": { + "type": "string", + "description": "Protected Audience: - SHA-256 hash should include IG owner, ad render url, reportWin() UDF url and one of the following: - If buyerAndSellerReportingId exists, this hash should include that. - Else if buyerReportingId exists, hash should include that. - Else IG name should be included in the hash. Note: By design, an adtech can use either buyerReportingId or buyerAndSellerReportingId. Note: Currently reporting Ids are not in scope of Protected Audience for Android and Protected App Signal for Android." + } + }, + "additionalProperties": false, + "type": "object", + "title": "K Anon Join Candidate", + "description": "Join candidates for K-Anonymity for the winner. This should include key hashes corresponding to the winning ad only. Refer https://wicg.github.io/turtledove/#k-anonymity for details around how clients call the Join API using the hash." + }, + "GhostWinnerForTopLevelAuction": { + "properties": { + "adRenderUrl": { + "type": "string", + "description": "The ad render url corresponding to ghost winner." + }, + "adComponentRenderUrls": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Render URLs for ads which are components of the main ghost winning ad." + }, + "modifiedBid": { + "type": "number", + "description": "Modified bid price corresponding to ghost winning bid." + }, + "bidCurrency": { + "type": "string", + "description": "Optional. Indicates the currency used for the bid price corresponding to the ghost winner (expressed as ISO 4217 alpha code)." + }, + "adMetadata": { + "type": "string", + "description": "Arbitrary metadata associated with ghost winner to pass to the top-level seller during top level auction." + }, + "buyerAndSellerReportingId": { + "type": "string", + "description": "BuyerAndSellerReportingId of the ghost winning Ad. This will be verified with the buyerAndSellerReportingId in the Ad properties on the browser." + } + }, + "additionalProperties": false, + "type": "object", + "title": "Ghost Winner For Top Level Auction", + "description": "In case of multiseller auction, the associated data for the ghost winner will be returned so that the ghost winning bid can be scored during top level auction. This is true if there is a ghost winner after component level auction. - In case of device orchestrated component auction for web, this data will returned to the browser so that the ghost winner can be passed to the top level auction on-device. - In case of server orchestrated component auction (for web, Android), this data will be returned to the top level seller by the component level sellers." + }, + "GhostWinnerPrivateAggregationSignals": { + "properties": { + "bucket": { + "type": "string", + "description": "128 bit integer in the form of bytestring.", + "format": "binary" + }, + "value": { + "type": "integer" + } + }, + "additionalProperties": false, + "type": "object", + "title": "Ghost Winner Private Aggregation Signals", + "description": "Private aggregation signals for the ghost winner. Single seller auctions: This would correspond to a ghost winner if available. Note: Event type is \"reserved.loss\" and bid rejection reason is 8 when K-Anonymity threshold is not. The client should incorporate these static values; these won't be sent back." + }, + "KAnonGhostWinner": { + "properties": { + "kAnonJoinCandidates": { + "$ref": "#/$defs/KAnonJoinCandidate", + "additionalProperties": false, + "description": "Join candidates for the K-Anon ghost winner." + }, + "interestGroupIndex": { + "type": "integer", + "description": "Interest group index corresponding to buyer / DSP that generated the ghost winning bid. Note: This is only relevant in case of Protected Audience." + }, + "owner": { + "type": "string", + "description": "Origin (Chrome) and domain (Android) of the buyer / DSP who owns the ghost winner. Protected Audience: This refers to the Interest Group owner. Proptected App Signal: This refers to the buyer domain." + }, + "ghostWinnerPrivateAggregationSignals": { + "$ref": "#/$defs/GhostWinnerPrivateAggregationSignals", + "additionalProperties": false + }, + "ghostWinnerForTopLevelAuction": { + "$ref": "#/$defs/GhostWinnerForTopLevelAuction", + "additionalProperties": false + } + }, + "additionalProperties": false, + "type": "object", + "title": "K Anon Ghost Winner", + "description": "Data for the ghost winner sent back to the client. This should also include key-hashes corresponding to the ghost winning ad. Refer https://wicg.github.io/turtledove/#k-anonymity for details around how clients call the Join API using the hash." } }, "properties": { @@ -98,6 +202,21 @@ "topLevelSeller": { "type": "string", "description": "Optional name/domain for top-level seller in case this is a component auction." + }, + "kAnonWinnerJoinCandidates": { + "$ref": "#/$defs/KAnonJoinCandidate", + "additionalProperties": false + }, + "kAnonWinnerPositionalIndex": { + "type": "integer", + "description": "Positional index of the k-anon winner, if there is a winner returned to the client (web browser, Android) for emitting metrics. Note: Positional index \u003e= 0. - If this is equal to 0, that would imply the highest scored bid is also K-Anonymous and hence a winner. - If this is greater than 0, the positional index would imply the index of first K-Anonymous scored bid in a sorted list in decreasing order of scored bids. In this case, the highest scored bid that is not K-Anonymous is the ghost winner. - In case all scored bids fail the K-Anonymity constraint, this would be set to -1 since there is no winner. - In case all scored bids \u003c= 0, this would be set to -1 since there is no winner." + }, + "kAnonGhostWinners": { + "items": { + "$ref": "#/$defs/KAnonGhostWinner" + }, + "additionalProperties": false, + "type": "array" } } } diff --git a/services/seller_frontend_service/schemas/examples/auction_response.json b/services/seller_frontend_service/schemas/examples/auction_response.json index 1f06f83b..21d7a2f0 100644 --- a/services/seller_frontend_service/schemas/examples/auction_response.json +++ b/services/seller_frontend_service/schemas/examples/auction_response.json @@ -1,6 +1,6 @@ { "adRenderURL": "http://foo.com/ad", - "componentAds": ["http://foo/com/ad/component1", "http://foo/com/ad/component2"], + "components": ["http://foo/com/ad/component1", "http://foo/com/ad/component2"], "interestGroupName": "ad_loving_group", "interestGroupOwner": "ad_showing_company", "biddingGroups": { @@ -33,5 +33,34 @@ "error": { "code": 0, "message": "Success" - } + }, + "kAnonWinnerJoinCandidates": { + "adRenderUrlHash": "ad-render-url-hash", + "adComponentRenderUrlsHash": ["ad-component-render-urls-hash"], + "reportingIdHash": "reporting-id-hash" + }, + "kAnonWinnerPositionalIndex": 0, + "kAnonGhostWinners": [ + { + "kAnonJoinCandidates": { + "adRenderUrlHash": "ad-render-url-hash", + "adComponentRenderUrlsHash": ["ad-component-render-urls-hash"], + "reportingIdHash": "reporting-id-hash" + }, + "interestGroupIndex": 23, + "owner": "test-owner", + "ghostWinnerPrivateAggregationSignals": { + "bucket": "a-test-bucket", + "value": 51 + }, + "ghostWinnerForTopLevelAuction": { + "adRenderUrl": "an-ad-render-url", + "adComponentRenderUrls": ["ad-component-render-url1"], + "modifiedBid": 123.1234, + "bidCurrency": "USD", + "adMetadata": "arbitrary-data", + "buyerAndSellerReportingId": "test-id" + } + } + ] } diff --git a/services/seller_frontend_service/select_ad_reactor.cc b/services/seller_frontend_service/select_ad_reactor.cc index 6233aa14..66a7ccb9 100644 --- a/services/seller_frontend_service/select_ad_reactor.cc +++ b/services/seller_frontend_service/select_ad_reactor.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include "absl/container/flat_hash_set.h" #include "absl/flags/flag.h" @@ -30,6 +31,7 @@ #include "api/bidding_auction_servers.pb.h" #include "quiche/oblivious_http/oblivious_http_gateway.h" #include "services/common/constants/user_error_strings.h" +#include "services/common/feature_flags.h" #include "services/common/reporters/async_reporter.h" #include "services/common/util/auction_scope_util.h" #include "services/common/util/reporting_util.h" @@ -212,11 +214,19 @@ grpc::Status SelectAdReactor::DecryptRequest() { std::visit( [this](auto& input) { buyer_inputs_ = GetDecodedBuyerinputs(input.buyer_input()); - for (auto& each_buyer_input : *input.mutable_buyer_input()) { - each_buyer_input.second = "encoded buyer input erased"; - } }, protected_auction_input_); + + if (absl::GetFlag(FLAGS_enable_chaffing)) { + std::visit( + [this](const auto& protected_auction_input) { + std::size_t hash = absl::Hash{}( + protected_auction_input.generation_id()); + generator_ = std::mt19937(hash); + }, + protected_auction_input_); + } + return grpc::Status::OK; } @@ -275,11 +285,6 @@ void SelectAdReactor::MayPopulateAdServerVisibleErrors() { } } - if (request_->client_type() == CLIENT_TYPE_UNKNOWN) { - ReportError(ErrorVisibility::AD_SERVER_VISIBLE, kUnknownClientType, - ErrorCode::CLIENT_SIDE); - } - // Device Component Auction not allowed with Android client type. if (request_->client_type() == CLIENT_TYPE_ANDROID && auction_scope_ == @@ -305,6 +310,153 @@ void SelectAdReactor::MayLogBuyerInput() { } } +ChaffingConfig SelectAdReactor::GetChaffingConfig( + const absl::flat_hash_set& auction_config_buyer_set) { + if (!absl::GetFlag(FLAGS_enable_chaffing)) { + return {}; + } + + int num_chaff_requests = 0; + absl::flat_hash_set chaff_request_candidates; + + // 'real' requests means non-chaff requests, e.g. buyers that will actually + // generate bids to show an ad to this client. + int num_real_requests = 0; + + // Loop through the buyers this SFE instance is configured to talk to. + for (const auto& [buyer, unused] : clients_.buyer_factory.Entries()) { + (void)unused; + if (!auction_config_buyer_set.contains(buyer)) { + continue; + } + + // Check if the buyer is contained in the top level auction config of the + // SelectAd request. + if (buyer_inputs_->find(buyer) == buyer_inputs_->end()) { + // If the buyer is NOT in the ciphertext, e.g. the buyer's interest + // groups, they are a chaff request candidate. + chaff_request_candidates.emplace(buyer); + } else { + // If the buyer was present in all 3 (SFE config, auction config, and + // ciphertext), we will send this buyer a 'real' GetBids request. + num_real_requests++; + } + } + + // Use the RNG seeded w/ the generation ID to deterministically determine + // how many chaff requests to send for a given ciphertext. + if (!chaff_request_candidates.empty()) { + // We cannot have a static lower bound for the number of chaff requests the + // SFE sends. If we did and a network snooper ever saw that many requests go + // out, that would leak that the client was not associated with any of the + // buyers. To avoid this, we (attempt to) set the lower bound for + // num_chaff_requests to 2 when num_real_requests = 0, and 1 otherwise. + int num_chaff_requests_lower_bound = kMinChaffRequests; + if (num_real_requests == 0) { + num_chaff_requests_lower_bound = kMinChaffRequestsWithNoRealRequests; + } + + num_chaff_requests_lower_bound = std::min( + num_chaff_requests_lower_bound, (int)chaff_request_candidates.size()); + std::uniform_int_distribution num_chaff_buyers_dist( + num_chaff_requests_lower_bound, chaff_request_candidates.size()); + num_chaff_requests = num_chaff_buyers_dist(*generator_); + } + + return {.chaff_request_candidates = std::move(chaff_request_candidates), + .num_chaff_requests = num_chaff_requests, + .num_real_requests = num_real_requests}; +} + +void SelectAdReactor::FetchBids() { + const bool chaffing_enabled = absl::GetFlag(FLAGS_enable_chaffing); + + int num_buyers_solicited = 0; + absl::flat_hash_set auction_config_buyer_set( + request_->auction_config().buyer_list().begin(), + request_->auction_config().buyer_list().end()); + + ChaffingConfig chaffing_config = GetChaffingConfig(auction_config_buyer_set); + if (chaffing_enabled) { + async_task_tracker_.SetNumTasksToTrack(chaffing_config.num_real_requests + + chaffing_config.num_chaff_requests); + + if ((chaffing_config.num_real_requests + + chaffing_config.num_chaff_requests) == 0) { + OnAllBidsDone(true); + return; + } + } + + for (const auto& buyer_ig_owner : request_->auction_config().buyer_list()) { + if (chaffing_enabled && + chaffing_config.chaff_request_candidates.contains(buyer_ig_owner)) { + // We verify above that any buyers in chaff_request_candidates are not + // present in the ciphertext. + continue; + } + + if (auction_config_buyer_set.erase(buyer_ig_owner) == 0) { + PS_VLOG(kNoisyWarn, log_context_) + << "Duplicate buyer found " << buyer_ig_owner << ", skipping buyer"; + async_task_tracker_.TaskCompleted(TaskStatus::SKIPPED); + continue; + } + + if (num_buyers_solicited >= max_buyers_solicited_) { + // Skipped buyers should not be left pending. + async_task_tracker_.TaskCompleted(TaskStatus::SKIPPED); + PS_VLOG(kNoisyWarn, log_context_) + << "Exceeded cap of " << max_buyers_solicited_ + << " buyers called. Skipping buyer"; + continue; + } + + const auto& buyer_input_iterator = buyer_inputs_->find(buyer_ig_owner); + if (buyer_input_iterator == buyer_inputs_->end()) { + PS_VLOG(kNoisyWarn, log_context_) + << "No buyer input found for buyer: " << buyer_ig_owner + << ", skipping buyer"; + + // Pending bids count is set on reactor construction to + // buyer_list_size(). If no BuyerInput is found for a buyer in + // buyer_list, must decrement pending bids count. + async_task_tracker_.TaskCompleted(TaskStatus::SKIPPED); + continue; + } + + auto get_bids_request = + CreateGetBidsRequest(buyer_ig_owner, buyer_input_iterator->second); + FetchBid(buyer_ig_owner, std::move(get_bids_request)); + num_buyers_solicited++; + } + + if (!chaffing_enabled || chaffing_config.chaff_request_candidates.empty()) { + return; + } + + // Loop through chaff candidates and send 'num_chaff_requests' requests. + for (auto it = chaffing_config.chaff_request_candidates.begin(); + it != chaffing_config.chaff_request_candidates.end(); ++it) { + if (std::distance(it, chaffing_config.chaff_request_candidates.begin()) >= + chaffing_config.num_chaff_requests) { + break; + } + + auto get_bids_request = + std::make_unique(); + get_bids_request->set_is_chaff(true); + auto* log_context = get_bids_request->mutable_log_context(); + std::visit( + [log_context](const auto& protected_auction_input) { + log_context->set_generation_id( + protected_auction_input.generation_id()); + }, + protected_auction_input_); + FetchBid(it->data(), std::move(get_bids_request)); + } +} + void SelectAdReactor::Execute() { grpc::Status decrypt_status = DecryptRequest(); @@ -346,6 +498,8 @@ void SelectAdReactor::Execute() { PS_VLOG(kEncrypted, log_context_) << "Encrypted SelectAdRequest:\n" << request_->ShortDebugString(); + log_context_.SetEventMessageField(*request_); + PS_VLOG(kPlain, log_context_) << "Headers:\n" << absl::StrJoin(context_->client_metadata(), "\n", @@ -358,7 +512,11 @@ void SelectAdReactor::Execute() { return; } std::visit( - [this](const auto& input) { + [this](auto& input) { + log_context_.SetEventMessageField(input); + for (auto& each_buyer_input : *input.mutable_buyer_input()) { + each_buyer_input.second = "encoded buyer input erased"; + } PS_VLOG(kPlain, log_context_) << (is_protected_auction_request_ ? "ProtectedAuctionInput" : "ProtectedAudienceInput") @@ -411,46 +569,7 @@ void SelectAdReactor::Execute() { PS_VLOG(6, log_context_) << "Buyer list size: " << request_->auction_config().buyer_list().size(); - int num_buyers_solicited = 0; - absl::flat_hash_set buyer_set( - request_->auction_config().buyer_list().begin(), - request_->auction_config().buyer_list().end()); - - for (const auto& buyer_ig_owner : request_->auction_config().buyer_list()) { - if (buyer_set.erase(buyer_ig_owner) == 0) { - PS_VLOG(kNoisyWarn, log_context_) - << "Duplicate buyer found " << buyer_ig_owner << ", skipping buyer"; - async_task_tracker_.TaskCompleted(TaskStatus::SKIPPED); - continue; - } - - if (num_buyers_solicited >= max_buyers_solicited_) { - // Skipped buyers should not be left pending. - async_task_tracker_.TaskCompleted(TaskStatus::SKIPPED); - PS_VLOG(kNoisyWarn, log_context_) - << "Exceeded cap of " << max_buyers_solicited_ - << " buyers called. Skipping buyer"; - continue; - } - - const auto& buyer_input_iterator = buyer_inputs_->find(buyer_ig_owner); - if (buyer_input_iterator == buyer_inputs_->end()) { - PS_VLOG(kNoisyWarn, log_context_) - << "No buyer input found for buyer: " << buyer_ig_owner - << ", skipping buyer"; - - // Pending bids count is set on reactor construction to - // buyer_list_size(). If no BuyerInput is found for a buyer in - // buyer_list, must decrement pending bids count. - async_task_tracker_.TaskCompleted(TaskStatus::SKIPPED); - continue; - } - - FetchBid(buyer_ig_owner, buyer_input_iterator->second); - num_buyers_solicited++; - } - PS_VLOG(5, log_context_) - << "Finishing execute call, response may be available later"; + FetchBids(); } std::unique_ptr @@ -477,6 +596,7 @@ SelectAdReactor::CreateGetBidsRequest(const std::string& buyer_ig_owner, per_buyer_config_itr->second.buyer_signals()); } } + *get_bids_request->mutable_buyer_input() = buyer_input; get_bids_request->set_top_level_seller( request_->auction_config().top_level_seller()); @@ -509,56 +629,69 @@ SelectAdReactor::CreateGetBidsRequest(const std::string& buyer_ig_owner, return get_bids_request; } -void SelectAdReactor::FetchBid(const std::string& buyer_ig_owner, - const BuyerInput& buyer_input) { - auto buyer_client = clients_.buyer_factory.Get(buyer_ig_owner); +void SelectAdReactor::FetchBid( + const std::string& buyer_ig_owner, + std::unique_ptr get_bids_request) { + const std::shared_ptr& buyer_client = + clients_.buyer_factory.Get(buyer_ig_owner); + if (buyer_client == nullptr) { PS_VLOG(kNoisyWarn, log_context_) << "No buyer client found for buyer: " << buyer_ig_owner; async_task_tracker_.TaskCompleted(TaskStatus::SKIPPED); - } else { - PS_VLOG(6, log_context_) << "Getting bid from a BFE"; - absl::Duration timeout = absl::Milliseconds( - config_client_.GetIntParameter(GET_BID_RPC_TIMEOUT_MS)); - if (request_->auction_config().buyer_timeout_ms() > 0) { - timeout = - absl::Milliseconds(request_->auction_config().buyer_timeout_ms()); - } - auto get_bids_request = CreateGetBidsRequest(buyer_ig_owner, buyer_input); - auto bfe_request = - metric::MakeInitiatedRequest(metric::kBfe, metric_context_.get()); - bfe_request->SetBuyer(buyer_ig_owner); - bfe_request->SetRequestSize((int)get_bids_request->ByteSizeLong()); - absl::Status execute_result = buyer_client->ExecuteInternal( - std::move(get_bids_request), buyer_metadata_, - [buyer_ig_owner, this, bfe_request = std::move(bfe_request)]( - absl::StatusOr> - response) mutable { - { - int response_size = - response.ok() ? (int)response->get()->ByteSizeLong() : 0; - bfe_request->SetResponseSize(response_size); - - // destruct bfe_request, destructor measures request time - auto not_used = std::move(bfe_request); - } - PS_VLOG(6, log_context_) << "Received a response from a BFE"; - OnFetchBidsDone(std::move(response), buyer_ig_owner); - }, - timeout); - if (!execute_result.ok()) { - LogIfError( - metric_context_->AccumulateMetric( - 1, metric::kSfeGetBidsFailedToCall)); - PS_LOG(ERROR, log_context_) << absl::StrFormat( - "Failed to make async GetBids call: (buyer: %s, " - "seller: %s, error: " - "%s)", - buyer_ig_owner, request_->auction_config().seller(), - execute_result.ToString()); - async_task_tracker_.TaskCompleted(TaskStatus::ERROR); - } + return; + } + + PS_VLOG(6, log_context_) << "Getting bid from a BFE"; + + absl::Duration timeout = absl::Milliseconds( + config_client_.GetIntParameter(GET_BID_RPC_TIMEOUT_MS)); + if (request_->auction_config().buyer_timeout_ms() > 0) { + timeout = absl::Milliseconds(request_->auction_config().buyer_timeout_ms()); + } + auto bfe_request = + metric::MakeInitiatedRequest(metric::kBfe, metric_context_.get()); + bfe_request->SetBuyer(buyer_ig_owner); + bfe_request->SetRequestSize((int)get_bids_request->ByteSizeLong()); + + size_t chaff_request_size = 0; + if (absl::GetFlag(FLAGS_enable_chaffing)) { + std::uniform_int_distribution request_size_dist( + kMinChaffRequestSizeBytes, kMaxChaffRequestSizeBytes); + chaff_request_size = request_size_dist(*generator_); + } + + RequestConfig request_config = {.chaff_request_size = chaff_request_size}; + + absl::Status execute_result = buyer_client->ExecuteInternal( + std::move(get_bids_request), buyer_metadata_, + [buyer_ig_owner, this, bfe_request = std::move(bfe_request)]( + absl::StatusOr> + response, + ResponseMetadata response_metadata) mutable { + { + int response_size = std::max(0UL, response_metadata.response_size); + bfe_request->SetResponseSize(response_size); + + // destruct bfe_request, destructor measures request time + auto not_used = std::move(bfe_request); + } + PS_VLOG(6, log_context_) << "Received a response from a BFE"; + OnFetchBidsDone(std::move(response), buyer_ig_owner); + }, + timeout, request_config); + if (!execute_result.ok()) { + LogIfError( + metric_context_->AccumulateMetric( + 1, metric::kSfeGetBidsFailedToCall)); + PS_LOG(ERROR, log_context_) << absl::StrFormat( + "Failed to make async GetBids call: (buyer: %s, " + "seller: %s, error: " + "%s)", + buyer_ig_owner, request_->auction_config().seller(), + execute_result.ToString()); + async_task_tracker_.TaskCompleted(TaskStatus::ERROR); } } @@ -909,7 +1042,8 @@ void SelectAdReactor::ScoreAds() { auto on_scoring_done = [this, auction_request = std::move(auction_request)]( absl::StatusOr> - result) mutable { + result, + ResponseMetadata response_metadata) mutable { { int response_size = result.ok() ? (int)result->get()->ByteSizeLong() : 0; @@ -944,6 +1078,7 @@ void SelectAdReactor::FinishWithStatus(const grpc::Status& status) { static_cast((absl::Now() - start_) / absl::Milliseconds(1)))); } benchmarking_logger_->End(); + log_context_.ExportEventMessage(); Finish(status); } diff --git a/services/seller_frontend_service/select_ad_reactor.h b/services/seller_frontend_service/select_ad_reactor.h index e9cf6f47..eecb5919 100644 --- a/services/seller_frontend_service/select_ad_reactor.h +++ b/services/seller_frontend_service/select_ad_reactor.h @@ -64,6 +64,18 @@ inline constexpr absl::string_view kWinningAd = "winning_ad"; inline constexpr char kValidCurrencyCodePattern[] = "^[A-Z]{3}$"; const std::regex kValidCurrencyCodeRegex(kValidCurrencyCodePattern); +inline constexpr int kMinChaffRequestSizeBytes = 9000; +inline constexpr int kMaxChaffRequestSizeBytes = 95000; + +inline constexpr int kMinChaffRequests = 1; +inline constexpr int kMinChaffRequestsWithNoRealRequests = 2; + +struct ChaffingConfig { + absl::flat_hash_set chaff_request_candidates; + int num_chaff_requests = 0; + int num_real_requests = 0; +}; + // This is a gRPC reactor that serves a single GenerateBidsRequest. // It stores state relevant to the request and after the // response is finished being served, SelectAdReactor cleans up all @@ -203,13 +215,18 @@ class SelectAdReactor : public grpc::ServerUnaryReactor { // whether decryption was successful. grpc::Status DecryptRequest(); + // Dispatches the GetBids calls for both 'real' and 'fake' (AKA chaff) buyers. + void FetchBids(); + // Fetches the bids from a single buyer by initiating an asynchronous GetBids // rpc. // // buyer: a string representing the buyer, identified as an IG owner. - // buyer_input: input for bidding. - void FetchBid(const std::string& buyer_ig_owner, - const BuyerInput& buyer_input); + // get_bids_request: the GetBids request proto for the specified buyer. + void FetchBid( + const std::string& buyer_ig_owner, + std::unique_ptr get_bids_request); + // Handles recording the fetched bid to state. // This is called by the grpc buyer client when the request is finished, // and will subsequently call update pending bids state which will update how @@ -314,7 +331,7 @@ class SelectAdReactor : public grpc::ServerUnaryReactor { // Encryption context needed throughout the lifecycle of the request. std::unique_ptr decrypted_request_; - server_common::log::ContextImpl log_context_; + RequestLogContext log_context_; // Decompressed and decoded buyer inputs. absl::StatusOr> @@ -346,6 +363,9 @@ class SelectAdReactor : public grpc::ServerUnaryReactor { // Temporary workaround for compliance, will be removed (b/308032414). const int max_buyers_solicited_; + // Pseudo random number generator for use in chaffing. + std::optional generator_; + private: // Keeps track of how many buyer bids were expected initially and how many // were erroneous. If all bids ended up in an error state then that should be @@ -358,6 +378,9 @@ class SelectAdReactor : public grpc::ServerUnaryReactor { const absl::Status& status, absl::string_view buyer = ""); + ChaffingConfig GetChaffingConfig( + const absl::flat_hash_set& auction_config_buyer_set); + absl::Time start_ = absl::Now(); }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/select_ad_reactor_app.cc b/services/seller_frontend_service/select_ad_reactor_app.cc index 394f2abf..2c5e38ce 100644 --- a/services/seller_frontend_service/select_ad_reactor_app.cc +++ b/services/seller_frontend_service/select_ad_reactor_app.cc @@ -92,6 +92,7 @@ absl::StatusOr SelectAdReactorForApp::GetNonEncryptedResponse( } PS_VLOG(kPlain, log_context_) << "AuctionResult:\n" << auction_result.ShortDebugString(); + log_context_.SetEventMessageField(auction_result); // Serialized the data to bytes array. std::string serialized_result = auction_result.SerializeAsString(); diff --git a/services/seller_frontend_service/select_ad_reactor_app_test.cc b/services/seller_frontend_service/select_ad_reactor_app_test.cc index efa310f6..2f7822d1 100644 --- a/services/seller_frontend_service/select_ad_reactor_app_test.cc +++ b/services/seller_frontend_service/select_ad_reactor_app_test.cc @@ -32,6 +32,7 @@ #include "quiche/oblivious_http/oblivious_http_client.h" #include "quiche/oblivious_http/oblivious_http_gateway.h" #include "services/common/compression/gzip.h" +#include "services/common/feature_flags.h" #include "services/common/metric/server_definition.h" #include "services/common/test/mocks.h" #include "services/common/test/random.h" @@ -55,12 +56,12 @@ using ::testing::HasSubstr; using ::testing::Return; using EncodedBueryInputs = ::google::protobuf::Map; using DecodedBueryInputs = ::google::protobuf::Map; -using GetBidDoneCallback = - absl::AnyInvocable>) &&>; -using ScoreAdsDoneCallback = - absl::AnyInvocable>) &&>; +using GetBidDoneCallback = absl::AnyInvocable< + void(absl::StatusOr>, + ResponseMetadata response_metadata) &&>; +using ScoreAdsDoneCallback = absl::AnyInvocable< + void(absl::StatusOr>, + ResponseMetadata response_metadata) &&>; inline constexpr int kTestBidValue = 10.0; inline constexpr int kTestAdCost = 2.0; @@ -86,6 +87,7 @@ class SelectAdReactorForAppTest : public ::testing::Test { config_.SetFlagForTest("", CONSENTED_DEBUG_TOKEN); config_.SetFlagForTest(kFalse, ENABLE_PROTECTED_APP_SIGNALS); config_.SetFlagForTest(kTrue, ENABLE_PROTECTED_AUDIENCE); + absl::SetFlag(&FLAGS_enable_chaffing, false); } TrustedServersConfigClient config_ = CreateConfig(); @@ -111,6 +113,9 @@ TYPED_TEST(SelectAdReactorForAppTest, VerifyEncoding) { CLIENT_TYPE_ANDROID, kNonZeroBidValue, scoring_signals_provider, scoring_client, buyer_front_end_async_client_factory_mock, key_fetcher_manager.get(), expected_buyer_bids, kSellerOriginDomain); + MockEntriesCallOnBuyerFactory( + request_with_context.protected_auction_input.buyer_input(), + buyer_front_end_async_client_factory_mock); SelectAdResponse encrypted_response = RunReactorRequest( @@ -189,6 +194,10 @@ TYPED_TEST(SelectAdReactorForAppTest, VerifyEncodingWithReportingUrls) { key_fetcher_manager.get(), expected_buyer_bids, kSellerOriginDomain, expect_all_buyers_solicited, top_level_seller, enable_reporting); + MockEntriesCallOnBuyerFactory( + request_with_context.protected_auction_input.buyer_input(), + buyer_front_end_async_client_factory_mock); + SelectAdResponse encrypted_response = RunReactorRequest( this->config_, clients, request_with_context.select_ad_request); @@ -206,7 +215,6 @@ TYPED_TEST(SelectAdReactorForAppTest, VerifyEncodingWithReportingUrls) { EXPECT_EQ(payload_size, 1 << log_2_payload); EXPECT_GE(payload_size, kMinAuctionResultBytes); - // Unframe the framed response. // Unframe the framed response. absl::StatusOr unframed_response = server_common::DecodeRequestPayload(*decrypted_response); @@ -280,6 +288,10 @@ TYPED_TEST(SelectAdReactorForAppTest, VerifyEncodingForServerComponentAuction) { {&crypto_client, EncryptionCloudPlatform::ENCRYPTION_CLOUD_PLATFORM_GCP}); + MockEntriesCallOnBuyerFactory( + request_with_context.protected_auction_input.buyer_input(), + buyer_front_end_async_client_factory_mock); + SelectAdResponse encrypted_response = RunReactorRequest( this->config_, clients, request_with_context.select_ad_request); @@ -333,6 +345,9 @@ TYPED_TEST(SelectAdReactorForAppTest, VerifyChaffedResponse) { CLIENT_TYPE_ANDROID, kZeroBidValue, scoring_signals_provider, scoring_client, buyer_front_end_async_client_factory_mock, key_fetcher_manager.get(), expected_buyer_bids, kSellerOriginDomain); + MockEntriesCallOnBuyerFactory( + request_with_context.protected_auction_input.buyer_input(), + buyer_front_end_async_client_factory_mock); SelectAdResponse encrypted_response = RunReactorRequest( @@ -445,6 +460,7 @@ class SelectAdReactorPASTest : public ::testing::Test { config_.SetFlagForTest("", CONSENTED_DEBUG_TOKEN); config_.SetFlagForTest(kTrue, ENABLE_PROTECTED_APP_SIGNALS); config_.SetFlagForTest(kTrue, ENABLE_PROTECTED_AUDIENCE); + absl::SetFlag(&FLAGS_enable_chaffing, false); EXPECT_CALL(*key_fetcher_manager_, GetPrivateKey) .WillRepeatedly(Return(GetPrivateKey())); @@ -599,7 +615,8 @@ TEST_F(SelectAdReactorPASTest, PASBuyerInputIsPopulatedForGetBids) { get_bids_raw_request, const RequestMetadata& metadata, GetBidDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, + RequestConfig request_config) { // Expect PAS buyer inputs to be populated correctly in GetBids. EXPECT_TRUE(get_bids_raw_request->has_protected_app_signals_buyer_input()); EXPECT_TRUE(get_bids_raw_request->protected_app_signals_buyer_input() @@ -617,8 +634,7 @@ TEST_F(SelectAdReactorPASTest, PASBuyerInputIsPopulatedForGetBids) { }; auto setup_mock_buyer = [&mock_get_bids](std::unique_ptr buyer) { - EXPECT_CALL(*buyer, ExecuteInternal(_, _, _, _)) - .WillRepeatedly(mock_get_bids); + EXPECT_CALL(*buyer, ExecuteInternal).WillRepeatedly(mock_get_bids); return buyer; }; auto MockBuyerFactoryCall = [setup_mock_buyer](absl::string_view hostname) { @@ -638,8 +654,9 @@ TEST_F(SelectAdReactorPASTest, PASBuyerInputIsClearedIfFeatureNotAvailable) { auto mock_get_bids = [](std::unique_ptr get_bids_raw_request, const RequestMetadata& metadata, - GetBidDoneCallback on_done, absl::Duration timeout) { - // Expect PAS buyer inputs to be not present in GetBids. + GetBidDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { + // Expect PAS buyer inputs to be populated correctly in GetBids. EXPECT_FALSE(get_bids_raw_request->has_protected_app_signals_buyer_input()); // Ensure PA buyer inputs doesn't have the PAS data. @@ -649,8 +666,7 @@ TEST_F(SelectAdReactorPASTest, PASBuyerInputIsClearedIfFeatureNotAvailable) { }; auto setup_mock_buyer = [&mock_get_bids](std::unique_ptr buyer) { - EXPECT_CALL(*buyer, ExecuteInternal(_, _, _, _)) - .WillRepeatedly(mock_get_bids); + EXPECT_CALL(*buyer, ExecuteInternal).WillRepeatedly(mock_get_bids); return buyer; }; auto MockBuyerFactoryCall = [setup_mock_buyer](absl::string_view hostname) { @@ -675,16 +691,16 @@ TEST_F(SelectAdReactorPASTest, PASAdWithBidIsSentForScoring) { get_bids_raw_request, const RequestMetadata& metadata, GetBidDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, + RequestConfig request_config) { auto response = std::make_unique(); response->mutable_protected_app_signals_bids()->Add(GetTestPASAdWithBid()); - std::move(on_done)(std::move(response)); + std::move(on_done)(std::move(response), /* response_metadata= */ {}); return absl::OkStatus(); }; auto setup_mock_buyer = [&mock_get_bids](std::unique_ptr buyer) { - EXPECT_CALL(*buyer, ExecuteInternal(_, _, _, _)) - .WillRepeatedly(mock_get_bids); + EXPECT_CALL(*buyer, ExecuteInternal).WillRepeatedly(mock_get_bids); return buyer; }; auto MockBuyerFactoryCall = [setup_mock_buyer](absl::string_view hostname) { @@ -692,13 +708,16 @@ TEST_F(SelectAdReactorPASTest, PASAdWithBidIsSentForScoring) { }; EXPECT_CALL(buyer_front_end_async_client_factory_mock_, Get(_)) .WillRepeatedly(MockBuyerFactoryCall); + MockEntriesCallOnBuyerFactory( + request_with_context.protected_auction_input.buyer_input(), + buyer_front_end_async_client_factory_mock_); EXPECT_CALL(scoring_client_, ExecuteInternal) .WillOnce( [this, &select_ad_req, &protected_auction_input]( std::unique_ptr request, const RequestMetadata& metadata, ScoreAdsDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { EXPECT_EQ(request->publisher_hostname(), protected_auction_input.publisher_name()); EXPECT_EQ(request->seller_signals(), @@ -750,16 +769,16 @@ TEST_F(SelectAdReactorPASTest, get_bids_raw_request, const RequestMetadata& metadata, GetBidDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, + RequestConfig request_config) { auto response = std::make_unique(); response->mutable_protected_app_signals_bids()->Add(GetTestPASAdWithBid()); - std::move(on_done)(std::move(response)); + std::move(on_done)(std::move(response), /* response_metadata= */ {}); return absl::OkStatus(); }; auto setup_mock_buyer = [&mock_get_bids](std::unique_ptr buyer) { - EXPECT_CALL(*buyer, ExecuteInternal(_, _, _, _)) - .WillRepeatedly(mock_get_bids); + EXPECT_CALL(*buyer, ExecuteInternal).WillRepeatedly(mock_get_bids); return buyer; }; auto MockBuyerFactoryCall = [setup_mock_buyer](absl::string_view hostname) { @@ -785,7 +804,8 @@ TEST_F(SelectAdReactorPASTest, PASOnlyBuyerInputIsAllowed) { get_bids_raw_request, const RequestMetadata& metadata, GetBidDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, + RequestConfig request_config) { // Expect PAS buyer inputs to be populated correctly in GetBids. EXPECT_TRUE(get_bids_raw_request->has_protected_app_signals_buyer_input()); EXPECT_TRUE(get_bids_raw_request->protected_app_signals_buyer_input() @@ -803,8 +823,7 @@ TEST_F(SelectAdReactorPASTest, PASOnlyBuyerInputIsAllowed) { }; auto setup_mock_buyer = [&mock_get_bids](std::unique_ptr buyer) { - EXPECT_CALL(*buyer, ExecuteInternal(_, _, _, _)) - .WillRepeatedly(mock_get_bids); + EXPECT_CALL(*buyer, ExecuteInternal).WillRepeatedly(mock_get_bids); return buyer; }; auto MockBuyerFactoryCall = [setup_mock_buyer](absl::string_view hostname) { @@ -831,7 +850,8 @@ TEST_F(SelectAdReactorPASTest, BothPASAndPAInputsMissingIsAnError) { get_bids_raw_request, const RequestMetadata& metadata, GetBidDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, + RequestConfig request_config) { // Expect PAS buyer inputs to be populated correctly in GetBids. EXPECT_TRUE(get_bids_raw_request->has_protected_app_signals_buyer_input()); EXPECT_TRUE(get_bids_raw_request->protected_app_signals_buyer_input() @@ -849,8 +869,7 @@ TEST_F(SelectAdReactorPASTest, BothPASAndPAInputsMissingIsAnError) { }; auto setup_mock_buyer = [&mock_get_bids](std::unique_ptr buyer) { - EXPECT_CALL(*buyer, ExecuteInternal(_, _, _, _)) - .WillRepeatedly(mock_get_bids); + EXPECT_CALL(*buyer, ExecuteInternal).WillRepeatedly(mock_get_bids); return buyer; }; auto MockBuyerFactoryCall = [setup_mock_buyer](absl::string_view hostname) { @@ -896,22 +915,24 @@ TEST_F(SelectAdReactorPASTest, CreateSelectAdRequest(kSellerOriginDomain, /*add_interest_group=*/true, /*add_protected_app_signals=*/false); - auto mock_get_bids = [](std::unique_ptr - get_bids_raw_request, - const RequestMetadata& metadata, - GetBidDoneCallback on_done, absl::Duration timeout) { - // Expect PAS buyer inputs to not be be populated in GetBids. - EXPECT_FALSE(get_bids_raw_request->has_protected_app_signals_buyer_input()); - - // Ensure PA buyer inputs doesn't have the PAS data. - EXPECT_FALSE( - get_bids_raw_request->buyer_input().has_protected_app_signals()); - return absl::OkStatus(); - }; + auto mock_get_bids = + [](std::unique_ptr + get_bids_raw_request, + const RequestMetadata& metadata, GetBidDoneCallback on_done, + absl::Duration timeout, + RequestConfig request_config) { // Expect PAS buyer inputs to not be + // be populated in GetBids. + EXPECT_FALSE( + get_bids_raw_request->has_protected_app_signals_buyer_input()); + + // Ensure PA buyer inputs doesn't have the PAS data. + EXPECT_FALSE( + get_bids_raw_request->buyer_input().has_protected_app_signals()); + return absl::OkStatus(); + }; auto setup_mock_buyer = [&mock_get_bids](std::unique_ptr buyer) { - EXPECT_CALL(*buyer, ExecuteInternal(_, _, _, _)) - .WillRepeatedly(mock_get_bids); + EXPECT_CALL(*buyer, ExecuteInternal).WillRepeatedly(mock_get_bids); return buyer; }; auto MockBuyerFactoryCall = [setup_mock_buyer](absl::string_view hostname) { @@ -937,22 +958,24 @@ TEST_F(SelectAdReactorPASTest, /*add_protected_app_signals=*/true, /*app_install_signals=*/""); - auto mock_get_bids = [](std::unique_ptr - get_bids_raw_request, - const RequestMetadata& metadata, - GetBidDoneCallback on_done, absl::Duration timeout) { - // Expect PAS buyer inputs to not be be populated in GetBids. - EXPECT_FALSE(get_bids_raw_request->has_protected_app_signals_buyer_input()); - - // Ensure PA buyer inputs doesn't have the PAS data. - EXPECT_FALSE( - get_bids_raw_request->buyer_input().has_protected_app_signals()); - return absl::OkStatus(); - }; + auto mock_get_bids = + [](std::unique_ptr + get_bids_raw_request, + const RequestMetadata& metadata, GetBidDoneCallback on_done, + absl::Duration timeout, + RequestConfig request_config) { // Expect PAS buyer inputs to not be + // be populated in GetBids. + EXPECT_FALSE( + get_bids_raw_request->has_protected_app_signals_buyer_input()); + + // Ensure PA buyer inputs doesn't have the PAS data. + EXPECT_FALSE( + get_bids_raw_request->buyer_input().has_protected_app_signals()); + return absl::OkStatus(); + }; auto setup_mock_buyer = [&mock_get_bids](std::unique_ptr buyer) { - EXPECT_CALL(*buyer, ExecuteInternal(_, _, _, _)) - .WillRepeatedly(mock_get_bids); + EXPECT_CALL(*buyer, ExecuteInternal).WillRepeatedly(mock_get_bids); return buyer; }; auto MockBuyerFactoryCall = [setup_mock_buyer](absl::string_view hostname) { @@ -979,7 +1002,8 @@ TEST_F(SelectAdReactorPASTest, get_bids_raw_request, const RequestMetadata& metadata, GetBidDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, + RequestConfig request_config) { // Expect PAS buyer inputs to be populated correctly in GetBids. EXPECT_TRUE(get_bids_raw_request->has_protected_app_signals_buyer_input()); EXPECT_TRUE(get_bids_raw_request->protected_app_signals_buyer_input() @@ -1003,8 +1027,7 @@ TEST_F(SelectAdReactorPASTest, }; auto setup_mock_buyer = [&mock_get_bids](std::unique_ptr buyer) { - EXPECT_CALL(*buyer, ExecuteInternal(_, _, _, _)) - .WillRepeatedly(mock_get_bids); + EXPECT_CALL(*buyer, ExecuteInternal).WillRepeatedly(mock_get_bids); return buyer; }; auto MockBuyerFactoryCall = [setup_mock_buyer](absl::string_view hostname) { @@ -1024,20 +1047,19 @@ TEST_F(SelectAdReactorPASTest, auto request_with_context = CreateSelectAdRequest(kSellerOriginDomain); // Setup BFE to return a PA bid and no PAS bid. - auto mock_get_bids = [this](std::unique_ptr - get_bids_raw_request, - const RequestMetadata& metadata, - GetBidDoneCallback on_done, - absl::Duration timeout) { - auto response = std::make_unique(); - response->mutable_bids()->Add(GetTestPAAdWithBid()); - std::move(on_done)(std::move(response)); - return absl::OkStatus(); - }; + auto mock_get_bids = + [this](std::unique_ptr + get_bids_raw_request, + const RequestMetadata& metadata, GetBidDoneCallback on_done, + absl::Duration timeout, RequestConfig request_config) { + auto response = std::make_unique(); + response->mutable_bids()->Add(GetTestPAAdWithBid()); + std::move(on_done)(std::move(response), /* response_metadata= */ {}); + return absl::OkStatus(); + }; auto setup_mock_buyer = [&mock_get_bids](std::unique_ptr buyer) { - EXPECT_CALL(*buyer, ExecuteInternal(_, _, _, _)) - .WillRepeatedly(mock_get_bids); + EXPECT_CALL(*buyer, ExecuteInternal).WillRepeatedly(mock_get_bids); return buyer; }; auto MockBuyerFactoryCall = [setup_mock_buyer](absl::string_view hostname) { @@ -1065,7 +1087,8 @@ TEST_F(SelectAdReactorPASTest, get_bids_raw_request, const RequestMetadata& metadata, GetBidDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, + RequestConfig request_config) { // Expect no interest groups are present. EXPECT_EQ(get_bids_raw_request->buyer_input().interest_groups_size(), 0); @@ -1086,8 +1109,7 @@ TEST_F(SelectAdReactorPASTest, }; auto setup_mock_buyer = [&mock_get_bids](std::unique_ptr buyer) { - EXPECT_CALL(*buyer, ExecuteInternal(_, _, _, _)) - .WillRepeatedly(mock_get_bids); + EXPECT_CALL(*buyer, ExecuteInternal).WillRepeatedly(mock_get_bids); return buyer; }; auto MockBuyerFactoryCall = [setup_mock_buyer](absl::string_view hostname) { diff --git a/services/seller_frontend_service/select_ad_reactor_invalid_client.cc b/services/seller_frontend_service/select_ad_reactor_invalid.cc similarity index 51% rename from services/seller_frontend_service/select_ad_reactor_invalid_client.cc rename to services/seller_frontend_service/select_ad_reactor_invalid.cc index 66617734..2e236ef7 100644 --- a/services/seller_frontend_service/select_ad_reactor_invalid_client.cc +++ b/services/seller_frontend_service/select_ad_reactor_invalid.cc @@ -12,46 +12,61 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "services/seller_frontend_service/select_ad_reactor_invalid_client.h" +#include "services/seller_frontend_service/select_ad_reactor_invalid.h" #include namespace privacy_sandbox::bidding_auction_servers { -SelectAdReactorInvalidClient::SelectAdReactorInvalidClient( +SelectAdReactorInvalid::SelectAdReactorInvalid( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response, const ClientRegistry& clients, const TrustedServersConfigClient& config_client) : SelectAdReactor(context, request, response, clients, config_client), client_type_(request->client_type()) {} -void SelectAdReactorInvalidClient::Execute() { - Finish(grpc::Status(grpc::INVALID_ARGUMENT, - absl::StrCat(kUnsupportedClientType, " (", - ClientType_Name(client_type_), ")"))); +void SelectAdReactorInvalid::Execute() { + PS_VLOG(5) << "SelectAdRequest received:\n" + << request_->DebugString() + << ", request size: " << request_->ByteSizeLong(); + // This reactor is invoked in only following two cases: 1) When request is + // empty. 2) When the client type is not set appropriately. + if (request_->ByteSizeLong() == 0) { + ReportError(ErrorVisibility::AD_SERVER_VISIBLE, kEmptySelectAdRequest, + ErrorCode::CLIENT_SIDE); + } else { + ReportError(ErrorVisibility::AD_SERVER_VISIBLE, kUnsupportedClientType, + ErrorCode::CLIENT_SIDE); + } + LogIfError( + metric_context_->AccumulateMetric( + 1, metric::kSfeSelectAdRequestBadInput)); + PS_LOG(WARNING, log_context_) + << "Finishing the SelectAdRequest RPC with ad server visible error"; + + FinishWithStatus(grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, + error_accumulator_.GetAccumulatedErrorString( + ErrorVisibility::AD_SERVER_VISIBLE))); } -absl::StatusOr -SelectAdReactorInvalidClient::GetNonEncryptedResponse( +absl::StatusOr SelectAdReactorInvalid::GetNonEncryptedResponse( const std::optional& high_score, const std::optional& error) { return ""; } -ProtectedAudienceInput -SelectAdReactorInvalidClient::GetDecodedProtectedAudienceInput( +ProtectedAudienceInput SelectAdReactorInvalid::GetDecodedProtectedAudienceInput( absl::string_view encoded_data) { return {}; } -ProtectedAuctionInput -SelectAdReactorInvalidClient::GetDecodedProtectedAuctionInput( +ProtectedAuctionInput SelectAdReactorInvalid::GetDecodedProtectedAuctionInput( absl::string_view encoded_data) { return {}; } absl::flat_hash_map -SelectAdReactorInvalidClient::GetDecodedBuyerinputs( +SelectAdReactorInvalid::GetDecodedBuyerinputs( const google::protobuf::Map& encoded_buyer_inputs) { return {}; diff --git a/services/seller_frontend_service/select_ad_reactor_invalid_client.h b/services/seller_frontend_service/select_ad_reactor_invalid.h similarity index 84% rename from services/seller_frontend_service/select_ad_reactor_invalid_client.h rename to services/seller_frontend_service/select_ad_reactor_invalid.h index 894d513b..67acd19a 100644 --- a/services/seller_frontend_service/select_ad_reactor_invalid_client.h +++ b/services/seller_frontend_service/select_ad_reactor_invalid.h @@ -29,18 +29,17 @@ namespace privacy_sandbox::bidding_auction_servers { // A reactor to finish the client's RPC call with an error when an invalid // client type is specified in the request. -class SelectAdReactorInvalidClient : public SelectAdReactor { +class SelectAdReactorInvalid : public SelectAdReactor { public: - explicit SelectAdReactorInvalidClient( + explicit SelectAdReactorInvalid( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response, const ClientRegistry& clients, const TrustedServersConfigClient& config_client); - virtual ~SelectAdReactorInvalidClient() = default; + virtual ~SelectAdReactorInvalid() = default; - // SelectAdReactorInvalidClient is neither copyable nor movable. - SelectAdReactorInvalidClient(const SelectAdReactorInvalidClient&) = delete; - SelectAdReactorInvalidClient& operator=(const SelectAdReactorInvalidClient&) = - delete; + // SelectAdReactorInvalid is neither copyable nor movable. + SelectAdReactorInvalid(const SelectAdReactorInvalid&) = delete; + SelectAdReactorInvalid& operator=(const SelectAdReactorInvalid&) = delete; void Execute() override; diff --git a/services/seller_frontend_service/select_ad_reactor_test.cc b/services/seller_frontend_service/select_ad_reactor_test.cc index 52ca080d..1d290bdd 100644 --- a/services/seller_frontend_service/select_ad_reactor_test.cc +++ b/services/seller_frontend_service/select_ad_reactor_test.cc @@ -42,6 +42,7 @@ #include "google/protobuf/text_format.h" #include "gtest/gtest.h" #include "include/gmock/gmock.h" +#include "services/common/feature_flags.h" #include "services/common/metric/server_definition.h" #include "services/common/test/mocks.h" #include "services/common/test/random.h" @@ -68,15 +69,15 @@ using Response = SelectAdResponse; using BiddingGroupMap = ::google::protobuf::Map; -using GetBidDoneCallback = - absl::AnyInvocable>) &&>; +using GetBidDoneCallback = absl::AnyInvocable< + void(absl::StatusOr>, + ResponseMetadata) &&>; using ScoringSignalsDoneCallback = absl::AnyInvocable>) &&>; -using ScoreAdsDoneCallback = - absl::AnyInvocable>) &&>; +using ScoreAdsDoneCallback = absl::AnyInvocable< + void(absl::StatusOr>, + ResponseMetadata) &&>; using AdScore = ScoreAdsResponse::AdScore; using AdWithBidMetadata = @@ -98,6 +99,7 @@ class SellerFrontEndServiceTest : public ::testing::Test { config_.SetFlagForTest("", CONSENTED_DEBUG_TOKEN); config_.SetFlagForTest(kTrue, ENABLE_PROTECTED_APP_SIGNALS); config_.SetFlagForTest(kTrue, ENABLE_PROTECTED_AUDIENCE); + absl::SetFlag(&FLAGS_enable_chaffing, false); EXPECT_CALL(key_fetcher_manager_, GetPrivateKey) .Times(testing::AnyNumber()) .WillRepeatedly( @@ -186,7 +188,8 @@ TYPED_TEST(SellerFrontEndServiceTest, FetchesBidsFromAllBuyers) { std::unique_ptr get_bids_request, const RequestMetadata& metadata, - GetBidDoneCallback on_done, absl::Duration timeout) { + GetBidDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { google::protobuf::util::MessageDifferencer diff; std::string diff_output; diff.ReportDifferencesToString(&diff_output); @@ -223,6 +226,9 @@ TYPED_TEST(SellerFrontEndServiceTest, FetchesBidsFromAllBuyers) { return buyer; }; ErrorAccumulator error_accumulator; + std::vector>> + entries; for (const auto& buyer_ig_owner : this->request_.auction_config().buyer_list()) { auto decoded_buyer_input = DecodeBuyerInput( @@ -238,8 +244,14 @@ TYPED_TEST(SellerFrontEndServiceTest, FetchesBidsFromAllBuyers) { .WillOnce([SetupMockBuyer, buyer_input](absl::string_view hostname) { return SetupMockBuyer(buyer_input, hostname); }); + + entries.emplace_back(buyer_ig_owner, + std::make_shared()); } + EXPECT_CALL(buyer_clients, Entries) + .WillRepeatedly(Return(std::move(entries))); + // Reporting Client. std::unique_ptr async_reporter = std::make_unique( @@ -286,7 +298,8 @@ TYPED_TEST(SellerFrontEndServiceTest, std::unique_ptr get_bids_request, const RequestMetadata& metadata, - GetBidDoneCallback on_done, absl::Duration timeout) { + GetBidDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { google::protobuf::util::MessageDifferencer diff; std::string diff_output; diff.ReportDifferencesToString(&diff_output); @@ -326,6 +339,9 @@ TYPED_TEST(SellerFrontEndServiceTest, }; int num_buyers_solicited = 0; ErrorAccumulator error_accumulator; + std::vector>> + entries; for (const auto& buyer_ig_owner : this->request_.auction_config().buyer_list()) { auto decoded_buyer_input = DecodeBuyerInput( @@ -341,8 +357,13 @@ TYPED_TEST(SellerFrontEndServiceTest, ++num_buyers_solicited; return SetupMockBuyer(buyer_input, hostname); }); + entries.emplace_back(buyer_ig_owner, + std::make_shared()); } + EXPECT_CALL(buyer_clients, Entries) + .WillRepeatedly(Return(std::move(entries))); + // Reporting Client. std::unique_ptr async_reporter = std::make_unique( @@ -389,7 +410,8 @@ TYPED_TEST(SellerFrontEndServiceTest, FetchesTwoBidsGivenThreeBuyers) { std::unique_ptr get_values_request, const RequestMetadata& metadata, - GetBidDoneCallback on_done, absl::Duration timeout) { + GetBidDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { google::protobuf::util::MessageDifferencer diff; std::string diff_output; diff.ReportDifferencesToString(&diff_output); @@ -410,6 +432,9 @@ TYPED_TEST(SellerFrontEndServiceTest, FetchesTwoBidsGivenThreeBuyers) { }; int num_buyers_solicited = 0; ErrorAccumulator error_accumulator; + std::vector>> + entries; for (const auto& buyer_ig_owner : this->request_.auction_config().buyer_list()) { auto decoded_buyer_input = DecodeBuyerInput( @@ -425,8 +450,13 @@ TYPED_TEST(SellerFrontEndServiceTest, FetchesTwoBidsGivenThreeBuyers) { ++num_buyers_solicited; return SetupMockBuyer(buyer_input); }); + entries.emplace_back(buyer_ig_owner, + std::make_shared()); } + EXPECT_CALL(buyer_clients, Entries) + .WillRepeatedly(Return(std::move(entries))); + // Reporting Client. std::unique_ptr async_reporter = std::make_unique( @@ -474,7 +504,8 @@ TYPED_TEST(SellerFrontEndServiceTest, std::unique_ptr get_values_request, const RequestMetadata& metadata, - GetBidDoneCallback on_done, absl::Duration timeout) { + GetBidDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { google::protobuf::util::MessageDifferencer diff; std::string diff_output; diff.ReportDifferencesToString(&diff_output); @@ -488,6 +519,9 @@ TYPED_TEST(SellerFrontEndServiceTest, }; ErrorAccumulator error_accumulator; + std::vector>> + entries; for (const auto& buyer_ig_owner : this->request_.auction_config().buyer_list()) { auto decoded_buyer_input = DecodeBuyerInput( @@ -500,8 +534,13 @@ TYPED_TEST(SellerFrontEndServiceTest, .WillOnce([SetupMockBuyer, buyer_input](absl::string_view hostname) { return SetupMockBuyer(buyer_input); }); + entries.emplace_back(buyer_ig_owner, + std::make_shared()); } + EXPECT_CALL(buyer_clients, Entries) + .WillRepeatedly(Return(std::move(entries))); + // Reporting Client. std::unique_ptr async_reporter = std::make_unique( @@ -521,13 +560,15 @@ TYPED_TEST(SellerFrontEndServiceTest, /** * This test also includes a specified buyer_currency to demonstrate that - * setting a buyer currency but having no currency on each bid affects nothing + * setting a buyer currency but having no currency on each bid affects + nothing * adversely. */ TYPED_TEST(SellerFrontEndServiceTest, FetchesScoringSignalsWithBidResponseAdRenderUrls) { this->SetupRequest(/*num_buyers=*/2, /*set_buyer_egid=*/false, - /*set_seller_egid=*/true, /*seller_currency=*/kUsdIsoCode); + /*set_seller_egid=*/true, + /*seller_currency=*/kUsdIsoCode); // Scoring Client ScoringAsyncClientMock scoring_client; // Expects no calls because we do not finish fetching the decision logic @@ -552,6 +593,9 @@ TYPED_TEST(SellerFrontEndServiceTest, get_bids_response)); } + MockEntriesCallOnBuyerFactory(this->protected_auction_input_.buyer_input(), + buyer_clients); + // Scoring Signals Provider MockAsyncProvider scoring_signals_provider; @@ -614,6 +658,9 @@ TYPED_TEST(SellerFrontEndServiceTest, ScoresAdsAfterGettingSignals) { buyer, std::make_unique(response)); } + MockEntriesCallOnBuyerFactory(this->protected_auction_input_.buyer_input(), + buyer_clients); + // Scoring signals provider // Scoring signals must be nonzero for scoring to be attempted. std::string scoring_signals_value = @@ -631,7 +678,8 @@ TYPED_TEST(SellerFrontEndServiceTest, ScoresAdsAfterGettingSignals) { std::unique_ptr score_ads_raw_request, const RequestMetadata& metadata, - ScoreAdsDoneCallback on_done, absl::Duration timeout) { + ScoreAdsDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { google::protobuf::util::MessageDifferencer diff; std::string diff_output; diff.ReportDifferencesToString(&diff_output); @@ -696,6 +744,9 @@ TYPED_TEST(SellerFrontEndServiceTest, DoesNotScoreAdsAfterGettingEmptySignals) { buyer, std::make_unique(response)); } + MockEntriesCallOnBuyerFactory(this->protected_auction_input_.buyer_input(), + buyer_clients); + // Scoring signals provider std::string scoring_signals_value = ""; MockAsyncProvider @@ -752,6 +803,9 @@ TYPED_TEST(SellerFrontEndServiceTest, ReturnsWinningAdAfterScoring) { get_bid_response)); } + MockEntriesCallOnBuyerFactory(this->protected_auction_input_.buyer_input(), + buyer_clients); + // Scoring signal provider MockAsyncProvider scoring_signals_provider; @@ -771,7 +825,8 @@ TYPED_TEST(SellerFrontEndServiceTest, ReturnsWinningAdAfterScoring) { std::unique_ptr score_ads_request, const RequestMetadata& metadata, - ScoreAdsDoneCallback on_done, absl::Duration timeout) { + ScoreAdsDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { ScoreAdsResponse::ScoreAdsRawResponse response; float i = 1; ErrorAccumulator error_accumulator; @@ -812,7 +867,8 @@ TYPED_TEST(SellerFrontEndServiceTest, ReturnsWinningAdAfterScoring) { winner = score; } std::move(on_done)( - std::make_unique(response)); + std::make_unique(response), + {}); scoring_done.DecrementCount(); return absl::OkStatus(); }); @@ -1025,6 +1081,9 @@ TYPED_TEST(SellerFrontEndServiceTest, expected_filtered_buyer_bids_map, scoring_signals_value); + MockEntriesCallOnBuyerFactory(this->protected_auction_input_.buyer_input(), + buyer_clients); + // Scoring Client ScoringAsyncClientMock scoring_client; absl::BlockingCounter scoring_done(1); @@ -1035,7 +1094,8 @@ TYPED_TEST(SellerFrontEndServiceTest, std::unique_ptr score_ads_request, const RequestMetadata& metadata, - ScoreAdsDoneCallback on_done, absl::Duration timeout) { + ScoreAdsDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { ScoreAdsResponse::ScoreAdsRawResponse response; float i = 1; ErrorAccumulator error_accumulator; @@ -1080,7 +1140,8 @@ TYPED_TEST(SellerFrontEndServiceTest, winner = score; } std::move(on_done)( - std::make_unique(response)); + std::make_unique(response), + {}); scoring_done.DecrementCount(); return absl::OkStatus(); }); @@ -1198,10 +1259,11 @@ TYPED_TEST(SellerFrontEndServiceTest, ReturnsBiddingGroups) { EXPECT_CALL(scoring_client, ExecuteInternal) .WillOnce([](std::unique_ptr request, const RequestMetadata& metadata, - ScoreAdsDoneCallback on_done, absl::Duration timeout) { + ScoreAdsDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { EXPECT_EQ(request->ad_bids_size(), 3); - // We can return only one score as a winner, so we arbitrarily choose - // the first. + // We can return only one score as a winner, so we arbitrarily + // choose the first. const auto& winning_ad_with_bid = request->ad_bids(0); auto response = std::make_unique(); @@ -1216,7 +1278,7 @@ TYPED_TEST(SellerFrontEndServiceTest, ReturnsBiddingGroups) { score->set_buyer_bid(1); score->set_interest_group_name( winning_ad_with_bid.interest_group_name()); - std::move(on_done)(std::move(response)); + std::move(on_done)(std::move(response), /* response_metadata= */ {}); return absl::OkStatus(); }); @@ -1225,6 +1287,9 @@ TYPED_TEST(SellerFrontEndServiceTest, ReturnsBiddingGroups) { std::make_unique( std::make_unique()); + MockEntriesCallOnBuyerFactory(this->protected_auction_input_.buyer_input(), + buyer_clients); + // Client Registry ClientRegistry clients{ scoring_signals_provider, scoring_client, buyer_clients, @@ -1301,7 +1366,7 @@ TYPED_TEST(SellerFrontEndServiceTest, PerformsDebugReportingAfterScoring) { [decision_logic, &scoring_done, &winner]( std::unique_ptr request, const RequestMetadata& metadata, ScoreAdsDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { ScoreAdsResponse::ScoreAdsRawResponse response; float i = 1; // Last bid wins. @@ -1322,7 +1387,8 @@ TYPED_TEST(SellerFrontEndServiceTest, PerformsDebugReportingAfterScoring) { } std::move(on_done)( std::make_unique( - response)); + response), + {}); scoring_done.Notify(); return absl::OkStatus(); }); @@ -1342,6 +1408,9 @@ TYPED_TEST(SellerFrontEndServiceTest, PerformsDebugReportingAfterScoring) { reporting_count.DecrementCount(); }); + MockEntriesCallOnBuyerFactory(this->protected_auction_input_.buyer_input(), + buyer_clients); + // Client Registry ClientRegistry clients{ scoring_signals_provider, scoring_client, buyer_clients, @@ -1394,9 +1463,10 @@ TYPED_TEST(SellerFrontEndServiceTest, [&scoring_done]( std::unique_ptr request, const RequestMetadata& metadata, ScoreAdsDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { std::move(on_done)( - absl::Status(absl::StatusCode::kInternal, "No Ads found")); + absl::Status(absl::StatusCode::kInternal, "No Ads found"), + /* response_metadata= */ {}); scoring_done.Notify(); return absl::Status(absl::StatusCode::kInternal, "No Ads found"); }); @@ -1405,6 +1475,10 @@ TYPED_TEST(SellerFrontEndServiceTest, std::make_unique( std::make_unique()); EXPECT_CALL(*async_reporter, DoReport).Times(0); + + MockEntriesCallOnBuyerFactory(this->protected_auction_input_.buyer_input(), + buyer_clients); + // Client Registry ClientRegistry clients{ scoring_signals_provider, scoring_client, buyer_clients, @@ -1461,10 +1535,12 @@ TYPED_TEST(SellerFrontEndServiceTest, std::unique_ptr score_ads_request, const RequestMetadata& metadata, - ScoreAdsDoneCallback on_done, absl::Duration timeout) { + ScoreAdsDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { EXPECT_EQ(top_level_seller, score_ads_request->top_level_seller()); std::move(on_done)( - std::make_unique()); + std::make_unique(), + /* response_metadata= */ {}); scoring_done.Notify(); return absl::OkStatus(); }); @@ -1474,6 +1550,9 @@ TYPED_TEST(SellerFrontEndServiceTest, std::make_unique( std::make_unique()); + MockEntriesCallOnBuyerFactory(this->protected_auction_input_.buyer_input(), + buyer_clients); + // Client Registry ClientRegistry clients{ scoring_signals_provider, scoring_client, buyer_clients, @@ -1530,10 +1609,12 @@ TYPED_TEST(SellerFrontEndServiceTest, PassesFieldsForServerComponentAuction) { std::unique_ptr score_ads_request, const RequestMetadata& metadata, - ScoreAdsDoneCallback on_done, absl::Duration timeout) { + ScoreAdsDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { EXPECT_EQ(top_level_seller, score_ads_request->top_level_seller()); std::move(on_done)( - std::make_unique()); + std::make_unique(), + /* response_metadata= */ {}); scoring_done.Notify(); return absl::OkStatus(); }); @@ -1548,6 +1629,8 @@ TYPED_TEST(SellerFrontEndServiceTest, PassesFieldsForServerComponentAuction) { EXPECT_CALL(this->key_fetcher_manager_, GetPublicKey).WillOnce(Return(key)); MockCryptoClientWrapper crypto_client; SetupMockCrytoClient(crypto_client); + MockEntriesCallOnBuyerFactory(this->protected_auction_input_.buyer_input(), + buyer_clients); // Client Registry ClientRegistry clients{ @@ -1594,12 +1677,13 @@ TYPED_TEST(SellerFrontEndServiceTest, VerifyUnlimitedEgressFlagPropagates) { auto mock_get_bids = [](std::unique_ptr get_values_request, const RequestMetadata& metadata, GetBidDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto get_bids_response = std::make_unique(); auto* bid = get_bids_response->mutable_bids()->Add(); bid->set_bid(10.0); - std::move(on_done)(std::move(get_bids_response)); + std::move(on_done)(std::move(get_bids_response), + /* response_metadata= */ {}); return absl::OkStatus(); }; auto setup_mock_buyer = @@ -1608,7 +1692,7 @@ TYPED_TEST(SellerFrontEndServiceTest, VerifyUnlimitedEgressFlagPropagates) { EXPECT_CALL(*buyer, ExecuteInternal(Pointee(EqGetBidsRawRequestUnlimitedEgress( expected_get_bid_request)), - _, _, _)) + _, _, _, _)) .WillRepeatedly(mock_get_bids); return buyer; }; @@ -1618,6 +1702,9 @@ TYPED_TEST(SellerFrontEndServiceTest, VerifyUnlimitedEgressFlagPropagates) { EXPECT_CALL(buyer_front_end_async_client_factory_mock, Get) .WillRepeatedly(mock_buyer_factory); + MockEntriesCallOnBuyerFactory(this->protected_auction_input_.buyer_input(), + buyer_front_end_async_client_factory_mock); + // Set consented debug config that should be propagated to the downstream // services. auto [protected_auction_input, request, context] = @@ -1637,5 +1724,102 @@ TYPED_TEST(SellerFrontEndServiceTest, VerifyUnlimitedEgressFlagPropagates) { RunReactorRequest(this->config_, clients, request); } +TYPED_TEST(SellerFrontEndServiceTest, ChaffingEnabled_SendsChaffRequest) { + absl::SetFlag(&FLAGS_enable_chaffing, true); + + MockAsyncProvider + scoring_signals_provider; + ScoringAsyncClientMock scoring_client; + BuyerFrontEndAsyncClientFactoryMock buyer_front_end_async_client_factory_mock; + BuyerBidsResponseMap expected_buyer_bids; + std::unique_ptr key_fetcher_manager = + std::make_unique(); + EXPECT_CALL(*key_fetcher_manager, GetPrivateKey) + .WillRepeatedly(Return(GetPrivateKey())); + const float winning_bid = 999.0F; + auto [request_with_context, clients] = + GetSelectAdRequestAndClientRegistryForTest( + CLIENT_TYPE_BROWSER, winning_bid, scoring_signals_provider, + scoring_client, buyer_front_end_async_client_factory_mock, + key_fetcher_manager.get(), expected_buyer_bids, kSellerOriginDomain, + false); + + std::string chaff_buyer_name = "chaff_buyer"; + GetBidsResponse::GetBidsRawResponse response; + + // Set up buyer calls for chaff buyer. + auto MockGetBids = + [response]( + std::unique_ptr get_bids_request, + const RequestMetadata& metadata, GetBidDoneCallback on_done, + absl::Duration timeout, RequestConfig request_config) { + // Verify this is a chaff request by validating the is_chaff and + // generation_id fields. + EXPECT_TRUE(get_bids_request->is_chaff()); + EXPECT_FALSE(get_bids_request->log_context().generation_id().empty()); + ABSL_LOG(INFO) << "Returning mock bids"; + std::move(on_done)( + std::make_unique(response), + /* response_metadata= */ {}); + return absl::OkStatus(); + }; + auto SetupMockBuyer = + [MockGetBids](std::unique_ptr buyer) { + EXPECT_CALL(*buyer, ExecuteInternal) + .Times(testing::Exactly(1)) + .WillRepeatedly(MockGetBids); + return buyer; + }; + + auto MockBuyerFactoryCall = + [SetupMockBuyer](absl::string_view chaff_buyer_name) { + return SetupMockBuyer(std::make_unique()); + }; + EXPECT_CALL(buyer_front_end_async_client_factory_mock, Get(chaff_buyer_name)) + .Times(testing::Exactly(1)) + .WillRepeatedly(MockBuyerFactoryCall); + + // Have the Entries() call on the buyer factory return the 'real' and chaff + // buyer. + std::vector>> + entries; + for (const auto& [buyer, unused] : + request_with_context.protected_auction_input.buyer_input()) { + entries.emplace_back(buyer, + std::make_shared()); + } + entries.emplace_back(chaff_buyer_name, + std::make_shared()); + EXPECT_CALL(buyer_front_end_async_client_factory_mock, Entries) + .WillRepeatedly(Return(std::move(entries))); + + // Append the chaff buyer to the auction_config field in the SelectAd + // request. + SelectAdRequest select_ad_request = + std::move(request_with_context.select_ad_request); + *select_ad_request.mutable_auction_config()->mutable_buyer_list()->Add() = + chaff_buyer_name; + SelectAdResponse encrypted_response = + RunReactorRequest(this->config_, clients, + select_ad_request); + EXPECT_FALSE(encrypted_response.auction_result_ciphertext().empty()); + auto decrypted_response = FromObliviousHTTPResponse( + *encrypted_response.mutable_auction_result_ciphertext(), + request_with_context.context, kBiddingAuctionOhttpResponseLabel); + ASSERT_TRUE(decrypted_response.ok()) << decrypted_response.status(); + + absl::StatusOr decompressed_response = + UnframeAndDecompressAuctionResult(*decrypted_response); + ASSERT_TRUE(decompressed_response.ok()) + << decompressed_response.status().message(); + + absl::StatusOr auction_result = + CborDecodeAuctionResultToProto(*decompressed_response); + EXPECT_TRUE(auction_result.ok()) << auction_result.status(); + EXPECT_FALSE(auction_result->is_chaff()); + EXPECT_EQ(auction_result->bid(), winning_bid); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/select_ad_reactor_web.cc b/services/seller_frontend_service/select_ad_reactor_web.cc index 78aea27c..dd45b50a 100644 --- a/services/seller_frontend_service/select_ad_reactor_web.cc +++ b/services/seller_frontend_service/select_ad_reactor_web.cc @@ -73,9 +73,14 @@ absl::StatusOr SelectAdReactorForWeb::GetNonEncryptedResponse( auto error_handler = absl::bind_front(&SelectAdReactorForWeb::FinishWithStatus, this); std::string encoded_data; - const auto decode_lambda = [&encoded_data]() { + const auto decode_lambda = [&encoded_data, this]() { auto result = CborDecodeAuctionResultToProto(encoded_data); - return result.ok() ? result->DebugString() : result.status().ToString(); + if (result.ok()) { + log_context_.SetEventMessageField(*result); + return result->DebugString(); + } else { + return result.status().ToString(); + } }; if (auction_scope_ == @@ -107,6 +112,7 @@ absl::StatusOr SelectAdReactorForWeb::GetNonEncryptedResponse( PS_VLOG(kPlain, log_context_) << "AuctionResult:\n" << auction_result.ShortDebugString(); + log_context_.SetEventMessageField(auction_result); } else { // SINGLE_SELLER or SERVER_TOP_LEVEL Auction PS_ASSIGN_OR_RETURN( diff --git a/services/seller_frontend_service/select_ad_reactor_web_test.cc b/services/seller_frontend_service/select_ad_reactor_web_test.cc index 564eba03..100b14cd 100644 --- a/services/seller_frontend_service/select_ad_reactor_web_test.cc +++ b/services/seller_frontend_service/select_ad_reactor_web_test.cc @@ -30,6 +30,7 @@ #include "absl/strings/str_format.h" #include "gtest/gtest.h" #include "quiche/oblivious_http/oblivious_http_client.h" +#include "services/common/feature_flags.h" #include "services/common/metric/server_definition.h" #include "services/common/test/mocks.h" #include "services/common/test/utils/cbor_test_utils.h" @@ -51,9 +52,9 @@ using ::testing::Pointee; using ::testing::Property; using ::testing::Return; using Context = ::quiche::ObliviousHttpRequest::Context; -using GetBidDoneCallback = - absl::AnyInvocable>) &&>; +using GetBidDoneCallback = absl::AnyInvocable< + void(absl::StatusOr>, + ResponseMetadata) &&>; using ScoringSignalsDoneCallback = absl::AnyInvocable>, GetByteSize) &&>; @@ -70,6 +71,7 @@ class SelectAdReactorForWebTest : public ::testing::Test { config_.SetFlagForTest("", CONSENTED_DEBUG_TOKEN); config_.SetFlagForTest(kFalse, ENABLE_PROTECTED_APP_SIGNALS); config_.SetFlagForTest(kTrue, ENABLE_PROTECTED_AUDIENCE); + absl::SetFlag(&FLAGS_enable_chaffing, false); } void SetProtectedAuctionCipherText(const T& protected_auction_input, @@ -116,6 +118,10 @@ TYPED_TEST(SelectAdReactorForWebTest, VerifyCborEncoding) { scoring_client, buyer_front_end_async_client_factory_mock, key_fetcher_manager.get(), expected_buyer_bids, kSellerOriginDomain); + MockEntriesCallOnBuyerFactory( + request_with_context.protected_auction_input.buyer_input(), + buyer_front_end_async_client_factory_mock); + SelectAdResponse response_with_cbor = RunReactorRequest( this->config_, clients, request_with_context.select_ad_request); @@ -184,6 +190,10 @@ TYPED_TEST(SelectAdReactorForWebTest, VerifyChaffedResponse) { scoring_client, buyer_front_end_async_client_factory_mock, key_fetcher_manager.get(), expected_buyer_bids, kSellerOriginDomain); + MockEntriesCallOnBuyerFactory( + request_with_context.protected_auction_input.buyer_input(), + buyer_front_end_async_client_factory_mock); + SelectAdResponse response_with_cbor = RunReactorRequest( this->config_, clients, request_with_context.select_ad_request); @@ -265,13 +275,14 @@ TYPED_TEST(SelectAdReactorForWebTest, VerifyLogContextPropagates) { auto MockGetBids = [](std::unique_ptr get_values_request, const RequestMetadata& metadata, GetBidDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto get_bids_response = std::make_unique(); auto* bid = get_bids_response->mutable_bids()->Add(); bid->set_bid(kSampleBidValue); bid->set_interest_group_name(kSampleBuyer); - std::move(on_done)(std::move(get_bids_response)); + std::move(on_done)(std::move(get_bids_response), + /* response_metadata= */ {}); return absl::OkStatus(); }; auto SetupMockBuyer = @@ -280,7 +291,7 @@ TYPED_TEST(SelectAdReactorForWebTest, VerifyLogContextPropagates) { EXPECT_CALL(*buyer, ExecuteInternal(Pointee(EqGetBidsRawRequestWithLogContext( expected_get_bid_request)), - _, _, _)) + _, _, _, _)) .WillRepeatedly(MockGetBids); return buyer; }; @@ -314,7 +325,7 @@ TYPED_TEST(SelectAdReactorForWebTest, VerifyLogContextPropagates) { EXPECT_CALL(scoring_client, ExecuteInternal(Pointee(EqScoreAdsRawRequestWithLogContext( expected_score_ads_request)), - _, _, _)); + _, _, _, _)); // Set log context that should be propagated to the downstream services. auto [protected_auction_input, request, context] = @@ -326,6 +337,9 @@ TYPED_TEST(SelectAdReactorForWebTest, VerifyLogContextPropagates) { buyer_config.set_buyer_debug_id(kSampleBuyerDebugId); buyer_config.set_buyer_signals(kSampleBuyerSignals); + MockEntriesCallOnBuyerFactory(protected_auction_input.buyer_input(), + buyer_front_end_async_client_factory_mock); + SelectAdResponse response_with_cbor = RunReactorRequest(this->config_, clients, request); } @@ -605,13 +619,14 @@ TYPED_TEST(SelectAdReactorForWebTest, VerifyConsentedDebugConfigPropagates) { auto MockGetBids = [](std::unique_ptr get_values_request, const RequestMetadata& metadata, GetBidDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto get_bids_response = std::make_unique(); auto* bid = get_bids_response->mutable_bids()->Add(); bid->set_bid(kSampleBidValue); bid->set_interest_group_name(kSampleBuyer); - std::move(on_done)(std::move(get_bids_response)); + std::move(on_done)(std::move(get_bids_response), + /* response_metadata= */ {}); return absl::OkStatus(); }; auto SetupMockBuyer = @@ -621,7 +636,7 @@ TYPED_TEST(SelectAdReactorForWebTest, VerifyConsentedDebugConfigPropagates) { *buyer, ExecuteInternal(Pointee(EqGetBidsRawRequestWithConsentedDebugConfig( expected_get_bid_request)), - _, _, _)) + _, _, _, _)) .WillRepeatedly(MockGetBids); return buyer; }; @@ -658,7 +673,7 @@ TYPED_TEST(SelectAdReactorForWebTest, VerifyConsentedDebugConfigPropagates) { scoring_client, ExecuteInternal(Pointee(EqScoreAdsRawRequestWithConsentedDebugConfig( expected_score_ads_request)), - _, _, _)); + _, _, _, _)); // Set consented debug config that should be propagated to the downstream // services. @@ -673,6 +688,9 @@ TYPED_TEST(SelectAdReactorForWebTest, VerifyConsentedDebugConfigPropagates) { buyer_config.set_buyer_debug_id(kSampleBuyerDebugId); buyer_config.set_buyer_signals(kSampleBuyerSignals); + MockEntriesCallOnBuyerFactory(protected_auction_input.buyer_input(), + buyer_front_end_async_client_factory_mock); + SelectAdResponse response_with_cbor = RunReactorRequest(this->config_, clients, request); } @@ -696,6 +714,10 @@ TYPED_TEST(SelectAdReactorForWebTest, /*expect_all_buyers_solicited=*/true, kTestTopLevelSellerOriginDomain); + MockEntriesCallOnBuyerFactory( + request_with_context.protected_auction_input.buyer_input(), + buyer_front_end_async_client_factory_mock); + SelectAdResponse response_with_cbor = RunReactorRequest( this->config_, clients, request_with_context.select_ad_request); @@ -721,16 +743,18 @@ TYPED_TEST(SelectAdReactorForWebTest, std::string base64_response; absl::Base64Escape(*decompressed_response, &base64_response); - ABSL_LOG(INFO) << "Decrypted, decompressed but CBOR encoded auction result:\n" - << base64_response; + ABSL_LOG(INFO) + << "Decrypted, decompressed but CBOR encoded auction result :\n " + << base64_response; absl::StatusOr deserialized_auction_result = CborDecodeAuctionResultToProto(*decompressed_response); EXPECT_TRUE(deserialized_auction_result.ok()); EXPECT_FALSE(deserialized_auction_result->is_chaff()); - ABSL_LOG(INFO) << "Decrypted, decompressed and CBOR decoded auction result:\n" - << MessageToJson(*deserialized_auction_result); + ABSL_LOG(INFO) + << "Decrypted, decompressed and CBOR decoded auction result :\n " + << MessageToJson(*deserialized_auction_result); // Validate that the bidding groups data is present. EXPECT_EQ(deserialized_auction_result->bidding_groups().size(), 1); @@ -770,6 +794,10 @@ TYPED_TEST(SelectAdReactorForWebTest, FailsEncodingWhenModifiedBidIsZero) { /*enable_reporting=*/false, /*force_set_modified_bid_to_zero=*/true); + MockEntriesCallOnBuyerFactory( + request_with_context.protected_auction_input.buyer_input(), + buyer_front_end_async_client_factory_mock); + SelectAdResponse response_with_cbor = RunReactorRequest( this->config_, clients, request_with_context.select_ad_request); @@ -782,6 +810,7 @@ TYPED_TEST(SelectAdReactorForWebTest, FailsEncodingWhenModifiedBidIsZero) { request_with_context.context, kBiddingAuctionOhttpResponseLabel); EXPECT_FALSE(decrypted_response.ok()) << decrypted_response.status(); } + TYPED_TEST(SelectAdReactorForWebTest, VerifyServerComponentAuctionProtoEncoding) { MockAsyncProvider @@ -809,6 +838,10 @@ TYPED_TEST(SelectAdReactorForWebTest, {&crypto_client, EncryptionCloudPlatform::ENCRYPTION_CLOUD_PLATFORM_GCP}); + MockEntriesCallOnBuyerFactory( + request_with_context.protected_auction_input.buyer_input(), + buyer_front_end_async_client_factory_mock); + SelectAdResponse response_with_proto = RunReactorRequest( this->config_, clients, request_with_context.select_ad_request); diff --git a/services/seller_frontend_service/select_auction_result_reactor.cc b/services/seller_frontend_service/select_auction_result_reactor.cc index 99f90cab..bdad37b9 100644 --- a/services/seller_frontend_service/select_auction_result_reactor.cc +++ b/services/seller_frontend_service/select_auction_result_reactor.cc @@ -80,7 +80,8 @@ void SelectAuctionResultReactor::ScoreAds( auto on_scoring_done = [this, auction_request_metric = std::move(auction_request_metric)]( absl::StatusOr> - result) mutable { + result, + ResponseMetadata response_metadata) mutable { { int response_size = result.ok() ? (int)result->get()->ByteSizeLong() : 0; @@ -90,6 +91,7 @@ void SelectAuctionResultReactor::ScoreAds( } OnScoreAdsDone(std::move(result)); }; + absl::Status execute_result = clients_.scoring.ExecuteInternal( std::move(raw_request), {}, std::move(on_scoring_done), absl::Milliseconds( diff --git a/services/seller_frontend_service/select_auction_result_reactor.h b/services/seller_frontend_service/select_auction_result_reactor.h index fdf1400c..ead8de5c 100644 --- a/services/seller_frontend_service/select_auction_result_reactor.h +++ b/services/seller_frontend_service/select_auction_result_reactor.h @@ -80,7 +80,7 @@ class SelectAuctionResultReactor : public grpc::ServerUnaryReactor { const ClientRegistry& clients_; const TrustedServersConfigClient& config_client_; - server_common::log::ContextImpl log_context_; + RequestLogContext log_context_; // Used to log metric, same life time as reactor. std::unique_ptr metric_context_; diff --git a/services/seller_frontend_service/select_auction_result_reactor_test.cc b/services/seller_frontend_service/select_auction_result_reactor_test.cc index 7663a491..a8aeffc0 100644 --- a/services/seller_frontend_service/select_auction_result_reactor_test.cc +++ b/services/seller_frontend_service/select_auction_result_reactor_test.cc @@ -118,37 +118,40 @@ TYPED_TEST(SelectAuctionResultReactorTest, CallsScoringWithComponentAuctions) { this->SetupComponentAuctionResults(2); EXPECT_CALL(this->scoring_client_, ExecuteInternal) .Times(1) - .WillOnce([this, &scoring_done]( - std::unique_ptr - score_ads_request, - const RequestMetadata& metadata, - absl::AnyInvocable>)&&> - on_done, - absl::Duration timeout) { - EXPECT_EQ(score_ads_request->auction_signals(), - this->request_.auction_config().auction_signals()); - EXPECT_EQ(score_ads_request->seller_signals(), - this->request_.auction_config().seller_signals()); - EXPECT_EQ(score_ads_request->seller(), - this->request_.auction_config().seller()); - EXPECT_EQ(score_ads_request->seller_currency(), - this->request_.auction_config().seller_currency()); - EXPECT_EQ(score_ads_request->component_auction_results_size(), - this->request_.component_auction_results_size()); - for (int i = 0; i < score_ads_request->component_auction_results_size(); - i++) { - // bidding groups are not sent to Auction server. - this->component_auction_results_[i].clear_bidding_groups(); - EXPECT_THAT(score_ads_request->component_auction_results(i), - EqualsProto(this->component_auction_results_[i])); - } - std::move(on_done)( - std::make_unique()); - scoring_done.Notify(); - return absl::OkStatus(); - }); + .WillOnce( + [this, &scoring_done]( + std::unique_ptr + score_ads_request, + const RequestMetadata& metadata, + absl::AnyInvocable>, + ResponseMetadata)&&> + on_done, + absl::Duration timeout, RequestConfig request_config) { + EXPECT_EQ(score_ads_request->auction_signals(), + this->request_.auction_config().auction_signals()); + EXPECT_EQ(score_ads_request->seller_signals(), + this->request_.auction_config().seller_signals()); + EXPECT_EQ(score_ads_request->seller(), + this->request_.auction_config().seller()); + EXPECT_EQ(score_ads_request->seller_currency(), + this->request_.auction_config().seller_currency()); + EXPECT_EQ(score_ads_request->component_auction_results_size(), + this->request_.component_auction_results_size()); + for (int i = 0; + i < score_ads_request->component_auction_results_size(); i++) { + // bidding groups are not sent to Auction server. + this->component_auction_results_[i].clear_bidding_groups(); + EXPECT_THAT(score_ads_request->component_auction_results(i), + EqualsProto(this->component_auction_results_[i])); + } + std::move(on_done)( + std::make_unique(), + /* response_metadata= */ {}); + scoring_done.Notify(); + return absl::OkStatus(); + }); ClientRegistry clients = { MockAsyncProvider(), this->scoring_client_, @@ -211,22 +214,25 @@ TYPED_TEST(SelectAuctionResultReactorTest, /*rejection_reason_ig_per_owner = */ 2); EXPECT_CALL(this->scoring_client_, ExecuteInternal) .Times(1) - .WillOnce([&scoring_done, &winner]( - std::unique_ptr - score_ads_request, - const RequestMetadata& metadata, - absl::AnyInvocable>)&&> - on_done, - absl::Duration timeout) { - auto response = - std::make_unique(); - response->mutable_ad_score()->MergeFrom(winner); - std::move(on_done)(std::move(response)); - scoring_done.Notify(); - return absl::OkStatus(); - }); + .WillOnce( + [&scoring_done, &winner]( + std::unique_ptr + score_ads_request, + const RequestMetadata& metadata, + absl::AnyInvocable>, + ResponseMetadata)&&> + on_done, + absl::Duration timeout, RequestConfig request_config) { + auto response = + std::make_unique(); + response->mutable_ad_score()->MergeFrom(winner); + std::move(on_done)(std::move(response), + /* response_metadata= */ {}); + scoring_done.Notify(); + return absl::OkStatus(); + }); ClientRegistry clients = { MockAsyncProvider(), this->scoring_client_, @@ -249,14 +255,16 @@ TYPED_TEST(SelectAuctionResultReactorTest, EXPECT_CALL(this->scoring_client_, ExecuteInternal) .Times(1) .WillOnce( - [&scoring_done](std::unique_ptr - score_ads_request, - const RequestMetadata& metadata, - absl::AnyInvocable>)&&> - on_done, - absl::Duration timeout) { + [&scoring_done]( + std::unique_ptr + score_ads_request, + const RequestMetadata& metadata, + absl::AnyInvocable>, + ResponseMetadata)&&> + on_done, + absl::Duration timeout, RequestConfig request_config) { scoring_done.Notify(); return absl::Status(absl::StatusCode::kInternal, "test msg"); }); @@ -302,16 +310,19 @@ TYPED_TEST(SelectAuctionResultReactorTest, EXPECT_CALL(this->scoring_client_, ExecuteInternal) .Times(1) .WillOnce( - [&scoring_done](std::unique_ptr - score_ads_request, - const RequestMetadata& metadata, - absl::AnyInvocable>)&&> - on_done, - absl::Duration timeout) { + [&scoring_done]( + std::unique_ptr + score_ads_request, + const RequestMetadata& metadata, + absl::AnyInvocable>, + ResponseMetadata)&&> + on_done, + absl::Duration timeout, RequestConfig request_config) { std::move(on_done)( - absl::Status(absl::StatusCode::kInvalidArgument, "test msg")); + absl::Status(absl::StatusCode::kInvalidArgument, "test msg"), + {}); scoring_done.Notify(); return absl::OkStatus(); }); @@ -335,16 +346,19 @@ TYPED_TEST(SelectAuctionResultReactorTest, EXPECT_CALL(this->scoring_client_, ExecuteInternal) .Times(1) .WillOnce( - [&scoring_done](std::unique_ptr - score_ads_request, - const RequestMetadata& metadata, - absl::AnyInvocable>)&&> - on_done, - absl::Duration timeout) { + [&scoring_done]( + std::unique_ptr + score_ads_request, + const RequestMetadata& metadata, + absl::AnyInvocable>, + ResponseMetadata)&&> + on_done, + absl::Duration timeout, RequestConfig request_config) { std::move(on_done)( - absl::Status(absl::StatusCode::kInternal, "test msg")); + absl::Status(absl::StatusCode::kInternal, "test msg"), + /* response_metadata= */ {}); scoring_done.Notify(); return absl::OkStatus(); }); diff --git a/services/seller_frontend_service/seller_frontend_main.cc b/services/seller_frontend_service/seller_frontend_main.cc index 1208c95d..2921c537 100644 --- a/services/seller_frontend_service/seller_frontend_main.cc +++ b/services/seller_frontend_service/seller_frontend_main.cc @@ -58,6 +58,9 @@ ABSL_FLAG( "Domain of this seller. Incoming requests are validated against this."); ABSL_FLAG(std::optional, auction_server_host, std::nullopt, "Domain address of the auction server used for ad scoring."); +ABSL_FLAG(std::optional, grpc_arg_default_authority, std::nullopt, + "Authority for auction server name, see " + "https://www.rfc-editor.org/rfc/rfc3986#section-3.2"); ABSL_FLAG(std::optional, key_value_signals_host, std::nullopt, "Domain address of the Key-Value server for the scoring signals."); ABSL_FLAG(std::optional, buyer_server_hosts, std::nullopt, @@ -118,6 +121,8 @@ absl::StatusOr GetConfigClient( SCORE_ADS_RPC_TIMEOUT_MS); config_client.SetFlag(FLAGS_seller_origin_domain, SELLER_ORIGIN_DOMAIN); config_client.SetFlag(FLAGS_auction_server_host, AUCTION_SERVER_HOST); + config_client.SetFlag(FLAGS_grpc_arg_default_authority, + GRPC_ARG_DEFAULT_AUTHORITY_VAL); config_client.SetFlag(FLAGS_key_value_signals_host, KEY_VALUE_SIGNALS_HOST); config_client.SetFlag(FLAGS_buyer_server_hosts, BUYER_SERVER_HOSTS); config_client.SetFlag(FLAGS_enable_buyer_compression, @@ -177,6 +182,7 @@ absl::StatusOr GetConfigClient( SFE_TCMALLOC_BACKGROUND_RELEASE_RATE_BYTES_PER_SECOND); config_client.SetFlag(FLAGS_sfe_tcmalloc_max_total_thread_cache_bytes, SFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES); + if (absl::GetFlag(FLAGS_init_config_client)) { PS_RETURN_IF_ERROR(config_client.Init(config_param_prefix)).LogError() << "Config client failed to initialize."; diff --git a/services/seller_frontend_service/seller_frontend_service.cc b/services/seller_frontend_service/seller_frontend_service.cc index a052109a..99b9bcdd 100644 --- a/services/seller_frontend_service/seller_frontend_service.cc +++ b/services/seller_frontend_service/seller_frontend_service.cc @@ -27,7 +27,7 @@ #include "services/seller_frontend_service/get_component_auction_ciphertexts_reactor.h" #include "services/seller_frontend_service/select_ad_reactor.h" #include "services/seller_frontend_service/select_ad_reactor_app.h" -#include "services/seller_frontend_service/select_ad_reactor_invalid_client.h" +#include "services/seller_frontend_service/select_ad_reactor_invalid.h" #include "services/seller_frontend_service/select_ad_reactor_web.h" #include "services/seller_frontend_service/select_auction_result_reactor.h" #include "src/telemetry/telemetry.h" @@ -48,7 +48,7 @@ std::unique_ptr GetSelectAdReactor( return std::make_unique(context, request, response, clients, config_client); default: - return std::make_unique( + return std::make_unique( context, request, response, clients, config_client); } } @@ -72,6 +72,12 @@ grpc::ServerUnaryReactor* SellerFrontEndService::SelectAd( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response) { LogCommonMetric(request, response); + if (request->ByteSizeLong() == 0) { + auto reactor = std::make_unique( + context, request, response, clients_, config_client_); + reactor->Execute(); + return reactor.release(); + } if (AuctionScope auction_scope = GetAuctionScope(*request); auction_scope == AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { auto reactor = std::make_unique( diff --git a/services/seller_frontend_service/seller_frontend_service.h b/services/seller_frontend_service/seller_frontend_service.h index 0abd6bab..f8e9bf3e 100644 --- a/services/seller_frontend_service/seller_frontend_service.h +++ b/services/seller_frontend_service/seller_frontend_service.h @@ -92,7 +92,11 @@ class SellerFrontEndService final : public SellerFrontEnd::CallbackService { .compression = config_client_.GetBooleanParameter( ENABLE_AUCTION_COMPRESSION), .secure_client = - config_client_.GetBooleanParameter(AUCTION_EGRESS_TLS)})), + config_client_.GetBooleanParameter(AUCTION_EGRESS_TLS), + .grpc_arg_default_authority = + config_client_ + .GetStringParameter(GRPC_ARG_DEFAULT_AUTHORITY_VAL) + .data()})), buyer_factory_([this]() { absl::StatusOr> ig_owner_to_bfe_domain_map = ParseIgOwnerToBfeDomainMap( diff --git a/services/seller_frontend_service/seller_frontend_service_test.cc b/services/seller_frontend_service/seller_frontend_service_test.cc index 7430d9af..f516e1c1 100644 --- a/services/seller_frontend_service/seller_frontend_service_test.cc +++ b/services/seller_frontend_service/seller_frontend_service_test.cc @@ -31,6 +31,7 @@ #include "api/bidding_auction_servers.grpc.pb.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "services/common/feature_flags.h" #include "services/common/metric/server_definition.h" #include "services/common/test/mocks.h" #include "services/common/test/random.h" @@ -61,12 +62,12 @@ constexpr absl::string_view kErrorIntendedForAdServer = using ::testing::_; using ::testing::HasSubstr; using ::testing::Return; -using GetBidDoneCallback = - absl::AnyInvocable>) &&>; -using ScoreAdsDoneCallback = - absl::AnyInvocable>) &&>; +using GetBidDoneCallback = absl::AnyInvocable< + void(absl::StatusOr>, + ResponseMetadata) &&>; +using ScoreAdsDoneCallback = absl::AnyInvocable< + void(absl::StatusOr>, + ResponseMetadata) &&>; using EncodedBuyerInputs = ::google::protobuf::Map; using DecodedBuyerInputs = ::google::protobuf::Map; @@ -101,6 +102,7 @@ class SellerFrontEndServiceTest : public ::testing::Test { config_.SetFlagForTest(kFalse, ENABLE_PROTECTED_APP_SIGNALS); config_.SetFlagForTest(kTrue, ENABLE_PROTECTED_AUDIENCE); config_.SetFlagForTest("{}", SELLER_CLOUD_PLATFORMS_MAP); + absl::SetFlag(&FLAGS_enable_chaffing, false); } ClientRegistry CreateValidClientRegistry() { @@ -234,6 +236,38 @@ TYPED_TEST(SellerFrontEndServiceTest, ReturnsInvalidInputOnInvalidClientType) { absl::StrContains(status.error_message(), kUnsupportedClientType)); } +TYPED_TEST(SellerFrontEndServiceTest, ReturnsInvalidInputOnEmptyRequest) { + // Reporting Client. + std::unique_ptr async_reporter = + std::make_unique( + std::make_unique()); + + server_common::MockKeyFetcherManager key_fetcher_manager; + auto async_provider = + MockAsyncProvider(); + auto scoring = ScoringAsyncClientMock(); + auto bfe_client = BuyerFrontEndAsyncClientFactoryMock(); + ClientRegistry clients{async_provider, + scoring, + bfe_client, + key_fetcher_manager, + /* crypto_client= */ nullptr, + std::move(async_reporter)}; + + SellerFrontEndService seller_frontend_service(&this->config_, + std::move(clients)); + auto start_sfe_result = StartLocalService(&seller_frontend_service); + auto stub = CreateServiceStub(start_sfe_result.port); + + grpc::ClientContext context; + SelectAdRequest request; + SelectAdResponse response; + grpc::Status status = stub->SelectAd(&context, request, &response); + + ASSERT_EQ(status.error_code(), grpc::StatusCode::INVALID_ARGUMENT); + EXPECT_TRUE(absl::StrContains(status.error_message(), kEmptySelectAdRequest)); +} + TYPED_TEST(SellerFrontEndServiceTest, ReturnsInvalidInputOnEmptyBuyerList) { this->config_.SetFlagForTest(kSampleSellerDomain, SELLER_ORIGIN_DOMAIN); @@ -271,7 +305,7 @@ TYPED_TEST(SellerFrontEndServiceTest, ReturnsInvalidInputOnEmptyBuyerList) { grpc::Status status = stub->SelectAd(&context, request, &response); ASSERT_EQ(status.error_code(), grpc::StatusCode::INVALID_ARGUMENT); - ASSERT_EQ(status.error_message(), kEmptyBuyerList); + EXPECT_EQ(status.error_message(), kEmptyBuyerList); } TYPED_TEST(SellerFrontEndServiceTest, @@ -311,7 +345,7 @@ TYPED_TEST(SellerFrontEndServiceTest, grpc::Status status = stub->SelectAd(&context, request, &response); ASSERT_EQ(status.error_code(), grpc::StatusCode::INVALID_ARGUMENT); - ASSERT_EQ(status.error_message(), kInvalidSellerCurrency); + EXPECT_EQ(status.error_message(), kInvalidSellerCurrency); } TYPED_TEST(SellerFrontEndServiceTest, @@ -355,7 +389,7 @@ TYPED_TEST(SellerFrontEndServiceTest, grpc::Status status = stub->SelectAd(&context, request, &response); ASSERT_EQ(status.error_code(), grpc::StatusCode::INVALID_ARGUMENT); - ASSERT_EQ(status.error_message(), kInvalidBuyerCurrency); + EXPECT_EQ(status.error_message(), kInvalidBuyerCurrency); } TYPED_TEST(SellerFrontEndServiceTest, ErrorsOnMissingBuyerInputs) { @@ -426,6 +460,10 @@ TYPED_TEST(SellerFrontEndServiceTest, SendsChaffOnMissingBuyerClient) { BuyerFrontEndAsyncClientFactoryMock client_factory_mock; EXPECT_CALL(client_factory_mock, Get) .WillOnce([](absl::string_view buyer_ig_owner) { return nullptr; }); + EXPECT_CALL(client_factory_mock, Entries).WillRepeatedly([]() { + return std::vector>>(); + }); server_common::MockKeyFetcherManager key_fetcher_manager; EXPECT_CALL(key_fetcher_manager, GetPrivateKey) .WillRepeatedly(Return(GetPrivateKey())); @@ -458,7 +496,7 @@ TYPED_TEST(SellerFrontEndServiceTest, SendsChaffOnMissingBuyerClient) { ASSERT_TRUE(status.ok()) << server_common::ToAbslStatus(status); AuctionResult auction_result = DecryptBrowserAuctionResult( *response.mutable_auction_result_ciphertext(), encryption_context); - ASSERT_TRUE(auction_result.is_chaff()); + EXPECT_TRUE(auction_result.is_chaff()); } TYPED_TEST(SellerFrontEndServiceTest, SendsChaffOnEmptyGetBidsResponse) { @@ -497,6 +535,8 @@ TYPED_TEST(SellerFrontEndServiceTest, SendsChaffOnEmptyGetBidsResponse) { // Buyer Clients BuyerFrontEndAsyncClientFactoryMock buyer_clients; + MockEntriesCallOnBuyerFactory(protected_auction_input.buyer_input(), + buyer_clients); EXPECT_EQ(request.auction_config().buyer_list_size(), protected_auction_input.buyer_input_size()); BuyerBidsResponseMap expected_buyer_bids; @@ -541,7 +581,7 @@ TYPED_TEST(SellerFrontEndServiceTest, SendsChaffOnEmptyGetBidsResponse) { ASSERT_TRUE(status.ok()) << server_common::ToAbslStatus(status); AuctionResult auction_result = DecryptBrowserAuctionResult( *response.mutable_auction_result_ciphertext(), encryption_context); - ASSERT_TRUE(auction_result.is_chaff()); + EXPECT_TRUE(auction_result.is_chaff()); } void SetupFailingBuyerClientMock( @@ -550,7 +590,7 @@ void SetupFailingBuyerClientMock( auto MockGetBids = [](std::unique_ptr get_values_request, const RequestMetadata& metadata, GetBidDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { return absl::InvalidArgumentError("Some Error"); }; auto SetupMockBuyer = @@ -571,9 +611,11 @@ void SetupBuyerClientMock( auto MockGetBids = [](std::unique_ptr get_values_request, const RequestMetadata& metadata, - GetBidDoneCallback on_done, absl::Duration timeout) { + GetBidDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { ABSL_LOG(INFO) << "Getting mock bids returning mocked response to callback"; - std::move(on_done)(std::make_unique()); + std::move(on_done)(std::make_unique(), + /* response_metadata= */ {}); ABSL_LOG(INFO) << "Getting mock bids returned mocked response to callback"; return absl::OkStatus(); }; @@ -647,6 +689,9 @@ TYPED_TEST(SellerFrontEndServiceTest, RawRequestFinishWithSuccess) { std::make_unique(response)); } + MockEntriesCallOnBuyerFactory(protected_auction_input.buyer_input(), + buyer_clients); + // Scoring signals provider std::string ad_render_urls = "test scoring signals"; MockAsyncProvider @@ -658,10 +703,11 @@ TYPED_TEST(SellerFrontEndServiceTest, RawRequestFinishWithSuccess) { .WillRepeatedly( [](std::unique_ptr request, const RequestMetadata& metadata, ScoreAdsDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = std::make_unique(); - std::move(on_done)(std::move(response)); + std::move(on_done)(std::move(response), + /* response_metadata= */ {}); return absl::OkStatus(); }); server_common::MockKeyFetcherManager key_fetcher_manager; @@ -685,7 +731,7 @@ TYPED_TEST(SellerFrontEndServiceTest, RawRequestFinishWithSuccess) { grpc::ClientContext context; grpc::Status status = stub->SelectAd(&context, request, &response); - ASSERT_TRUE(status.ok()) << server_common::ToAbslStatus(status); + EXPECT_TRUE(status.ok()) << server_common::ToAbslStatus(status); } TYPED_TEST(SellerFrontEndServiceTest, ErrorsWhenCannotContactSellerKVServer) { @@ -747,6 +793,9 @@ TYPED_TEST(SellerFrontEndServiceTest, ErrorsWhenCannotContactSellerKVServer) { std::make_unique(response)); } + MockEntriesCallOnBuyerFactory(protected_auction_input.buyer_input(), + buyer_clients); + // Scoring signals provider absl::Status error_to_return(absl::StatusCode::kUnavailable, "Could not reach Seller KV server."); @@ -759,10 +808,11 @@ TYPED_TEST(SellerFrontEndServiceTest, ErrorsWhenCannotContactSellerKVServer) { .WillRepeatedly( [](std::unique_ptr request, const RequestMetadata& metadata, ScoreAdsDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { auto response = std::make_unique(); - std::move(on_done)(std::move(response)); + std::move(on_done)(std::move(response), + /* response_metadata= */ {}); return absl::OkStatus(); }); server_common::MockKeyFetcherManager key_fetcher_manager; @@ -787,7 +837,7 @@ TYPED_TEST(SellerFrontEndServiceTest, ErrorsWhenCannotContactSellerKVServer) { grpc::Status status = stub->SelectAd(&context, request, &response); ASSERT_FALSE(status.ok()) << server_common::ToAbslStatus(status); - ASSERT_EQ(status.error_code(), grpc::StatusCode::INVALID_ARGUMENT); + EXPECT_EQ(status.error_code(), grpc::StatusCode::INVALID_ARGUMENT); } TYPED_TEST(SellerFrontEndServiceTest, @@ -836,6 +886,9 @@ TYPED_TEST(SellerFrontEndServiceTest, SetupFailingBuyerClientMock(local_buyer, buyer_clients); } + MockEntriesCallOnBuyerFactory(protected_auction_input.buyer_input(), + buyer_clients); + // Scoring signals provider mock. MockAsyncProvider scoring_signals_provider; @@ -863,8 +916,8 @@ TYPED_TEST(SellerFrontEndServiceTest, grpc::Status status = stub->SelectAd(&context, request, &response); ASSERT_FALSE(status.ok()) << server_common::ToAbslStatus(status); - ASSERT_EQ(status.error_code(), grpc::StatusCode::INTERNAL); - ASSERT_EQ(status.error_message(), kInternalServerError); + EXPECT_EQ(status.error_code(), grpc::StatusCode::INTERNAL); + EXPECT_EQ(status.error_message(), kInternalServerError); } TYPED_TEST(SellerFrontEndServiceTest, AnyBuyerNotErroringMeansOverallSuccess) { @@ -897,6 +950,9 @@ TYPED_TEST(SellerFrontEndServiceTest, AnyBuyerNotErroringMeansOverallSuccess) { i++; } + MockEntriesCallOnBuyerFactory(protected_auction_input.buyer_input(), + buyer_clients); + // Scoring signals provider mock. MockAsyncProvider scoring_signals_provider; @@ -969,6 +1025,9 @@ TYPED_TEST(SellerFrontEndServiceTest, ++i; } + MockEntriesCallOnBuyerFactory(protected_auction_input.buyer_input(), + buyer_clients); + // Scoring signals provider mock. MockAsyncProvider scoring_signals_provider; @@ -985,7 +1044,8 @@ TYPED_TEST(SellerFrontEndServiceTest, std::unique_ptr score_ads_request, const RequestMetadata& metadata, - ScoreAdsDoneCallback on_done, absl::Duration timeout) { + ScoreAdsDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { ScoreAdsResponse::ScoreAdsRawResponse response; float i = 1; ErrorAccumulator error_accumulator; @@ -1003,7 +1063,8 @@ TYPED_TEST(SellerFrontEndServiceTest, winner = score; } std::move(on_done)( - std::make_unique(response)); + std::make_unique(response), + {}); scoring_done.DecrementCount(); return absl::OkStatus(); }); @@ -1076,6 +1137,9 @@ TYPED_TEST(SellerFrontEndServiceTest, SkipsBuyerCallsAfterLimit) { get_bids_response)); } + MockEntriesCallOnBuyerFactory(protected_auction_input.buyer_input(), + buyer_clients); + // Scoring signals provider mock. MockAsyncProvider scoring_signals_provider; @@ -1094,7 +1158,8 @@ TYPED_TEST(SellerFrontEndServiceTest, SkipsBuyerCallsAfterLimit) { std::unique_ptr score_ads_request, const RequestMetadata& metadata, - ScoreAdsDoneCallback on_done, absl::Duration timeout) { + ScoreAdsDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { ScoreAdsResponse::ScoreAdsRawResponse response; float i = 1; ErrorAccumulator error_accumulator; @@ -1112,7 +1177,8 @@ TYPED_TEST(SellerFrontEndServiceTest, SkipsBuyerCallsAfterLimit) { winner = score; } std::move(on_done)( - std::make_unique(response)); + std::make_unique(response), + {}); notification.Notify(); return absl::OkStatus(); }); @@ -1185,6 +1251,9 @@ TYPED_TEST(SellerFrontEndServiceTest, InternalErrorsFromScoringCauseAChaff) { get_bids_response)); } + MockEntriesCallOnBuyerFactory(protected_auction_input.buyer_input(), + buyer_clients); + // Scoring signals provider mock. MockAsyncProvider scoring_signals_provider; @@ -1198,8 +1267,10 @@ TYPED_TEST(SellerFrontEndServiceTest, InternalErrorsFromScoringCauseAChaff) { .WillOnce([](std::unique_ptr score_ads_request, const RequestMetadata& metadata, - ScoreAdsDoneCallback on_done, absl::Duration timeout) { - std::move(on_done)(absl::InternalError("")); + ScoreAdsDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { + std::move(on_done)(absl::InternalError(""), + /* response_metadata= */ {}); return absl::OkStatus(); }); @@ -1269,6 +1340,9 @@ TYPED_TEST(SellerFrontEndServiceTest, get_bids_response)); } + MockEntriesCallOnBuyerFactory(protected_auction_input.buyer_input(), + buyer_clients); + // Scoring signals provider mock. MockAsyncProvider scoring_signals_provider; @@ -1282,9 +1356,11 @@ TYPED_TEST(SellerFrontEndServiceTest, .WillOnce([](std::unique_ptr score_ads_request, const RequestMetadata& metadata, - ScoreAdsDoneCallback on_done, absl::Duration timeout) { + ScoreAdsDoneCallback on_done, absl::Duration timeout, + RequestConfig request_config) { std::move(on_done)( - absl::ResourceExhaustedError(kErrorIntendedForAdServer)); + absl::ResourceExhaustedError(kErrorIntendedForAdServer), + /* response_metadata= */ {}); return absl::OkStatus(); }); diff --git a/services/seller_frontend_service/util/BUILD b/services/seller_frontend_service/util/BUILD index 4dd08a26..63ff0294 100644 --- a/services/seller_frontend_service/util/BUILD +++ b/services/seller_frontend_service/util/BUILD @@ -71,6 +71,7 @@ cc_library( "//api:bidding_auction_servers_cc_grpc_proto", "//api:bidding_auction_servers_cc_proto", "//services/common/compression:gzip", + "//services/common/util:data_util", "//services/common/util:error_accumulator", "//services/common/util:request_response_constants", "//services/common/util:scoped_cbor", @@ -258,13 +259,13 @@ cc_library( ":validation_utils", "//api:bidding_auction_servers_cc_proto", "//services/common/compression:gzip", + "//services/common/loggers:request_log_context", "//services/common/util:error_categories", "//services/common/util:hpke_utils", "//services/seller_frontend_service/data:seller_frontend_data", "//services/seller_frontend_service/util:framing_utils", "//services/seller_frontend_service/util:web_utils", "@google_privacysandbox_servers_common//src/communication:encoding_utils", - "@google_privacysandbox_servers_common//src/logger:request_context_impl", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", ], ) diff --git a/services/seller_frontend_service/util/proto_mapping_util.cc b/services/seller_frontend_service/util/proto_mapping_util.cc index 3af0f108..bfe6ff2a 100644 --- a/services/seller_frontend_service/util/proto_mapping_util.cc +++ b/services/seller_frontend_service/util/proto_mapping_util.cc @@ -20,8 +20,6 @@ namespace privacy_sandbox::bidding_auction_servers { namespace { -using server_common::log::ContextImpl; - void SetReportingUrls(const WinReportingUrls& win_reporting_urls, AuctionResult& auction_result) { auto* mutable_win_reporting_urls = @@ -112,7 +110,8 @@ absl::StatusOr PackageAuctionResultForWeb( const std::optional& high_score, const std::optional& maybe_bidding_group_map, const std::optional& error, - OhttpHpkeDecryptedMessage& decrypted_request, ContextImpl& log_context) { + OhttpHpkeDecryptedMessage& decrypted_request, + RequestLogContext& log_context) { std::string error_msg; absl::Notification wait_for_error_callback; auto error_handler = [&wait_for_error_callback, @@ -136,10 +135,14 @@ absl::StatusOr PackageAuctionResultForWeb( } else { PS_VLOG(kPlain, log_context) << "AuctionResult:\n" - << [](absl::string_view encoded_data) { + << [&](absl::string_view encoded_data) { auto result = CborDecodeAuctionResultToProto(encoded_data); - return result.ok() ? result->DebugString() - : result.status().ToString(); + if (result.ok()) { + log_context.SetEventMessageField(*result); + return result->DebugString(); + } else { + return result.status().ToString(); + } }(*serialized_data); } absl::string_view data_to_compress = @@ -151,12 +154,15 @@ absl::StatusOr PackageAuctionResultForWeb( absl::StatusOr PackageAuctionResultForApp( const std::optional& high_score, const std::optional& error, - OhttpHpkeDecryptedMessage& decrypted_request, ContextImpl& log_context) { + OhttpHpkeDecryptedMessage& decrypted_request, + RequestLogContext& log_context) { // Map to AuctionResult proto and serialized to bytes array. - std::string serialized_result = - MapAdScoreToAuctionResult(high_score, error).SerializeAsString(); - PS_VLOG(kPlain, log_context) << "AuctionResult:\n" << serialized_result; - return PackageAuctionResultCiphertext(serialized_result, decrypted_request); + + AuctionResult result = MapAdScoreToAuctionResult(high_score, error); + PS_VLOG(kPlain, log_context) << "AuctionResult:\n" << result.DebugString(); + log_context.SetEventMessageField(result); + return PackageAuctionResultCiphertext(result.SerializeAsString(), + decrypted_request); } absl::StatusOr PackageAuctionResultForInvalid( @@ -241,7 +247,7 @@ absl::StatusOr CreateWinningAuctionResultCiphertext( const ScoreAdsResponse::AdScore& ad_score, const std::optional& bidding_group_map, ClientType client_type, OhttpHpkeDecryptedMessage& decrypted_request, - ContextImpl& log_context) { + RequestLogContext& log_context) { absl::StatusOr auction_result_ciphertext; switch (client_type) { case CLIENT_TYPE_ANDROID: @@ -258,7 +264,8 @@ absl::StatusOr CreateWinningAuctionResultCiphertext( absl::StatusOr CreateErrorAuctionResultCiphertext( const AuctionResult::Error& auction_error, ClientType client_type, - OhttpHpkeDecryptedMessage& decrypted_request, ContextImpl& log_context) { + OhttpHpkeDecryptedMessage& decrypted_request, + RequestLogContext& log_context) { switch (client_type) { case CLIENT_TYPE_ANDROID: return PackageAuctionResultForApp( @@ -275,7 +282,7 @@ absl::StatusOr CreateErrorAuctionResultCiphertext( absl::StatusOr CreateChaffAuctionResultCiphertext( ClientType client_type, OhttpHpkeDecryptedMessage& decrypted_request, - ContextImpl& log_context) { + RequestLogContext& log_context) { switch (client_type) { case CLIENT_TYPE_ANDROID: return PackageAuctionResultForApp( @@ -362,7 +369,7 @@ std::vector DecryptAndValidateComponentAuctionResults( absl::string_view request_generation_id, CryptoClientWrapperInterface& crypto_client, server_common::KeyFetcherManagerInterface& key_fetcher_manager, - ErrorAccumulator& error_accumulator, ContextImpl& log_context) { + ErrorAccumulator& error_accumulator, RequestLogContext& log_context) { std::vector component_auction_results; // Keep track of encountered sellers. absl::flat_hash_set component_sellers; diff --git a/services/seller_frontend_service/util/proto_mapping_util.h b/services/seller_frontend_service/util/proto_mapping_util.h index 482de57e..f12846ea 100644 --- a/services/seller_frontend_service/util/proto_mapping_util.h +++ b/services/seller_frontend_service/util/proto_mapping_util.h @@ -23,6 +23,7 @@ #include "absl/status/status.h" #include "api/bidding_auction_servers.pb.h" #include "services/common/compression/gzip.h" +#include "services/common/loggers/request_log_context.h" #include "services/common/util/error_categories.h" #include "services/common/util/hpke_utils.h" #include "services/seller_frontend_service/data/scoring_signals.h" @@ -31,7 +32,6 @@ #include "services/seller_frontend_service/util/web_utils.h" #include "src/communication/encoding_utils.h" #include "src/encryption/key_fetcher/key_fetcher_manager.h" -#include "src/logger/request_context_impl.h" #include "src/util/status_macro/status_macros.h" namespace privacy_sandbox::bidding_auction_servers { @@ -66,20 +66,20 @@ absl::StatusOr CreateWinningAuctionResultCiphertext( const ScoreAdsResponse::AdScore& high_score, const std::optional& bidding_group_maps, ClientType client_type, OhttpHpkeDecryptedMessage& decrypted_request, - server_common::log::ContextImpl& log_context); + RequestLogContext& log_context); // Encodes, compresses and encrypts client error // as auction_result_ciphertext. absl::StatusOr CreateErrorAuctionResultCiphertext( const AuctionResult::Error& auction_error, ClientType client_type, OhttpHpkeDecryptedMessage& decrypted_request, - server_common::log::ContextImpl& log_context); + RequestLogContext& log_context); // Encodes, compresses and encrypts chaff response // as auction_result_ciphertext. absl::StatusOr CreateChaffAuctionResultCiphertext( ClientType client_type, OhttpHpkeDecryptedMessage& decrypted_request, - server_common::log::ContextImpl& log_context); + RequestLogContext& log_context); // Collate all Igs that received bids from all server component auction results. IgsWithBidsMap GetBuyerIgsWithBidsMap( @@ -99,8 +99,7 @@ std::vector DecryptAndValidateComponentAuctionResults( absl::string_view request_generation_id, CryptoClientWrapperInterface& crypto_client, server_common::KeyFetcherManagerInterface& key_fetcher_manager, - ErrorAccumulator& error_accumulator, - server_common::log::ContextImpl& log_context); + ErrorAccumulator& error_accumulator, RequestLogContext& log_context); template T AppProtectedAuctionInputDecodeHelper(absl::string_view encoded_data, diff --git a/services/seller_frontend_service/util/proto_mapping_util_test.cc b/services/seller_frontend_service/util/proto_mapping_util_test.cc index 73e46daf..3d950b08 100644 --- a/services/seller_frontend_service/util/proto_mapping_util_test.cc +++ b/services/seller_frontend_service/util/proto_mapping_util_test.cc @@ -31,7 +31,6 @@ namespace privacy_sandbox::bidding_auction_servers { namespace { using google::scp::core::test::EqualsProto; -using server_common::log::ContextImpl; constexpr char kTestSeller[] = "sample-seller"; constexpr char kBuyer1[] = "bg1"; @@ -402,7 +401,7 @@ class CreateAuctionResultCiphertextTest : public testing::Test { bidding_group_map_ = MakeARandomSingleSellerAuctionResult().bidding_groups(); absl::btree_map context_map; - log_context_ = std::make_unique( + log_context_ = std::make_unique( context_map, server_common::ConsentedDebugConfiguration()); } @@ -410,7 +409,7 @@ class CreateAuctionResultCiphertextTest : public testing::Test { ScoreAdsResponse::AdScore valid_score_; AuctionResult::Error valid_error_; IgsWithBidsMap bidding_group_map_; - std::unique_ptr log_context_; + std::unique_ptr log_context_; std::unique_ptr MakeDecryptedMessage() { auto [request, context] = GetFramedInputAndOhttpContext(MakeARandomString()); diff --git a/services/seller_frontend_service/util/select_ad_reactor_test_utils.cc b/services/seller_frontend_service/util/select_ad_reactor_test_utils.cc index 9c898036..ef2ae374 100644 --- a/services/seller_frontend_service/util/select_ad_reactor_test_utils.cc +++ b/services/seller_frontend_service/util/select_ad_reactor_test_utils.cc @@ -38,9 +38,9 @@ namespace privacy_sandbox::bidding_auction_servers { using ::testing::Matcher; using ::testing::Return; -using GetBidDoneCallback = - absl::AnyInvocable>) &&>; +using GetBidDoneCallback = absl::AnyInvocable< + void(absl::StatusOr>, + ResponseMetadata) &&>; using AdWithBidMetadata = ScoreAdsRequest::ScoreAdsRawRequest::AdWithBidMetadata; using ScoringSignalsDoneCallback = @@ -113,7 +113,7 @@ void SetupBuyerClientMock( [bid, num_buyers_solicited, top_level_seller]( std::unique_ptr get_values_request, const RequestMetadata& metadata, GetBidDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { ABSL_LOG(INFO) << "Returning mock bids"; // Check top level seller is populated for component auctions. if (!top_level_seller.empty()) { @@ -121,7 +121,8 @@ void SetupBuyerClientMock( } if (bid.has_value()) { std::move(on_done)( - std::make_unique(*bid)); + std::make_unique(*bid), + /* response_metadata= */ {}); } if (num_buyers_solicited) { ++(*num_buyers_solicited); @@ -137,6 +138,7 @@ void SetupBuyerClientMock( .WillRepeatedly(MockGetBids); return buyer; }; + auto MockBuyerFactoryCall = [SetupMockBuyer](absl::string_view hostname) { return SetupMockBuyer(std::make_unique()); }; @@ -304,6 +306,9 @@ TrustedServersConfigClient CreateConfig() { config.SetFlagForTest("2", KEY_VALUE_SIGNALS_FETCH_RPC_TIMEOUT_MS); config.SetFlagForTest("3", SCORE_ADS_RPC_TIMEOUT_MS); config.SetFlagForTest(kAuctionHost, AUCTION_SERVER_HOST); + config.SetFlagForTest( + "auction-seller1-tjs-appmesh-virtual-server.seller1-frontend.com", + GRPC_ARG_DEFAULT_AUTHORITY_VAL); config.SetFlagForTest(kTrue, ENABLE_SELLER_FRONTEND_BENCHMARKING); config.SetFlagForTest(kSellerOriginDomain, SELLER_ORIGIN_DOMAIN); config.SetFlagForTest(kTrue, ENABLE_PROTECTED_AUDIENCE); @@ -482,4 +487,18 @@ std::vector GetPASAdWithBidsInMultipleCurrencies( return pas_ads_with_bids; } +void MockEntriesCallOnBuyerFactory( + const google::protobuf::Map& buyer_input, + const BuyerFrontEndAsyncClientFactoryMock& factory) { + std::vector>> + entries; + for (const auto& [buyer, unused] : buyer_input) { + entries.emplace_back(buyer, + std::make_shared()); + } + + EXPECT_CALL(factory, Entries).WillRepeatedly(Return(std::move(entries))); +} + } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/util/select_ad_reactor_test_utils.h b/services/seller_frontend_service/util/select_ad_reactor_test_utils.h index 41de44a2..699efe7d 100644 --- a/services/seller_frontend_service/util/select_ad_reactor_test_utils.h +++ b/services/seller_frontend_service/util/select_ad_reactor_test_utils.h @@ -146,6 +146,10 @@ std::vector GetPASAdWithBidsInMultipleCurrencies( absl::string_view matching_currency, absl::string_view mismatching_currency, absl::string_view base_ad_render_url); +void MockEntriesCallOnBuyerFactory( + const google::protobuf::Map& buyer_input, + const BuyerFrontEndAsyncClientFactoryMock& factory); + TrustedServersConfigClient CreateConfig(); template @@ -356,7 +360,8 @@ GetSelectAdRequestAndClientRegistryForTest( } using ScoreAdsDoneCallback = absl::AnyInvocable>) &&>; + ScoreAdsResponse::ScoreAdsRawResponse>>, + ResponseMetadata) &&>; // Sets up scoring Client EXPECT_CALL(scoring_client, ExecuteInternal) .WillRepeatedly( @@ -364,7 +369,7 @@ GetSelectAdRequestAndClientRegistryForTest( force_set_modified_bid_to_zero]( std::unique_ptr request, const RequestMetadata& metadata, ScoreAdsDoneCallback on_done, - absl::Duration timeout) { + absl::Duration timeout, RequestConfig request_config) { for (const auto& bid : request->ad_bids()) { auto response = std::make_unique(); @@ -416,7 +421,8 @@ GetSelectAdRequestAndClientRegistryForTest( if (client_type == CLIENT_TYPE_ANDROID) { score->set_ad_type(AdType::AD_TYPE_PROTECTED_AUDIENCE_AD); } - std::move(on_done)(std::move(response)); + std::move(on_done)(std::move(response), + /* response_metadata= */ {}); // Expect only one bid. break; } diff --git a/services/seller_frontend_service/util/validation_utils.cc b/services/seller_frontend_service/util/validation_utils.cc index cb73aa6e..d82595f4 100644 --- a/services/seller_frontend_service/util/validation_utils.cc +++ b/services/seller_frontend_service/util/validation_utils.cc @@ -43,7 +43,7 @@ bool ValidateEncryptedSelectAdRequest(const SelectAdRequest& request, report_error_lambda(seller_domain != auction_config.seller(), kWrongSellerDomain); report_error_lambda(request.client_type() == CLIENT_TYPE_UNKNOWN, - kUnknownClientType); + kUnsupportedClientType); if (auction_scope == AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { report_error_lambda(request.component_auction_results_size() == 0, diff --git a/services/seller_frontend_service/util/validation_utils_test.cc b/services/seller_frontend_service/util/validation_utils_test.cc index c9af0db1..de51e51c 100644 --- a/services/seller_frontend_service/util/validation_utils_test.cc +++ b/services/seller_frontend_service/util/validation_utils_test.cc @@ -59,7 +59,7 @@ TEST(ValidateEncryptedSelectAdRequest, AddsErrorsToAccumulatorForSingleSeller) { EXPECT_THAT(errors, Contains(kEmptyAuctionSignals)); EXPECT_THAT(errors, Contains(kEmptySeller)); EXPECT_THAT(errors, Contains(kWrongSellerDomain)); - EXPECT_THAT(errors, Contains(kUnknownClientType)); + EXPECT_THAT(errors, Contains(kUnsupportedClientType)); } TEST(ValidateEncryptedSelectAdRequest, ValidatesDeviceComponentSellerInput) { @@ -93,7 +93,7 @@ TEST(ValidateEncryptedSelectAdRequest, EXPECT_THAT(errors, Contains(kEmptyAuctionSignals)); EXPECT_THAT(errors, Contains(kEmptySeller)); EXPECT_THAT(errors, Contains(kWrongSellerDomain)); - EXPECT_THAT(errors, Contains(kUnknownClientType)); + EXPECT_THAT(errors, Contains(kUnsupportedClientType)); } TEST(ValidateEncryptedSelectAdRequest, ValidatesServerComponentSellerInput) { @@ -127,7 +127,7 @@ TEST(ValidateEncryptedSelectAdRequest, EXPECT_THAT(errors, Contains(kEmptyAuctionSignals)); EXPECT_THAT(errors, Contains(kEmptySeller)); EXPECT_THAT(errors, Contains(kWrongSellerDomain)); - EXPECT_THAT(errors, Contains(kUnknownClientType)); + EXPECT_THAT(errors, Contains(kUnsupportedClientType)); } TEST(ValidateEncryptedSelectAdRequest, @@ -146,7 +146,7 @@ TEST(ValidateEncryptedSelectAdRequest, EXPECT_THAT(errors, Contains(kEmptyAuctionSignals)); EXPECT_THAT(errors, Contains(kEmptySeller)); EXPECT_THAT(errors, Contains(kWrongSellerDomain)); - EXPECT_THAT(errors, Contains(kUnknownClientType)); + EXPECT_THAT(errors, Contains(kUnsupportedClientType)); EXPECT_THAT(errors, Contains(kNoComponentAuctionResults)); } diff --git a/services/seller_frontend_service/util/web_utils.h b/services/seller_frontend_service/util/web_utils.h index 5138d825..fd1736c1 100644 --- a/services/seller_frontend_service/util/web_utils.h +++ b/services/seller_frontend_service/util/web_utils.h @@ -27,6 +27,7 @@ #include "absl/strings/str_format.h" #include "api/bidding_auction_servers.grpc.pb.h" #include "api/bidding_auction_servers.pb.h" +#include "services/common/util/data_util.h" #include "services/common/util/error_accumulator.h" #include "services/common/util/request_response_constants.h" #include "services/common/util/scoped_cbor.h" @@ -148,18 +149,6 @@ absl::StatusOr EncodeComponent( const std::optional& error, const std::function& error_handler); -// Finds the provided string needle index in the haystack array. -template -int FindItemIndex(const std::array& haystack, - absl::string_view needle) { - auto it = std::find(haystack.begin(), haystack.end(), needle); - if (it == haystack.end()) { - return -1; - } - - return std::distance(haystack.begin(), it); -} - // Helper to validate the type of a CBOR object. bool IsTypeValid( absl::AnyInvocable is_valid_type, diff --git a/services/seller_frontend_service/util/web_utils_test.cc b/services/seller_frontend_service/util/web_utils_test.cc index 5b60e639..e31ec3dc 100644 --- a/services/seller_frontend_service/util/web_utils_test.cc +++ b/services/seller_frontend_service/util/web_utils_test.cc @@ -71,8 +71,7 @@ inline constexpr char kConsentedDebugToken[] = "xyz"; inline constexpr char kUsdIso[] = "USD"; inline constexpr char kTestBuyerReportingId[] = "testBuyerReportingId"; -server_common::log::ContextImpl log_context{ - {}, server_common::ConsentedDebugConfiguration()}; +RequestLogContext log_context{{}, server_common::ConsentedDebugConfiguration()}; using BiddingGroupMap = ::google::protobuf::Map; diff --git a/third_party/cddl/BUILD b/third_party/cddl/BUILD new file mode 100644 index 00000000..dd5d6a3d --- /dev/null +++ b/third_party/cddl/BUILD @@ -0,0 +1,17 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package( + default_visibility = ["//visibility:public"], +) diff --git a/third_party/cddl/BUILD.bazel b/third_party/cddl/BUILD.bazel new file mode 100644 index 00000000..de71067e --- /dev/null +++ b/third_party/cddl/BUILD.bazel @@ -0,0 +1,15 @@ +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +exports_files( + [ + "cddl.h", + ], + visibility = ["//visibility:public"], +) + +copy_file( + name = "cddl_h", + src = ":cddl.h", + out = "include/cddl.h", + visibility = ["//visibility:public"], +) diff --git a/third_party/cddl/Cargo.lock b/third_party/cddl/Cargo.lock new file mode 100644 index 00000000..2374d274 --- /dev/null +++ b/third_party/cddl/Cargo.lock @@ -0,0 +1,1400 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "abnf" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33741baa462d86e43fdec5e8ffca7c6ac82847ad06cbfb382c1bdbf527de9e6b" +dependencies = [ + "abnf-core", + "nom", +] + +[[package]] +name = "abnf-core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c44e09c43ae1c368fb91a03a566472d0087c26cf7e1b9e8e289c14ede681dd7d" +dependencies = [ + "nom", +] + +[[package]] +name = "abnf_to_pest" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "939d59666dd9a7964a3a5312b9d24c9c107630752ee64f2dd5038189a23fe331" +dependencies = [ + "abnf", + "indexmap", + "itertools 0.10.5", + "pretty", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "base16" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27c3610c36aee21ce8ac510e6224498de4228ad772a171ed65643a24693a5a8" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64-url" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb9fb9fb058cc3063b5fc88d9a21eefa2735871498a04e1650da76ed511c8569" +dependencies = [ + "base64", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "cc" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" + +[[package]] +name = "cddl" +version = "0.9.4" +dependencies = [ + "abnf_to_pest", + "base16", + "base64-url", + "chrono", + "ciborium", + "clap", + "codespan-reporting", + "console_error_panic_hook", + "crossterm", + "data-encoding", + "displaydoc", + "hexf-parse", + "indoc", + "itertools 0.11.0", + "lexical-core", + "libc", + "log", + "pest_meta", + "pest_vm", + "pretty_assertions", + "regex", + "regex-syntax 0.7.5", + "serde", + "serde-wasm-bindgen", + "serde_json", + "simplelog", + "uriparse", + "wasm-bindgen", + "wasm-bindgen-test", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.52.5", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_derive", + "clap_lex", + "indexmap", + "once_cell", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crossterm" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +dependencies = [ + "bitflags 2.5.0", + "crossterm_winapi", + "libc", + "mio", + "parking_lot", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "either" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hexf-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "memchr" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.5", +] + +[[package]] +name = "pest" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_meta" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + +[[package]] +name = "pest_vm" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36985db609c630fd9cb74e1808e553a221a9658fbcc94446b3c252e7e32b8b11" +dependencies = [ + "pest", + "pest_meta", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "pretty" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83f3aa1e3ca87d3b124db7461265ac176b40c277f37e503eaa29c9c75c037846" +dependencies = [ + "arrayvec", + "log", + "typed-arena", + "unicode-segmentation", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + +[[package]] +name = "regex" +version = "1.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax 0.8.3", +] + +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.3", +] + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + +[[package]] +name = "regex-syntax" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-wasm-bindgen" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3b143e2833c57ab9ad3ea280d21fd34e285a42837aeb0ee301f4f41890fa00e" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "serde_derive" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "serde_json" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "simplelog" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0" +dependencies = [ + "log", + "termcolor", + "time", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + +[[package]] +name = "thiserror" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + +[[package]] +name = "unicode-width" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" + +[[package]] +name = "uriparse" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0200d0fc04d809396c2ad43f3c95da3582a2556eba8d453c1087f4120ee352ff" +dependencies = [ + "fnv", + "lazy_static", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.66", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9bf62a58e0780af3e852044583deee40983e5886da43a271dd772379987667b" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "scoped-tls", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" diff --git a/third_party/cddl/Cargo.toml b/third_party/cddl/Cargo.toml new file mode 100644 index 00000000..46c9da2e --- /dev/null +++ b/third_party/cddl/Cargo.toml @@ -0,0 +1,118 @@ +[package] +name = "cddl" +description = "Parser for the Concise data definition language (CDDL)" +repository = "https://github.com/anweiss/cddl" +homepage = "https://cddl.anweiss.tech" +categories = ["parser-implementations", "encoding", "development-tools", "wasm"] +license = "MIT" +version = "0.9.4" +authors = ["Andrew Weiss "] +readme = "README.md" +edition = "2018" +exclude = [ + "cddl-lsp/**/*", + "www/**/*", + ".github/**/*", + ".devcontainer/**/*", + "pkg/**/*", + ".dockerignore", + "Dockerfile", + "tests/**/*", +] + +[lib] +name = "cddl" +path = "src/lib.rs" +required-features = ["std", "json", "cbor"] +crate-type = ["cdylib", "rlib", "staticlib"] + +[dependencies] +base16 = { version = "0.2.1", default-features = false } +# base64 = { version = "0.21.0", default-features = false } +data-encoding = { version = "2.3.3", default-features = false } +chrono = { version = "0.4.38", optional = true } +clap = { version = "3.2.23", optional = true, features = ["derive"] } +codespan-reporting = "0.11.1" +hexf-parse = "0.2.1" +itertools = "0.11.0" +lexical-core = "0.8.5" +regex = { version = "1.5.4", default-features = false, features = [ + "std", + "unicode-perl", +] } +regex-syntax = { version = "0.7.1", optional = true } +libc = { version = "0.2", default-features = true } +serde = { version = "1.0.203", optional = true, features = ["derive"] } +ciborium = { version = "0.2.0", optional = true } +serde_json = { version = "1.0.66", optional = true, default-features = false, features = [ + "std", +] } +uriparse = { version = "0.6.3", optional = true } +base64-url = { version = "2.0.2", optional = true } +abnf_to_pest = "0.5.1" +pest_meta = "2.7.10" +pest_vm = "2.1.0" +displaydoc = { version = "0.2.3", default-features = false } +log = "0.4.21" +simplelog = "0.12.2" + +[dev-dependencies] +indoc = "2.0.1" +pretty_assertions = "1.2.0" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +crossterm = { version = "0.27.0", optional = true } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +console_error_panic_hook = "0.1.6" +serde-wasm-bindgen = { version = "0.5.0", optional = true } +wasm-bindgen = { version = "0.2", optional = true } + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies] +wasm-bindgen-test = "0.3.25" + +[features] +default = [ + "std", + "ast-span", + "ast-comments", + "json", + "cbor", + "additional-controls", + "ast-parent", +] +std = [ + "base16/alloc", + "data-encoding/alloc", + "serde_json", + "ciborium", + "serde", + "chrono", + "wasm-bindgen", + "serde-wasm-bindgen", + "clap", + "crossterm", + "uriparse", + "base64-url", + "regex-syntax", +] +lsp = ["std"] +additional-controls = [] +ast-span = [] +ast-comments = [] +ast-parent = [] +json = ["std"] +cbor = ["std"] + +[[bin]] +name = "cddl" +required-features = ["std", "json", "cbor"] +path = "src/bin/cli.rs" +test = false + +[profile.release] +opt-level = "s" +lto = true + +[badges] +maintenance = { status = "actively-developed" } diff --git a/third_party/cddl/WORKSPACE b/third_party/cddl/WORKSPACE new file mode 100644 index 00000000..d6793f74 --- /dev/null +++ b/third_party/cddl/WORKSPACE @@ -0,0 +1,37 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_rust", + sha256 = "36ab8f9facae745c9c9c1b33d225623d976e78f2cc3f729b7973d8c20934ab95", + urls = ["https://github.com/bazelbuild/rules_rust/releases/download/0.31.0/rules_rust-v0.31.0.tar.gz"], +) + +load("@rules_rust//rust:repositories.bzl", "rules_rust_dependencies", "rust_register_toolchains") + +rules_rust_dependencies() + +rust_register_toolchains( + edition = "2018", + versions = [ + "1.74.0", + ], +) + +load("@rules_rust//crate_universe:repositories.bzl", "crate_universe_dependencies") + +crate_universe_dependencies() + +load("@rules_rust//crate_universe:defs.bzl", "crates_repository") + +crates_repository( + name = "cddl_crate_index", + cargo_lockfile = "//:Cargo.lock", + lockfile = "//:cargo-bazel-lock.json", + manifests = [ + "//:Cargo.toml", + ], +) + +load("@cddl_crate_index//:defs.bzl", cddl_crate_repositories = "crate_repositories") + +cddl_crate_repositories() diff --git a/third_party/cddl/cargo-bazel-lock.json b/third_party/cddl/cargo-bazel-lock.json new file mode 100644 index 00000000..db56832b --- /dev/null +++ b/third_party/cddl/cargo-bazel-lock.json @@ -0,0 +1,8114 @@ +{ + "checksum": "5fa053e8d77b4b72eb5b46082323a268bbd947f38350c35427f14e2825a1f46b", + "crates": { + "abnf 0.12.0": { + "name": "abnf", + "version": "0.12.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/abnf/0.12.0/download", + "sha256": "33741baa462d86e43fdec5e8ffca7c6ac82847ad06cbfb382c1bdbf527de9e6b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "abnf", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "abnf", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "abnf-core 0.5.0", + "target": "abnf_core" + }, + { + "id": "nom 7.1.3", + "target": "nom" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.12.0" + }, + "license": "MIT OR Apache-2.0" + }, + "abnf-core 0.5.0": { + "name": "abnf-core", + "version": "0.5.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/abnf-core/0.5.0/download", + "sha256": "c44e09c43ae1c368fb91a03a566472d0087c26cf7e1b9e8e289c14ede681dd7d" + } + }, + "targets": [ + { + "Library": { + "crate_name": "abnf_core", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "abnf_core", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "nom 7.1.3", + "target": "nom" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.5.0" + }, + "license": "MIT OR Apache-2.0" + }, + "abnf_to_pest 0.5.1": { + "name": "abnf_to_pest", + "version": "0.5.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/abnf_to_pest/0.5.1/download", + "sha256": "939d59666dd9a7964a3a5312b9d24c9c107630752ee64f2dd5038189a23fe331" + } + }, + "targets": [ + { + "Library": { + "crate_name": "abnf_to_pest", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "abnf_to_pest", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "abnf 0.12.0", + "target": "abnf" + }, + { + "id": "indexmap 1.9.3", + "target": "indexmap" + }, + { + "id": "itertools 0.10.5", + "target": "itertools" + }, + { + "id": "pretty 0.11.3", + "target": "pretty" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.5.1" + }, + "license": "MIT OR Apache-2.0" + }, + "aho-corasick 1.1.3": { + "name": "aho-corasick", + "version": "1.1.3", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/aho-corasick/1.1.3/download", + "sha256": "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" + } + }, + "targets": [ + { + "Library": { + "crate_name": "aho_corasick", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "aho_corasick", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "1.1.3" + }, + "license": "Unlicense OR MIT" + }, + "android-tzdata 0.1.1": { + "name": "android-tzdata", + "version": "0.1.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/android-tzdata/0.1.1/download", + "sha256": "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + } + }, + "targets": [ + { + "Library": { + "crate_name": "android_tzdata", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "android_tzdata", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "0.1.1" + }, + "license": "MIT OR Apache-2.0" + }, + "android_system_properties 0.1.5": { + "name": "android_system_properties", + "version": "0.1.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/android_system_properties/0.1.5/download", + "sha256": "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" + } + }, + "targets": [ + { + "Library": { + "crate_name": "android_system_properties", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "android_system_properties", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.1.5" + }, + "license": "MIT/Apache-2.0" + }, + "arrayvec 0.5.2": { + "name": "arrayvec", + "version": "0.5.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/arrayvec/0.5.2/download", + "sha256": "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "arrayvec", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "arrayvec", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "edition": "2018", + "version": "0.5.2" + }, + "license": "MIT/Apache-2.0" + }, + "atty 0.2.14": { + "name": "atty", + "version": "0.2.14", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/atty/0.2.14/download", + "sha256": "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "atty", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "atty", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [], + "selects": { + "cfg(target_os = \"hermit\")": [ + { + "id": "hermit-abi 0.1.19", + "target": "hermit_abi" + } + ], + "cfg(unix)": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ], + "cfg(windows)": [ + { + "id": "winapi 0.3.9", + "target": "winapi" + } + ] + } + }, + "edition": "2015", + "version": "0.2.14" + }, + "license": "MIT" + }, + "autocfg 1.3.0": { + "name": "autocfg", + "version": "1.3.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/autocfg/1.3.0/download", + "sha256": "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + } + }, + "targets": [ + { + "Library": { + "crate_name": "autocfg", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "autocfg", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "1.3.0" + }, + "license": "Apache-2.0 OR MIT" + }, + "base16 0.2.1": { + "name": "base16", + "version": "0.2.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/base16/0.2.1/download", + "sha256": "d27c3610c36aee21ce8ac510e6224498de4228ad772a171ed65643a24693a5a8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "base16", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "base16", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc" + ], + "selects": {} + }, + "edition": "2018", + "version": "0.2.1" + }, + "license": "CC0-1.0" + }, + "base64 0.21.7": { + "name": "base64", + "version": "0.21.7", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/base64/0.21.7/download", + "sha256": "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + } + }, + "targets": [ + { + "Library": { + "crate_name": "base64", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "base64", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "std" + ], + "selects": {} + }, + "edition": "2018", + "version": "0.21.7" + }, + "license": "MIT OR Apache-2.0" + }, + "base64-url 2.0.2": { + "name": "base64-url", + "version": "2.0.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/base64-url/2.0.2/download", + "sha256": "fb9fb9fb058cc3063b5fc88d9a21eefa2735871498a04e1650da76ed511c8569" + } + }, + "targets": [ + { + "Library": { + "crate_name": "base64_url", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "base64_url", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "base64 0.21.7", + "target": "base64" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "2.0.2" + }, + "license": "MIT" + }, + "bitflags 1.3.2": { + "name": "bitflags", + "version": "1.3.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/bitflags/1.3.2/download", + "sha256": "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + } + }, + "targets": [ + { + "Library": { + "crate_name": "bitflags", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "bitflags", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "edition": "2018", + "version": "1.3.2" + }, + "license": "MIT/Apache-2.0" + }, + "bitflags 2.5.0": { + "name": "bitflags", + "version": "2.5.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/bitflags/2.5.0/download", + "sha256": "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + } + }, + "targets": [ + { + "Library": { + "crate_name": "bitflags", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "bitflags", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "2.5.0" + }, + "license": "MIT OR Apache-2.0" + }, + "block-buffer 0.10.4": { + "name": "block-buffer", + "version": "0.10.4", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/block-buffer/0.10.4/download", + "sha256": "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" + } + }, + "targets": [ + { + "Library": { + "crate_name": "block_buffer", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "block_buffer", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "generic-array 0.14.7", + "target": "generic_array" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.10.4" + }, + "license": "MIT OR Apache-2.0" + }, + "bumpalo 3.16.0": { + "name": "bumpalo", + "version": "3.16.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/bumpalo/3.16.0/download", + "sha256": "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "bumpalo", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "bumpalo", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "edition": "2021", + "version": "3.16.0" + }, + "license": "MIT OR Apache-2.0" + }, + "cc 1.0.98": { + "name": "cc", + "version": "1.0.98", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/cc/1.0.98/download", + "sha256": "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" + } + }, + "targets": [ + { + "Library": { + "crate_name": "cc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "cc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "1.0.98" + }, + "license": "MIT OR Apache-2.0" + }, + "cddl 0.9.4": { + "name": "cddl", + "version": "0.9.4", + "repository": null, + "targets": [ + { + "Library": { + "crate_name": "cddl", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "cddl", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "additional-controls", + "ast-comments", + "ast-parent", + "ast-span", + "base64-url", + "cbor", + "chrono", + "ciborium", + "clap", + "crossterm", + "default", + "json", + "regex-syntax", + "serde", + "serde-wasm-bindgen", + "serde_json", + "std", + "uriparse", + "wasm-bindgen" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "abnf_to_pest 0.5.1", + "target": "abnf_to_pest" + }, + { + "id": "base16 0.2.1", + "target": "base16" + }, + { + "id": "base64-url 2.0.2", + "target": "base64_url" + }, + { + "id": "chrono 0.4.38", + "target": "chrono" + }, + { + "id": "ciborium 0.2.2", + "target": "ciborium" + }, + { + "id": "clap 3.2.25", + "target": "clap" + }, + { + "id": "codespan-reporting 0.11.1", + "target": "codespan_reporting" + }, + { + "id": "data-encoding 2.6.0", + "target": "data_encoding" + }, + { + "id": "hexf-parse 0.2.1", + "target": "hexf_parse" + }, + { + "id": "itertools 0.11.0", + "target": "itertools" + }, + { + "id": "lexical-core 0.8.5", + "target": "lexical_core" + }, + { + "id": "libc 0.2.155", + "target": "libc" + }, + { + "id": "log 0.4.21", + "target": "log" + }, + { + "id": "pest_meta 2.7.10", + "target": "pest_meta" + }, + { + "id": "pest_vm 2.7.10", + "target": "pest_vm" + }, + { + "id": "regex 1.10.4", + "target": "regex" + }, + { + "id": "regex-syntax 0.7.5", + "target": "regex_syntax" + }, + { + "id": "serde 1.0.203", + "target": "serde" + }, + { + "id": "serde_json 1.0.117", + "target": "serde_json" + }, + { + "id": "simplelog 0.12.2", + "target": "simplelog" + }, + { + "id": "uriparse 0.6.4", + "target": "uriparse" + } + ], + "selects": { + "cfg(not(target_arch = \"wasm32\"))": [ + { + "id": "crossterm 0.27.0", + "target": "crossterm" + } + ], + "cfg(target_arch = \"wasm32\")": [ + { + "id": "console_error_panic_hook 0.1.7", + "target": "console_error_panic_hook" + }, + { + "id": "serde-wasm-bindgen 0.5.0", + "target": "serde_wasm_bindgen" + }, + { + "id": "wasm-bindgen 0.2.92", + "target": "wasm_bindgen" + } + ] + } + }, + "deps_dev": { + "common": [ + { + "id": "pretty_assertions 1.4.0", + "target": "pretty_assertions" + } + ], + "selects": { + "cfg(target_arch = \"wasm32\")": [ + { + "id": "wasm-bindgen-test 0.3.42", + "target": "wasm_bindgen_test" + } + ] + } + }, + "edition": "2018", + "proc_macro_deps": { + "common": [ + { + "id": "displaydoc 0.2.4", + "target": "displaydoc" + } + ], + "selects": {} + }, + "proc_macro_deps_dev": { + "common": [ + { + "id": "indoc 2.0.5", + "target": "indoc" + } + ], + "selects": {} + }, + "version": "0.9.4" + }, + "license": "MIT" + }, + "cfg-if 1.0.0": { + "name": "cfg-if", + "version": "1.0.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/cfg-if/1.0.0/download", + "sha256": "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + } + }, + "targets": [ + { + "Library": { + "crate_name": "cfg_if", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "cfg_if", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "1.0.0" + }, + "license": "MIT/Apache-2.0" + }, + "chrono 0.4.38": { + "name": "chrono", + "version": "0.4.38", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/chrono/0.4.38/download", + "sha256": "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" + } + }, + "targets": [ + { + "Library": { + "crate_name": "chrono", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "chrono", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "android-tzdata", + "clock", + "default", + "iana-time-zone", + "js-sys", + "now", + "oldtime", + "std", + "wasm-bindgen", + "wasmbind", + "winapi", + "windows-targets" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "num-traits 0.2.19", + "target": "num_traits" + } + ], + "selects": { + "cfg(all(target_arch = \"wasm32\", not(any(target_os = \"emscripten\", target_os = \"wasi\"))))": [ + { + "id": "js-sys 0.3.69", + "target": "js_sys" + }, + { + "id": "wasm-bindgen 0.2.92", + "target": "wasm_bindgen" + } + ], + "cfg(target_os = \"android\")": [ + { + "id": "android-tzdata 0.1.1", + "target": "android_tzdata" + } + ], + "cfg(unix)": [ + { + "id": "iana-time-zone 0.1.60", + "target": "iana_time_zone" + } + ], + "cfg(windows)": [ + { + "id": "windows-targets 0.52.5", + "target": "windows_targets" + } + ] + } + }, + "edition": "2021", + "version": "0.4.38" + }, + "license": "MIT OR Apache-2.0" + }, + "ciborium 0.2.2": { + "name": "ciborium", + "version": "0.2.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/ciborium/0.2.2/download", + "sha256": "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" + } + }, + "targets": [ + { + "Library": { + "crate_name": "ciborium", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "ciborium", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "ciborium-io 0.2.2", + "target": "ciborium_io" + }, + { + "id": "ciborium-ll 0.2.2", + "target": "ciborium_ll" + }, + { + "id": "serde 1.0.203", + "target": "serde" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.2.2" + }, + "license": "Apache-2.0" + }, + "ciborium-io 0.2.2": { + "name": "ciborium-io", + "version": "0.2.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/ciborium-io/0.2.2/download", + "sha256": "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + } + }, + "targets": [ + { + "Library": { + "crate_name": "ciborium_io", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "ciborium_io", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "std" + ], + "selects": {} + }, + "edition": "2021", + "version": "0.2.2" + }, + "license": "Apache-2.0" + }, + "ciborium-ll 0.2.2": { + "name": "ciborium-ll", + "version": "0.2.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/ciborium-ll/0.2.2/download", + "sha256": "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" + } + }, + "targets": [ + { + "Library": { + "crate_name": "ciborium_ll", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "ciborium_ll", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "ciborium-io 0.2.2", + "target": "ciborium_io" + }, + { + "id": "half 2.4.1", + "target": "half" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.2.2" + }, + "license": "Apache-2.0" + }, + "clap 3.2.25": { + "name": "clap", + "version": "3.2.25", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/clap/3.2.25/download", + "sha256": "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" + } + }, + "targets": [ + { + "Library": { + "crate_name": "clap", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "clap", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "atty", + "clap_derive", + "color", + "default", + "derive", + "once_cell", + "std", + "strsim", + "suggestions", + "termcolor" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "atty 0.2.14", + "target": "atty" + }, + { + "id": "bitflags 1.3.2", + "target": "bitflags" + }, + { + "id": "clap_lex 0.2.4", + "target": "clap_lex" + }, + { + "id": "indexmap 1.9.3", + "target": "indexmap" + }, + { + "id": "once_cell 1.19.0", + "target": "once_cell" + }, + { + "id": "strsim 0.10.0", + "target": "strsim" + }, + { + "id": "termcolor 1.4.1", + "target": "termcolor" + }, + { + "id": "textwrap 0.16.1", + "target": "textwrap" + } + ], + "selects": {} + }, + "edition": "2021", + "proc_macro_deps": { + "common": [ + { + "id": "clap_derive 3.2.25", + "target": "clap_derive" + } + ], + "selects": {} + }, + "version": "3.2.25" + }, + "license": "MIT OR Apache-2.0" + }, + "clap_derive 3.2.25": { + "name": "clap_derive", + "version": "3.2.25", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/clap_derive/3.2.25/download", + "sha256": "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "clap_derive", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "clap_derive", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "heck 0.4.1", + "target": "heck" + }, + { + "id": "proc-macro-error 1.0.4", + "target": "proc_macro_error" + }, + { + "id": "proc-macro2 1.0.85", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.36", + "target": "quote" + }, + { + "id": "syn 1.0.109", + "target": "syn" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "3.2.25" + }, + "license": "MIT OR Apache-2.0" + }, + "clap_lex 0.2.4": { + "name": "clap_lex", + "version": "0.2.4", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/clap_lex/0.2.4/download", + "sha256": "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" + } + }, + "targets": [ + { + "Library": { + "crate_name": "clap_lex", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "clap_lex", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "os_str_bytes 6.6.1", + "target": "os_str_bytes" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.2.4" + }, + "license": "MIT OR Apache-2.0" + }, + "codespan-reporting 0.11.1": { + "name": "codespan-reporting", + "version": "0.11.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/codespan-reporting/0.11.1/download", + "sha256": "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" + } + }, + "targets": [ + { + "Library": { + "crate_name": "codespan_reporting", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "codespan_reporting", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "termcolor 1.4.1", + "target": "termcolor" + }, + { + "id": "unicode-width 0.1.13", + "target": "unicode_width" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.11.1" + }, + "license": "Apache-2.0" + }, + "console_error_panic_hook 0.1.7": { + "name": "console_error_panic_hook", + "version": "0.1.7", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/console_error_panic_hook/0.1.7/download", + "sha256": "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" + } + }, + "targets": [ + { + "Library": { + "crate_name": "console_error_panic_hook", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "console_error_panic_hook", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + }, + { + "id": "wasm-bindgen 0.2.92", + "target": "wasm_bindgen" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.1.7" + }, + "license": "Apache-2.0/MIT" + }, + "core-foundation-sys 0.8.6": { + "name": "core-foundation-sys", + "version": "0.8.6", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/core-foundation-sys/0.8.6/download", + "sha256": "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + } + }, + "targets": [ + { + "Library": { + "crate_name": "core_foundation_sys", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "core_foundation_sys", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "link" + ], + "selects": {} + }, + "edition": "2018", + "version": "0.8.6" + }, + "license": "MIT OR Apache-2.0" + }, + "cpufeatures 0.2.12": { + "name": "cpufeatures", + "version": "0.2.12", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/cpufeatures/0.2.12/download", + "sha256": "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" + } + }, + "targets": [ + { + "Library": { + "crate_name": "cpufeatures", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "cpufeatures", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [], + "selects": { + "aarch64-linux-android": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ], + "cfg(all(target_arch = \"aarch64\", target_os = \"linux\"))": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ], + "cfg(all(target_arch = \"aarch64\", target_vendor = \"apple\"))": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ], + "cfg(all(target_arch = \"loongarch64\", target_os = \"linux\"))": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ] + } + }, + "edition": "2018", + "version": "0.2.12" + }, + "license": "MIT OR Apache-2.0" + }, + "crossterm 0.27.0": { + "name": "crossterm", + "version": "0.27.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/crossterm/0.27.0/download", + "sha256": "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" + } + }, + "targets": [ + { + "Library": { + "crate_name": "crossterm", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "crossterm", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "bracketed-paste", + "default", + "events", + "windows" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "bitflags 2.5.0", + "target": "bitflags" + }, + { + "id": "parking_lot 0.12.3", + "target": "parking_lot" + } + ], + "selects": { + "cfg(unix)": [ + { + "id": "libc 0.2.155", + "target": "libc" + }, + { + "id": "mio 0.8.11", + "target": "mio" + }, + { + "id": "signal-hook 0.3.17", + "target": "signal_hook" + }, + { + "id": "signal-hook-mio 0.2.3", + "target": "signal_hook_mio" + } + ], + "cfg(windows)": [ + { + "id": "crossterm_winapi 0.9.1", + "target": "crossterm_winapi" + }, + { + "id": "winapi 0.3.9", + "target": "winapi" + } + ] + } + }, + "edition": "2021", + "version": "0.27.0" + }, + "license": "MIT" + }, + "crossterm_winapi 0.9.1": { + "name": "crossterm_winapi", + "version": "0.9.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/crossterm_winapi/0.9.1/download", + "sha256": "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "crossterm_winapi", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "crossterm_winapi", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [], + "selects": { + "cfg(windows)": [ + { + "id": "winapi 0.3.9", + "target": "winapi" + } + ] + } + }, + "edition": "2018", + "version": "0.9.1" + }, + "license": "MIT" + }, + "crunchy 0.2.2": { + "name": "crunchy", + "version": "0.2.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/crunchy/0.2.2/download", + "sha256": "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + } + }, + "targets": [ + { + "Library": { + "crate_name": "crunchy", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "crunchy", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "crunchy 0.2.2", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.2.2" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT" + }, + "crypto-common 0.1.6": { + "name": "crypto-common", + "version": "0.1.6", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/crypto-common/0.1.6/download", + "sha256": "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" + } + }, + "targets": [ + { + "Library": { + "crate_name": "crypto_common", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "crypto_common", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "generic-array 0.14.7", + "target": "generic_array" + }, + { + "id": "typenum 1.17.0", + "target": "typenum" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.1.6" + }, + "license": "MIT OR Apache-2.0" + }, + "data-encoding 2.6.0": { + "name": "data-encoding", + "version": "2.6.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/data-encoding/2.6.0/download", + "sha256": "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" + } + }, + "targets": [ + { + "Library": { + "crate_name": "data_encoding", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "data_encoding", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc" + ], + "selects": {} + }, + "edition": "2018", + "version": "2.6.0" + }, + "license": "MIT" + }, + "deranged 0.3.11": { + "name": "deranged", + "version": "0.3.11", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/deranged/0.3.11/download", + "sha256": "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" + } + }, + "targets": [ + { + "Library": { + "crate_name": "deranged", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "deranged", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "powerfmt", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "powerfmt 0.2.0", + "target": "powerfmt" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.3.11" + }, + "license": "MIT OR Apache-2.0" + }, + "diff 0.1.13": { + "name": "diff", + "version": "0.1.13", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/diff/0.1.13/download", + "sha256": "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "diff", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "diff", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "0.1.13" + }, + "license": "MIT OR Apache-2.0" + }, + "digest 0.10.7": { + "name": "digest", + "version": "0.10.7", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/digest/0.10.7/download", + "sha256": "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" + } + }, + "targets": [ + { + "Library": { + "crate_name": "digest", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "digest", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "block-buffer", + "core-api", + "default" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "block-buffer 0.10.4", + "target": "block_buffer" + }, + { + "id": "crypto-common 0.1.6", + "target": "crypto_common" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.10.7" + }, + "license": "MIT OR Apache-2.0" + }, + "displaydoc 0.2.4": { + "name": "displaydoc", + "version": "0.2.4", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/displaydoc/0.2.4/download", + "sha256": "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "displaydoc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "displaydoc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "proc-macro2 1.0.85", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.36", + "target": "quote" + }, + { + "id": "syn 2.0.66", + "target": "syn" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.2.4" + }, + "license": "MIT OR Apache-2.0" + }, + "either 1.12.0": { + "name": "either", + "version": "1.12.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/either/1.12.0/download", + "sha256": "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "either", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "either", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "use_std" + ], + "selects": {} + }, + "edition": "2018", + "version": "1.12.0" + }, + "license": "MIT OR Apache-2.0" + }, + "fnv 1.0.7": { + "name": "fnv", + "version": "1.0.7", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/fnv/1.0.7/download", + "sha256": "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + } + }, + "targets": [ + { + "Library": { + "crate_name": "fnv", + "crate_root": "lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "fnv", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "edition": "2015", + "version": "1.0.7" + }, + "license": "Apache-2.0 / MIT" + }, + "generic-array 0.14.7": { + "name": "generic-array", + "version": "0.14.7", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/generic-array/0.14.7/download", + "sha256": "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" + } + }, + "targets": [ + { + "Library": { + "crate_name": "generic_array", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "generic_array", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "more_lengths" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "generic-array 0.14.7", + "target": "build_script_build" + }, + { + "id": "typenum 1.17.0", + "target": "typenum" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.14.7" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "version_check 0.9.4", + "target": "version_check" + } + ], + "selects": {} + } + }, + "license": "MIT" + }, + "half 2.4.1": { + "name": "half", + "version": "2.4.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/half/2.4.1/download", + "sha256": "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" + } + }, + "targets": [ + { + "Library": { + "crate_name": "half", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "half", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + } + ], + "selects": { + "cfg(target_arch = \"spirv\")": [ + { + "id": "crunchy 0.2.2", + "target": "crunchy" + } + ] + } + }, + "edition": "2021", + "version": "2.4.1" + }, + "license": "MIT OR Apache-2.0" + }, + "hashbrown 0.12.3": { + "name": "hashbrown", + "version": "0.12.3", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/hashbrown/0.12.3/download", + "sha256": "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + } + }, + "targets": [ + { + "Library": { + "crate_name": "hashbrown", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "hashbrown", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "raw" + ], + "selects": {} + }, + "edition": "2021", + "version": "0.12.3" + }, + "license": "MIT OR Apache-2.0" + }, + "heck 0.4.1": { + "name": "heck", + "version": "0.4.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/heck/0.4.1/download", + "sha256": "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "heck", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "heck", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "edition": "2018", + "version": "0.4.1" + }, + "license": "MIT OR Apache-2.0" + }, + "hermit-abi 0.1.19": { + "name": "hermit-abi", + "version": "0.1.19", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/hermit-abi/0.1.19/download", + "sha256": "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" + } + }, + "targets": [ + { + "Library": { + "crate_name": "hermit_abi", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "hermit_abi", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.1.19" + }, + "license": "MIT/Apache-2.0" + }, + "hexf-parse 0.2.1": { + "name": "hexf-parse", + "version": "0.2.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/hexf-parse/0.2.1/download", + "sha256": "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + } + }, + "targets": [ + { + "Library": { + "crate_name": "hexf_parse", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "hexf_parse", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "0.2.1" + }, + "license": "CC0-1.0" + }, + "iana-time-zone 0.1.60": { + "name": "iana-time-zone", + "version": "0.1.60", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/iana-time-zone/0.1.60/download", + "sha256": "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" + } + }, + "targets": [ + { + "Library": { + "crate_name": "iana_time_zone", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "iana_time_zone", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "fallback" + ], + "selects": {} + }, + "deps": { + "common": [], + "selects": { + "cfg(any(target_os = \"macos\", target_os = \"ios\"))": [ + { + "id": "core-foundation-sys 0.8.6", + "target": "core_foundation_sys" + } + ], + "cfg(target_arch = \"wasm32\")": [ + { + "id": "js-sys 0.3.69", + "target": "js_sys" + }, + { + "id": "wasm-bindgen 0.2.92", + "target": "wasm_bindgen" + } + ], + "cfg(target_os = \"android\")": [ + { + "id": "android_system_properties 0.1.5", + "target": "android_system_properties" + } + ], + "cfg(target_os = \"haiku\")": [ + { + "id": "iana-time-zone-haiku 0.1.2", + "target": "iana_time_zone_haiku" + } + ], + "cfg(target_os = \"windows\")": [ + { + "id": "windows-core 0.52.0", + "target": "windows_core" + } + ] + } + }, + "edition": "2018", + "version": "0.1.60" + }, + "license": "MIT OR Apache-2.0" + }, + "iana-time-zone-haiku 0.1.2": { + "name": "iana-time-zone-haiku", + "version": "0.1.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/iana-time-zone-haiku/0.1.2/download", + "sha256": "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" + } + }, + "targets": [ + { + "Library": { + "crate_name": "iana_time_zone_haiku", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "iana_time_zone_haiku", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "iana-time-zone-haiku 0.1.2", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.1.2" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cc 1.0.98", + "target": "cc" + } + ], + "selects": {} + } + }, + "license": "MIT OR Apache-2.0" + }, + "indexmap 1.9.3": { + "name": "indexmap", + "version": "1.9.3", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/indexmap/1.9.3/download", + "sha256": "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" + } + }, + "targets": [ + { + "Library": { + "crate_name": "indexmap", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "indexmap", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "hashbrown 0.12.3", + "target": "hashbrown" + }, + { + "id": "indexmap 1.9.3", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.9.3" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "autocfg 1.3.0", + "target": "autocfg" + } + ], + "selects": {} + } + }, + "license": "Apache-2.0 OR MIT" + }, + "indoc 2.0.5": { + "name": "indoc", + "version": "2.0.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/indoc/2.0.5/download", + "sha256": "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "indoc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "indoc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "2.0.5" + }, + "license": "MIT OR Apache-2.0" + }, + "itertools 0.10.5": { + "name": "itertools", + "version": "0.10.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/itertools/0.10.5/download", + "sha256": "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" + } + }, + "targets": [ + { + "Library": { + "crate_name": "itertools", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "itertools", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "use_alloc", + "use_std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "either 1.12.0", + "target": "either" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.10.5" + }, + "license": "MIT/Apache-2.0" + }, + "itertools 0.11.0": { + "name": "itertools", + "version": "0.11.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/itertools/0.11.0/download", + "sha256": "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" + } + }, + "targets": [ + { + "Library": { + "crate_name": "itertools", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "itertools", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "use_alloc", + "use_std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "either 1.12.0", + "target": "either" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.11.0" + }, + "license": "MIT OR Apache-2.0" + }, + "itoa 1.0.11": { + "name": "itoa", + "version": "1.0.11", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/itoa/1.0.11/download", + "sha256": "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "itoa", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "itoa", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "1.0.11" + }, + "license": "MIT OR Apache-2.0" + }, + "js-sys 0.3.69": { + "name": "js-sys", + "version": "0.3.69", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/js-sys/0.3.69/download", + "sha256": "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" + } + }, + "targets": [ + { + "Library": { + "crate_name": "js_sys", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "js_sys", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "wasm-bindgen 0.2.92", + "target": "wasm_bindgen" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.3.69" + }, + "license": "MIT OR Apache-2.0" + }, + "lazy_static 1.4.0": { + "name": "lazy_static", + "version": "1.4.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/lazy_static/1.4.0/download", + "sha256": "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + } + }, + "targets": [ + { + "Library": { + "crate_name": "lazy_static", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "lazy_static", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "1.4.0" + }, + "license": "MIT/Apache-2.0" + }, + "lexical-core 0.8.5": { + "name": "lexical-core", + "version": "0.8.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/lexical-core/0.8.5/download", + "sha256": "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" + } + }, + "targets": [ + { + "Library": { + "crate_name": "lexical_core", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "lexical_core", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "floats", + "integers", + "lexical-parse-float", + "lexical-parse-integer", + "lexical-write-float", + "lexical-write-integer", + "parse", + "parse-floats", + "parse-integers", + "std", + "write", + "write-floats", + "write-integers" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "lexical-parse-float 0.8.5", + "target": "lexical_parse_float" + }, + { + "id": "lexical-parse-integer 0.8.6", + "target": "lexical_parse_integer" + }, + { + "id": "lexical-util 0.8.5", + "target": "lexical_util" + }, + { + "id": "lexical-write-float 0.8.5", + "target": "lexical_write_float" + }, + { + "id": "lexical-write-integer 0.8.5", + "target": "lexical_write_integer" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.8.5" + }, + "license": "MIT/Apache-2.0" + }, + "lexical-parse-float 0.8.5": { + "name": "lexical-parse-float", + "version": "0.8.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/lexical-parse-float/0.8.5/download", + "sha256": "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" + } + }, + "targets": [ + { + "Library": { + "crate_name": "lexical_parse_float", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "lexical_parse_float", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "lexical-parse-integer 0.8.6", + "target": "lexical_parse_integer" + }, + { + "id": "lexical-util 0.8.5", + "target": "lexical_util" + }, + { + "id": "static_assertions 1.1.0", + "target": "static_assertions" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.8.5" + }, + "license": "MIT/Apache-2.0" + }, + "lexical-parse-integer 0.8.6": { + "name": "lexical-parse-integer", + "version": "0.8.6", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/lexical-parse-integer/0.8.6/download", + "sha256": "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" + } + }, + "targets": [ + { + "Library": { + "crate_name": "lexical_parse_integer", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "lexical_parse_integer", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "lexical-util 0.8.5", + "target": "lexical_util" + }, + { + "id": "static_assertions 1.1.0", + "target": "static_assertions" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.8.6" + }, + "license": "MIT/Apache-2.0" + }, + "lexical-util 0.8.5": { + "name": "lexical-util", + "version": "0.8.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/lexical-util/0.8.5/download", + "sha256": "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" + } + }, + "targets": [ + { + "Library": { + "crate_name": "lexical_util", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "lexical_util", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "floats", + "integers", + "parse", + "parse-floats", + "parse-integers", + "std", + "write", + "write-floats", + "write-integers" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "static_assertions 1.1.0", + "target": "static_assertions" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.8.5" + }, + "license": "MIT/Apache-2.0" + }, + "lexical-write-float 0.8.5": { + "name": "lexical-write-float", + "version": "0.8.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/lexical-write-float/0.8.5/download", + "sha256": "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" + } + }, + "targets": [ + { + "Library": { + "crate_name": "lexical_write_float", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "lexical_write_float", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "lexical-util 0.8.5", + "target": "lexical_util" + }, + { + "id": "lexical-write-integer 0.8.5", + "target": "lexical_write_integer" + }, + { + "id": "static_assertions 1.1.0", + "target": "static_assertions" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.8.5" + }, + "license": "MIT/Apache-2.0" + }, + "lexical-write-integer 0.8.5": { + "name": "lexical-write-integer", + "version": "0.8.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/lexical-write-integer/0.8.5/download", + "sha256": "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" + } + }, + "targets": [ + { + "Library": { + "crate_name": "lexical_write_integer", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "lexical_write_integer", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "lexical-util 0.8.5", + "target": "lexical_util" + }, + { + "id": "static_assertions 1.1.0", + "target": "static_assertions" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.8.5" + }, + "license": "MIT/Apache-2.0" + }, + "libc 0.2.155": { + "name": "libc", + "version": "0.2.155", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/libc/0.2.155/download", + "sha256": "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "libc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "libc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "libc 0.2.155", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.2.155" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "lock_api 0.4.12": { + "name": "lock_api", + "version": "0.4.12", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/lock_api/0.4.12/download", + "sha256": "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" + } + }, + "targets": [ + { + "Library": { + "crate_name": "lock_api", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "lock_api", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "atomic_usize", + "default" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "lock_api 0.4.12", + "target": "build_script_build" + }, + { + "id": "scopeguard 1.2.0", + "target": "scopeguard" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.4.12" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "autocfg 1.3.0", + "target": "autocfg" + } + ], + "selects": {} + } + }, + "license": "MIT OR Apache-2.0" + }, + "log 0.4.21": { + "name": "log", + "version": "0.4.21", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/log/0.4.21/download", + "sha256": "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "log", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "log", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "std" + ], + "selects": {} + }, + "edition": "2021", + "version": "0.4.21" + }, + "license": "MIT OR Apache-2.0" + }, + "memchr 2.7.2": { + "name": "memchr", + "version": "2.7.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/memchr/2.7.2/download", + "sha256": "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" + } + }, + "targets": [ + { + "Library": { + "crate_name": "memchr", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "memchr", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "default", + "std" + ], + "selects": {} + }, + "edition": "2021", + "version": "2.7.2" + }, + "license": "Unlicense OR MIT" + }, + "minimal-lexical 0.2.1": { + "name": "minimal-lexical", + "version": "0.2.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/minimal-lexical/0.2.1/download", + "sha256": "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + } + }, + "targets": [ + { + "Library": { + "crate_name": "minimal_lexical", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "minimal_lexical", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "std" + ], + "selects": {} + }, + "edition": "2018", + "version": "0.2.1" + }, + "license": "MIT/Apache-2.0" + }, + "mio 0.8.11": { + "name": "mio", + "version": "0.8.11", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/mio/0.8.11/download", + "sha256": "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "mio", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "mio", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "log", + "net", + "os-ext", + "os-poll" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "log 0.4.21", + "target": "log" + } + ], + "selects": { + "cfg(target_os = \"wasi\")": [ + { + "id": "libc 0.2.155", + "target": "libc" + }, + { + "id": "wasi 0.11.0+wasi-snapshot-preview1", + "target": "wasi" + } + ], + "cfg(unix)": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ], + "cfg(windows)": [ + { + "id": "windows-sys 0.48.0", + "target": "windows_sys" + } + ] + } + }, + "edition": "2018", + "version": "0.8.11" + }, + "license": "MIT" + }, + "nom 7.1.3": { + "name": "nom", + "version": "7.1.3", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/nom/7.1.3/download", + "sha256": "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" + } + }, + "targets": [ + { + "Library": { + "crate_name": "nom", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "nom", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "memchr 2.7.2", + "target": "memchr" + }, + { + "id": "minimal-lexical 0.2.1", + "target": "minimal_lexical" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "7.1.3" + }, + "license": "MIT" + }, + "num-conv 0.1.0": { + "name": "num-conv", + "version": "0.1.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/num-conv/0.1.0/download", + "sha256": "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + } + }, + "targets": [ + { + "Library": { + "crate_name": "num_conv", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "num_conv", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "0.1.0" + }, + "license": "MIT OR Apache-2.0" + }, + "num-traits 0.2.19": { + "name": "num-traits", + "version": "0.2.19", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/num-traits/0.2.19/download", + "sha256": "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" + } + }, + "targets": [ + { + "Library": { + "crate_name": "num_traits", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "num_traits", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "num-traits 0.2.19", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.2.19" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "autocfg 1.3.0", + "target": "autocfg" + } + ], + "selects": {} + } + }, + "license": "MIT OR Apache-2.0" + }, + "num_threads 0.1.7": { + "name": "num_threads", + "version": "0.1.7", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/num_threads/0.1.7/download", + "sha256": "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" + } + }, + "targets": [ + { + "Library": { + "crate_name": "num_threads", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "num_threads", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [], + "selects": { + "cfg(any(target_os = \"macos\", target_os = \"ios\", target_os = \"freebsd\"))": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ] + } + }, + "edition": "2015", + "version": "0.1.7" + }, + "license": "MIT OR Apache-2.0" + }, + "once_cell 1.19.0": { + "name": "once_cell", + "version": "1.19.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/once_cell/1.19.0/download", + "sha256": "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + } + }, + "targets": [ + { + "Library": { + "crate_name": "once_cell", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "once_cell", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "default", + "race", + "std" + ], + "selects": {} + }, + "edition": "2021", + "version": "1.19.0" + }, + "license": "MIT OR Apache-2.0" + }, + "os_str_bytes 6.6.1": { + "name": "os_str_bytes", + "version": "6.6.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/os_str_bytes/6.6.1/download", + "sha256": "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + } + }, + "targets": [ + { + "Library": { + "crate_name": "os_str_bytes", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "os_str_bytes", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "raw_os_str" + ], + "selects": {} + }, + "edition": "2021", + "version": "6.6.1" + }, + "license": "MIT OR Apache-2.0" + }, + "parking_lot 0.12.3": { + "name": "parking_lot", + "version": "0.12.3", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/parking_lot/0.12.3/download", + "sha256": "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" + } + }, + "targets": [ + { + "Library": { + "crate_name": "parking_lot", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "parking_lot", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "lock_api 0.4.12", + "target": "lock_api" + }, + { + "id": "parking_lot_core 0.9.10", + "target": "parking_lot_core" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.12.3" + }, + "license": "MIT OR Apache-2.0" + }, + "parking_lot_core 0.9.10": { + "name": "parking_lot_core", + "version": "0.9.10", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/parking_lot_core/0.9.10/download", + "sha256": "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "parking_lot_core", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "parking_lot_core", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + }, + { + "id": "parking_lot_core 0.9.10", + "target": "build_script_build" + }, + { + "id": "smallvec 1.13.2", + "target": "smallvec" + } + ], + "selects": { + "cfg(target_os = \"redox\")": [ + { + "id": "redox_syscall 0.5.1", + "target": "syscall" + } + ], + "cfg(unix)": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ], + "cfg(windows)": [ + { + "id": "windows-targets 0.52.5", + "target": "windows_targets" + } + ] + } + }, + "edition": "2021", + "version": "0.9.10" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "pest 2.7.10": { + "name": "pest", + "version": "2.7.10", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/pest/2.7.10/download", + "sha256": "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "pest", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "pest", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "memchr", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "memchr 2.7.2", + "target": "memchr" + }, + { + "id": "thiserror 1.0.61", + "target": "thiserror" + }, + { + "id": "ucd-trie 0.1.6", + "target": "ucd_trie" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "2.7.10" + }, + "license": "MIT OR Apache-2.0" + }, + "pest_meta 2.7.10": { + "name": "pest_meta", + "version": "2.7.10", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/pest_meta/2.7.10/download", + "sha256": "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" + } + }, + "targets": [ + { + "Library": { + "crate_name": "pest_meta", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "pest_meta", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "once_cell 1.19.0", + "target": "once_cell" + }, + { + "id": "pest 2.7.10", + "target": "pest" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "2.7.10" + }, + "license": "MIT OR Apache-2.0" + }, + "pest_vm 2.7.10": { + "name": "pest_vm", + "version": "2.7.10", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/pest_vm/2.7.10/download", + "sha256": "36985db609c630fd9cb74e1808e553a221a9658fbcc94446b3c252e7e32b8b11" + } + }, + "targets": [ + { + "Library": { + "crate_name": "pest_vm", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "pest_vm", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "pest 2.7.10", + "target": "pest" + }, + { + "id": "pest_meta 2.7.10", + "target": "pest_meta" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "2.7.10" + }, + "license": "MIT OR Apache-2.0" + }, + "powerfmt 0.2.0": { + "name": "powerfmt", + "version": "0.2.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/powerfmt/0.2.0/download", + "sha256": "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + } + }, + "targets": [ + { + "Library": { + "crate_name": "powerfmt", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "powerfmt", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "0.2.0" + }, + "license": "MIT OR Apache-2.0" + }, + "pretty 0.11.3": { + "name": "pretty", + "version": "0.11.3", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/pretty/0.11.3/download", + "sha256": "83f3aa1e3ca87d3b124db7461265ac176b40c277f37e503eaa29c9c75c037846" + } + }, + "targets": [ + { + "Library": { + "crate_name": "pretty", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "pretty", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "arrayvec 0.5.2", + "target": "arrayvec" + }, + { + "id": "log 0.4.21", + "target": "log" + }, + { + "id": "typed-arena 2.0.2", + "target": "typed_arena" + }, + { + "id": "unicode-segmentation 1.11.0", + "target": "unicode_segmentation" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.11.3" + }, + "license": "MIT" + }, + "pretty_assertions 1.4.0": { + "name": "pretty_assertions", + "version": "1.4.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/pretty_assertions/1.4.0/download", + "sha256": "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" + } + }, + "targets": [ + { + "Library": { + "crate_name": "pretty_assertions", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "pretty_assertions", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "diff 0.1.13", + "target": "diff" + }, + { + "id": "yansi 0.5.1", + "target": "yansi" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "1.4.0" + }, + "license": "MIT OR Apache-2.0" + }, + "proc-macro-error 1.0.4": { + "name": "proc-macro-error", + "version": "1.0.4", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/proc-macro-error/1.0.4/download", + "sha256": "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "proc_macro_error", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "proc_macro_error", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "syn", + "syn-error" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "proc-macro-error 1.0.4", + "target": "build_script_build" + }, + { + "id": "proc-macro2 1.0.85", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.36", + "target": "quote" + }, + { + "id": "syn 1.0.109", + "target": "syn" + } + ], + "selects": {} + }, + "edition": "2018", + "proc_macro_deps": { + "common": [ + { + "id": "proc-macro-error-attr 1.0.4", + "target": "proc_macro_error_attr" + } + ], + "selects": {} + }, + "version": "1.0.4" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "version_check 0.9.4", + "target": "version_check" + } + ], + "selects": {} + } + }, + "license": "MIT OR Apache-2.0" + }, + "proc-macro-error-attr 1.0.4": { + "name": "proc-macro-error-attr", + "version": "1.0.4", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/proc-macro-error-attr/1.0.4/download", + "sha256": "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "proc_macro_error_attr", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "proc_macro_error_attr", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "proc-macro-error-attr 1.0.4", + "target": "build_script_build" + }, + { + "id": "proc-macro2 1.0.85", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.36", + "target": "quote" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "1.0.4" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "version_check 0.9.4", + "target": "version_check" + } + ], + "selects": {} + } + }, + "license": "MIT OR Apache-2.0" + }, + "proc-macro2 1.0.85": { + "name": "proc-macro2", + "version": "1.0.85", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/proc-macro2/1.0.85/download", + "sha256": "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" + } + }, + "targets": [ + { + "Library": { + "crate_name": "proc_macro2", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "proc_macro2", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "proc-macro" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "proc-macro2 1.0.85", + "target": "build_script_build" + }, + { + "id": "unicode-ident 1.0.12", + "target": "unicode_ident" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.0.85" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "quote 1.0.36": { + "name": "quote", + "version": "1.0.36", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/quote/1.0.36/download", + "sha256": "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" + } + }, + "targets": [ + { + "Library": { + "crate_name": "quote", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "quote", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "proc-macro" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "proc-macro2 1.0.85", + "target": "proc_macro2" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "1.0.36" + }, + "license": "MIT OR Apache-2.0" + }, + "redox_syscall 0.5.1": { + "name": "redox_syscall", + "version": "0.5.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/redox_syscall/0.5.1/download", + "sha256": "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" + } + }, + "targets": [ + { + "Library": { + "crate_name": "syscall", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "syscall", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "bitflags 2.5.0", + "target": "bitflags" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.5.1" + }, + "license": "MIT" + }, + "regex 1.10.4": { + "name": "regex", + "version": "1.10.4", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/regex/1.10.4/download", + "sha256": "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "regex", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "regex", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "std", + "unicode-perl" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "regex-automata 0.4.6", + "target": "regex_automata" + }, + { + "id": "regex-syntax 0.8.3", + "target": "regex_syntax" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.10.4" + }, + "license": "MIT OR Apache-2.0" + }, + "regex-automata 0.4.6": { + "name": "regex-automata", + "version": "0.4.6", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/regex-automata/0.4.6/download", + "sha256": "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" + } + }, + "targets": [ + { + "Library": { + "crate_name": "regex_automata", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "regex_automata", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "meta", + "nfa-pikevm", + "nfa-thompson", + "std", + "syntax", + "unicode-perl", + "unicode-word-boundary" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "regex-syntax 0.8.3", + "target": "regex_syntax" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.4.6" + }, + "license": "MIT OR Apache-2.0" + }, + "regex-syntax 0.7.5": { + "name": "regex-syntax", + "version": "0.7.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/regex-syntax/0.7.5/download", + "sha256": "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + } + }, + "targets": [ + { + "Library": { + "crate_name": "regex_syntax", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "regex_syntax", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std", + "unicode", + "unicode-age", + "unicode-bool", + "unicode-case", + "unicode-gencat", + "unicode-perl", + "unicode-script", + "unicode-segment" + ], + "selects": {} + }, + "edition": "2021", + "version": "0.7.5" + }, + "license": "MIT OR Apache-2.0" + }, + "regex-syntax 0.8.3": { + "name": "regex-syntax", + "version": "0.8.3", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/regex-syntax/0.8.3/download", + "sha256": "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" + } + }, + "targets": [ + { + "Library": { + "crate_name": "regex_syntax", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "regex_syntax", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "std", + "unicode-perl" + ], + "selects": {} + }, + "edition": "2021", + "version": "0.8.3" + }, + "license": "MIT OR Apache-2.0" + }, + "ryu 1.0.18": { + "name": "ryu", + "version": "1.0.18", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/ryu/1.0.18/download", + "sha256": "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + } + }, + "targets": [ + { + "Library": { + "crate_name": "ryu", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "ryu", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "1.0.18" + }, + "license": "Apache-2.0 OR BSL-1.0" + }, + "scoped-tls 1.0.1": { + "name": "scoped-tls", + "version": "1.0.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/scoped-tls/1.0.1/download", + "sha256": "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + } + }, + "targets": [ + { + "Library": { + "crate_name": "scoped_tls", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "scoped_tls", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "1.0.1" + }, + "license": "MIT/Apache-2.0" + }, + "scopeguard 1.2.0": { + "name": "scopeguard", + "version": "1.2.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/scopeguard/1.2.0/download", + "sha256": "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + } + }, + "targets": [ + { + "Library": { + "crate_name": "scopeguard", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "scopeguard", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "1.2.0" + }, + "license": "MIT OR Apache-2.0" + }, + "serde 1.0.203": { + "name": "serde", + "version": "1.0.203", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/serde/1.0.203/download", + "sha256": "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" + } + }, + "targets": [ + { + "Library": { + "crate_name": "serde", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "serde", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "default", + "derive", + "serde_derive", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "serde 1.0.203", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "proc_macro_deps": { + "common": [ + { + "id": "serde_derive 1.0.203", + "target": "serde_derive" + } + ], + "selects": {} + }, + "version": "1.0.203" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "serde-wasm-bindgen 0.5.0": { + "name": "serde-wasm-bindgen", + "version": "0.5.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/serde-wasm-bindgen/0.5.0/download", + "sha256": "f3b143e2833c57ab9ad3ea280d21fd34e285a42837aeb0ee301f4f41890fa00e" + } + }, + "targets": [ + { + "Library": { + "crate_name": "serde_wasm_bindgen", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "serde_wasm_bindgen", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "js-sys 0.3.69", + "target": "js_sys" + }, + { + "id": "serde 1.0.203", + "target": "serde" + }, + { + "id": "wasm-bindgen 0.2.92", + "target": "wasm_bindgen" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.5.0" + }, + "license": "MIT" + }, + "serde_derive 1.0.203": { + "name": "serde_derive", + "version": "1.0.203", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/serde_derive/1.0.203/download", + "sha256": "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "serde_derive", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "serde_derive", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "proc-macro2 1.0.85", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.36", + "target": "quote" + }, + { + "id": "syn 2.0.66", + "target": "syn" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "1.0.203" + }, + "license": "MIT OR Apache-2.0" + }, + "serde_json 1.0.117": { + "name": "serde_json", + "version": "1.0.117", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/serde_json/1.0.117/download", + "sha256": "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" + } + }, + "targets": [ + { + "Library": { + "crate_name": "serde_json", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "serde_json", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "itoa 1.0.11", + "target": "itoa" + }, + { + "id": "ryu 1.0.18", + "target": "ryu" + }, + { + "id": "serde 1.0.203", + "target": "serde" + }, + { + "id": "serde_json 1.0.117", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.0.117" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "sha2 0.10.8": { + "name": "sha2", + "version": "0.10.8", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/sha2/0.10.8/download", + "sha256": "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "sha2", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "sha2", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + }, + { + "id": "digest 0.10.7", + "target": "digest" + } + ], + "selects": { + "cfg(any(target_arch = \"aarch64\", target_arch = \"x86_64\", target_arch = \"x86\"))": [ + { + "id": "cpufeatures 0.2.12", + "target": "cpufeatures" + } + ] + } + }, + "edition": "2018", + "version": "0.10.8" + }, + "license": "MIT OR Apache-2.0" + }, + "signal-hook 0.3.17": { + "name": "signal-hook", + "version": "0.3.17", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/signal-hook/0.3.17/download", + "sha256": "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" + } + }, + "targets": [ + { + "Library": { + "crate_name": "signal_hook", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "signal_hook", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "channel", + "default", + "iterator" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "libc 0.2.155", + "target": "libc" + }, + { + "id": "signal-hook 0.3.17", + "target": "build_script_build" + }, + { + "id": "signal-hook-registry 1.4.2", + "target": "signal_hook_registry" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.3.17" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "Apache-2.0/MIT" + }, + "signal-hook-mio 0.2.3": { + "name": "signal-hook-mio", + "version": "0.2.3", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/signal-hook-mio/0.2.3/download", + "sha256": "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" + } + }, + "targets": [ + { + "Library": { + "crate_name": "signal_hook_mio", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "signal_hook_mio", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "mio-0_8", + "support-v0_8" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "libc 0.2.155", + "target": "libc" + }, + { + "id": "mio 0.8.11", + "target": "mio", + "alias": "mio_0_8" + }, + { + "id": "signal-hook 0.3.17", + "target": "signal_hook" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.2.3" + }, + "license": "Apache-2.0/MIT" + }, + "signal-hook-registry 1.4.2": { + "name": "signal-hook-registry", + "version": "1.4.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/signal-hook-registry/1.4.2/download", + "sha256": "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" + } + }, + "targets": [ + { + "Library": { + "crate_name": "signal_hook_registry", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "signal_hook_registry", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "1.4.2" + }, + "license": "Apache-2.0/MIT" + }, + "simplelog 0.12.2": { + "name": "simplelog", + "version": "0.12.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/simplelog/0.12.2/download", + "sha256": "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0" + } + }, + "targets": [ + { + "Library": { + "crate_name": "simplelog", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "simplelog", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "local-offset", + "termcolor" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "log 0.4.21", + "target": "log" + }, + { + "id": "termcolor 1.4.1", + "target": "termcolor" + }, + { + "id": "time 0.3.36", + "target": "time" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.12.2" + }, + "license": "MIT OR Apache-2.0" + }, + "smallvec 1.13.2": { + "name": "smallvec", + "version": "1.13.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/smallvec/1.13.2/download", + "sha256": "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + } + }, + "targets": [ + { + "Library": { + "crate_name": "smallvec", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "smallvec", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "1.13.2" + }, + "license": "MIT OR Apache-2.0" + }, + "static_assertions 1.1.0": { + "name": "static_assertions", + "version": "1.1.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/static_assertions/1.1.0/download", + "sha256": "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + } + }, + "targets": [ + { + "Library": { + "crate_name": "static_assertions", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "static_assertions", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "1.1.0" + }, + "license": "MIT OR Apache-2.0" + }, + "strsim 0.10.0": { + "name": "strsim", + "version": "0.10.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/strsim/0.10.0/download", + "sha256": "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + } + }, + "targets": [ + { + "Library": { + "crate_name": "strsim", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "strsim", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "0.10.0" + }, + "license": "MIT" + }, + "syn 1.0.109": { + "name": "syn", + "version": "1.0.109", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/syn/1.0.109/download", + "sha256": "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" + } + }, + "targets": [ + { + "Library": { + "crate_name": "syn", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "syn", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "clone-impls", + "default", + "derive", + "full", + "parsing", + "printing", + "proc-macro", + "quote" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "proc-macro2 1.0.85", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.36", + "target": "quote" + }, + { + "id": "syn 1.0.109", + "target": "build_script_build" + }, + { + "id": "unicode-ident 1.0.12", + "target": "unicode_ident" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "1.0.109" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "syn 2.0.66": { + "name": "syn", + "version": "2.0.66", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/syn/2.0.66/download", + "sha256": "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" + } + }, + "targets": [ + { + "Library": { + "crate_name": "syn", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "syn", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "clone-impls", + "default", + "derive", + "parsing", + "printing", + "proc-macro" + ], + "selects": { + "wasm32-unknown-unknown": [ + "full", + "visit" + ], + "wasm32-wasi": [ + "full", + "visit" + ] + } + }, + "deps": { + "common": [ + { + "id": "proc-macro2 1.0.85", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.36", + "target": "quote" + }, + { + "id": "unicode-ident 1.0.12", + "target": "unicode_ident" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "2.0.66" + }, + "license": "MIT OR Apache-2.0" + }, + "termcolor 1.4.1": { + "name": "termcolor", + "version": "1.4.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/termcolor/1.4.1/download", + "sha256": "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" + } + }, + "targets": [ + { + "Library": { + "crate_name": "termcolor", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "termcolor", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [], + "selects": { + "cfg(windows)": [ + { + "id": "winapi-util 0.1.8", + "target": "winapi_util" + } + ] + } + }, + "edition": "2018", + "version": "1.4.1" + }, + "license": "Unlicense OR MIT" + }, + "textwrap 0.16.1": { + "name": "textwrap", + "version": "0.16.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/textwrap/0.16.1/download", + "sha256": "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + } + }, + "targets": [ + { + "Library": { + "crate_name": "textwrap", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "textwrap", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "0.16.1" + }, + "license": "MIT" + }, + "thiserror 1.0.61": { + "name": "thiserror", + "version": "1.0.61", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/thiserror/1.0.61/download", + "sha256": "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" + } + }, + "targets": [ + { + "Library": { + "crate_name": "thiserror", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "thiserror", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "thiserror 1.0.61", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "proc_macro_deps": { + "common": [ + { + "id": "thiserror-impl 1.0.61", + "target": "thiserror_impl" + } + ], + "selects": {} + }, + "version": "1.0.61" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "thiserror-impl 1.0.61": { + "name": "thiserror-impl", + "version": "1.0.61", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/thiserror-impl/1.0.61/download", + "sha256": "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "thiserror_impl", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "thiserror_impl", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "proc-macro2 1.0.85", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.36", + "target": "quote" + }, + { + "id": "syn 2.0.66", + "target": "syn" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.0.61" + }, + "license": "MIT OR Apache-2.0" + }, + "time 0.3.36": { + "name": "time", + "version": "0.3.36", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/time/0.3.36/download", + "sha256": "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" + } + }, + "targets": [ + { + "Library": { + "crate_name": "time", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "time", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "default", + "formatting", + "local-offset", + "macros", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "deranged 0.3.11", + "target": "deranged" + }, + { + "id": "itoa 1.0.11", + "target": "itoa" + }, + { + "id": "num-conv 0.1.0", + "target": "num_conv" + }, + { + "id": "powerfmt 0.2.0", + "target": "powerfmt" + }, + { + "id": "time-core 0.1.2", + "target": "time_core" + } + ], + "selects": { + "cfg(target_family = \"unix\")": [ + { + "id": "libc 0.2.155", + "target": "libc" + }, + { + "id": "num_threads 0.1.7", + "target": "num_threads" + } + ] + } + }, + "edition": "2021", + "proc_macro_deps": { + "common": [ + { + "id": "time-macros 0.2.18", + "target": "time_macros" + } + ], + "selects": {} + }, + "version": "0.3.36" + }, + "license": "MIT OR Apache-2.0" + }, + "time-core 0.1.2": { + "name": "time-core", + "version": "0.1.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/time-core/0.1.2/download", + "sha256": "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + } + }, + "targets": [ + { + "Library": { + "crate_name": "time_core", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "time_core", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "0.1.2" + }, + "license": "MIT OR Apache-2.0" + }, + "time-macros 0.2.18": { + "name": "time-macros", + "version": "0.2.18", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/time-macros/0.2.18/download", + "sha256": "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "time_macros", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "time_macros", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "formatting" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "num-conv 0.1.0", + "target": "num_conv" + }, + { + "id": "time-core 0.1.2", + "target": "time_core" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.2.18" + }, + "license": "MIT OR Apache-2.0" + }, + "typed-arena 2.0.2": { + "name": "typed-arena", + "version": "2.0.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/typed-arena/2.0.2/download", + "sha256": "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + } + }, + "targets": [ + { + "Library": { + "crate_name": "typed_arena", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "typed_arena", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "edition": "2015", + "version": "2.0.2" + }, + "license": "MIT" + }, + "typenum 1.17.0": { + "name": "typenum", + "version": "1.17.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/typenum/1.17.0/download", + "sha256": "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + } + }, + "targets": [ + { + "Library": { + "crate_name": "typenum", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_main", + "crate_root": "build/main.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "typenum", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "typenum 1.17.0", + "target": "build_script_main" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "1.17.0" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "ucd-trie 0.1.6": { + "name": "ucd-trie", + "version": "0.1.6", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/ucd-trie/0.1.6/download", + "sha256": "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + } + }, + "targets": [ + { + "Library": { + "crate_name": "ucd_trie", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "ucd_trie", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "std" + ], + "selects": {} + }, + "edition": "2021", + "version": "0.1.6" + }, + "license": "MIT OR Apache-2.0" + }, + "unicode-ident 1.0.12": { + "name": "unicode-ident", + "version": "1.0.12", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/unicode-ident/1.0.12/download", + "sha256": "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "unicode_ident", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "unicode_ident", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "1.0.12" + }, + "license": "(MIT OR Apache-2.0) AND Unicode-DFS-2016" + }, + "unicode-segmentation 1.11.0": { + "name": "unicode-segmentation", + "version": "1.11.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/unicode-segmentation/1.11.0/download", + "sha256": "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + } + }, + "targets": [ + { + "Library": { + "crate_name": "unicode_segmentation", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "unicode_segmentation", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "1.11.0" + }, + "license": "MIT/Apache-2.0" + }, + "unicode-width 0.1.13": { + "name": "unicode-width", + "version": "0.1.13", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/unicode-width/0.1.13/download", + "sha256": "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" + } + }, + "targets": [ + { + "Library": { + "crate_name": "unicode_width", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "unicode_width", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "edition": "2021", + "version": "0.1.13" + }, + "license": "MIT OR Apache-2.0" + }, + "uriparse 0.6.4": { + "name": "uriparse", + "version": "0.6.4", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/uriparse/0.6.4/download", + "sha256": "0200d0fc04d809396c2ad43f3c95da3582a2556eba8d453c1087f4120ee352ff" + } + }, + "targets": [ + { + "Library": { + "crate_name": "uriparse", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "uriparse", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "fnv 1.0.7", + "target": "fnv" + }, + { + "id": "lazy_static 1.4.0", + "target": "lazy_static" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.6.4" + }, + "license": "MIT" + }, + "version_check 0.9.4": { + "name": "version_check", + "version": "0.9.4", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/version_check/0.9.4/download", + "sha256": "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + } + }, + "targets": [ + { + "Library": { + "crate_name": "version_check", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "version_check", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "0.9.4" + }, + "license": "MIT/Apache-2.0" + }, + "wasi 0.11.0+wasi-snapshot-preview1": { + "name": "wasi", + "version": "0.11.0+wasi-snapshot-preview1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/wasi/0.11.0+wasi-snapshot-preview1/download", + "sha256": "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + } + }, + "targets": [ + { + "Library": { + "crate_name": "wasi", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "wasi", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "0.11.0+wasi-snapshot-preview1" + }, + "license": "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT" + }, + "wasm-bindgen 0.2.92": { + "name": "wasm-bindgen", + "version": "0.2.92", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/wasm-bindgen/0.2.92/download", + "sha256": "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "wasm_bindgen", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "wasm_bindgen", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "spans", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + }, + { + "id": "wasm-bindgen 0.2.92", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "proc_macro_deps": { + "common": [ + { + "id": "wasm-bindgen-macro 0.2.92", + "target": "wasm_bindgen_macro" + } + ], + "selects": {} + }, + "version": "0.2.92" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "wasm-bindgen-backend 0.2.92": { + "name": "wasm-bindgen-backend", + "version": "0.2.92", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/wasm-bindgen-backend/0.2.92/download", + "sha256": "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" + } + }, + "targets": [ + { + "Library": { + "crate_name": "wasm_bindgen_backend", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "wasm_bindgen_backend", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "spans" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "bumpalo 3.16.0", + "target": "bumpalo" + }, + { + "id": "log 0.4.21", + "target": "log" + }, + { + "id": "once_cell 1.19.0", + "target": "once_cell" + }, + { + "id": "proc-macro2 1.0.85", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.36", + "target": "quote" + }, + { + "id": "syn 2.0.66", + "target": "syn" + }, + { + "id": "wasm-bindgen-shared 0.2.92", + "target": "wasm_bindgen_shared" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.2.92" + }, + "license": "MIT OR Apache-2.0" + }, + "wasm-bindgen-futures 0.4.42": { + "name": "wasm-bindgen-futures", + "version": "0.4.42", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/wasm-bindgen-futures/0.4.42/download", + "sha256": "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" + } + }, + "targets": [ + { + "Library": { + "crate_name": "wasm_bindgen_futures", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "wasm_bindgen_futures", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + }, + { + "id": "js-sys 0.3.69", + "target": "js_sys" + }, + { + "id": "wasm-bindgen 0.2.92", + "target": "wasm_bindgen" + } + ], + "selects": { + "cfg(target_feature = \"atomics\")": [ + { + "id": "web-sys 0.3.69", + "target": "web_sys" + } + ] + } + }, + "edition": "2018", + "version": "0.4.42" + }, + "license": "MIT OR Apache-2.0" + }, + "wasm-bindgen-macro 0.2.92": { + "name": "wasm-bindgen-macro", + "version": "0.2.92", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/wasm-bindgen-macro/0.2.92/download", + "sha256": "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "wasm_bindgen_macro", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "wasm_bindgen_macro", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "spans" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "quote 1.0.36", + "target": "quote" + }, + { + "id": "wasm-bindgen-macro-support 0.2.92", + "target": "wasm_bindgen_macro_support" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.2.92" + }, + "license": "MIT OR Apache-2.0" + }, + "wasm-bindgen-macro-support 0.2.92": { + "name": "wasm-bindgen-macro-support", + "version": "0.2.92", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/wasm-bindgen-macro-support/0.2.92/download", + "sha256": "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" + } + }, + "targets": [ + { + "Library": { + "crate_name": "wasm_bindgen_macro_support", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "wasm_bindgen_macro_support", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "spans" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "proc-macro2 1.0.85", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.36", + "target": "quote" + }, + { + "id": "syn 2.0.66", + "target": "syn" + }, + { + "id": "wasm-bindgen-backend 0.2.92", + "target": "wasm_bindgen_backend" + }, + { + "id": "wasm-bindgen-shared 0.2.92", + "target": "wasm_bindgen_shared" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.2.92" + }, + "license": "MIT OR Apache-2.0" + }, + "wasm-bindgen-shared 0.2.92": { + "name": "wasm-bindgen-shared", + "version": "0.2.92", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/wasm-bindgen-shared/0.2.92/download", + "sha256": "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + } + }, + "targets": [ + { + "Library": { + "crate_name": "wasm_bindgen_shared", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "wasm_bindgen_shared", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "wasm-bindgen-shared 0.2.92", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.2.92" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "links": "wasm_bindgen" + }, + "license": "MIT OR Apache-2.0" + }, + "wasm-bindgen-test 0.3.42": { + "name": "wasm-bindgen-test", + "version": "0.3.42", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/wasm-bindgen-test/0.3.42/download", + "sha256": "d9bf62a58e0780af3e852044583deee40983e5886da43a271dd772379987667b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "wasm_bindgen_test", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "wasm_bindgen_test", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "console_error_panic_hook 0.1.7", + "target": "console_error_panic_hook" + }, + { + "id": "js-sys 0.3.69", + "target": "js_sys" + }, + { + "id": "scoped-tls 1.0.1", + "target": "scoped_tls" + }, + { + "id": "wasm-bindgen 0.2.92", + "target": "wasm_bindgen" + }, + { + "id": "wasm-bindgen-futures 0.4.42", + "target": "wasm_bindgen_futures" + } + ], + "selects": {} + }, + "edition": "2018", + "proc_macro_deps": { + "common": [ + { + "id": "wasm-bindgen-test-macro 0.3.42", + "target": "wasm_bindgen_test_macro" + } + ], + "selects": {} + }, + "version": "0.3.42" + }, + "license": "MIT OR Apache-2.0" + }, + "wasm-bindgen-test-macro 0.3.42": { + "name": "wasm-bindgen-test-macro", + "version": "0.3.42", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/wasm-bindgen-test-macro/0.3.42/download", + "sha256": "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "wasm_bindgen_test_macro", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "wasm_bindgen_test_macro", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "proc-macro2 1.0.85", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.36", + "target": "quote" + }, + { + "id": "syn 2.0.66", + "target": "syn" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.3.42" + }, + "license": "MIT OR Apache-2.0" + }, + "web-sys 0.3.69": { + "name": "web-sys", + "version": "0.3.69", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/web-sys/0.3.69/download", + "sha256": "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" + } + }, + "targets": [ + { + "Library": { + "crate_name": "web_sys", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "web_sys", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "js-sys 0.3.69", + "target": "js_sys" + }, + { + "id": "wasm-bindgen 0.2.92", + "target": "wasm_bindgen" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.3.69" + }, + "license": "MIT OR Apache-2.0" + }, + "winapi 0.3.9": { + "name": "winapi", + "version": "0.3.9", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/winapi/0.3.9/download", + "sha256": "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" + } + }, + "targets": [ + { + "Library": { + "crate_name": "winapi", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "winapi", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "consoleapi", + "handleapi", + "impl-default", + "minwinbase", + "minwindef", + "processenv", + "synchapi", + "winbase", + "winerror", + "winuser" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "winapi 0.3.9", + "target": "build_script_build" + } + ], + "selects": { + "i686-pc-windows-gnu": [ + { + "id": "winapi-i686-pc-windows-gnu 0.4.0", + "target": "winapi_i686_pc_windows_gnu" + } + ], + "x86_64-pc-windows-gnu": [ + { + "id": "winapi-x86_64-pc-windows-gnu 0.4.0", + "target": "winapi_x86_64_pc_windows_gnu" + } + ] + } + }, + "edition": "2015", + "version": "0.3.9" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT/Apache-2.0" + }, + "winapi-i686-pc-windows-gnu 0.4.0": { + "name": "winapi-i686-pc-windows-gnu", + "version": "0.4.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/winapi-i686-pc-windows-gnu/0.4.0/download", + "sha256": "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + } + }, + "targets": [ + { + "Library": { + "crate_name": "winapi_i686_pc_windows_gnu", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "winapi_i686_pc_windows_gnu", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "winapi-i686-pc-windows-gnu 0.4.0", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.4.0" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT/Apache-2.0" + }, + "winapi-util 0.1.8": { + "name": "winapi-util", + "version": "0.1.8", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/winapi-util/0.1.8/download", + "sha256": "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "winapi_util", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "winapi_util", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [], + "selects": { + "cfg(windows)": [ + { + "id": "windows-sys 0.52.0", + "target": "windows_sys" + } + ] + } + }, + "edition": "2021", + "version": "0.1.8" + }, + "license": "Unlicense OR MIT" + }, + "winapi-x86_64-pc-windows-gnu 0.4.0": { + "name": "winapi-x86_64-pc-windows-gnu", + "version": "0.4.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/winapi-x86_64-pc-windows-gnu/0.4.0/download", + "sha256": "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + } + }, + "targets": [ + { + "Library": { + "crate_name": "winapi_x86_64_pc_windows_gnu", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "winapi_x86_64_pc_windows_gnu", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "winapi-x86_64-pc-windows-gnu 0.4.0", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.4.0" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT/Apache-2.0" + }, + "windows-core 0.52.0": { + "name": "windows-core", + "version": "0.52.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows-core/0.52.0/download", + "sha256": "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_core", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_core", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows-targets 0.52.5", + "target": "windows_targets" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.0" + }, + "license": "MIT OR Apache-2.0" + }, + "windows-sys 0.48.0": { + "name": "windows-sys", + "version": "0.48.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows-sys/0.48.0/download", + "sha256": "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_sys", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_sys", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows-targets 0.48.5", + "target": "windows_targets" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.48.0" + }, + "license": "MIT OR Apache-2.0" + }, + "windows-sys 0.52.0": { + "name": "windows-sys", + "version": "0.52.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows-sys/0.52.0/download", + "sha256": "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_sys", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_sys", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "Win32", + "Win32_Foundation", + "Win32_Storage", + "Win32_Storage_FileSystem", + "Win32_System", + "Win32_System_Console", + "Win32_System_SystemInformation", + "default" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "windows-targets 0.52.5", + "target": "windows_targets" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.0" + }, + "license": "MIT OR Apache-2.0" + }, + "windows-targets 0.48.5": { + "name": "windows-targets", + "version": "0.48.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows-targets/0.48.5/download", + "sha256": "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_targets", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_targets", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [], + "selects": { + "aarch64-pc-windows-gnullvm": [ + { + "id": "windows_aarch64_gnullvm 0.48.5", + "target": "windows_aarch64_gnullvm" + } + ], + "cfg(all(target_arch = \"aarch64\", target_env = \"msvc\", not(windows_raw_dylib)))": [ + { + "id": "windows_aarch64_msvc 0.48.5", + "target": "windows_aarch64_msvc" + } + ], + "cfg(all(target_arch = \"x86\", target_env = \"gnu\", not(windows_raw_dylib)))": [ + { + "id": "windows_i686_gnu 0.48.5", + "target": "windows_i686_gnu" + } + ], + "cfg(all(target_arch = \"x86\", target_env = \"msvc\", not(windows_raw_dylib)))": [ + { + "id": "windows_i686_msvc 0.48.5", + "target": "windows_i686_msvc" + } + ], + "cfg(all(target_arch = \"x86_64\", target_env = \"gnu\", not(target_abi = \"llvm\"), not(windows_raw_dylib)))": [ + { + "id": "windows_x86_64_gnu 0.48.5", + "target": "windows_x86_64_gnu" + } + ], + "cfg(all(target_arch = \"x86_64\", target_env = \"msvc\", not(windows_raw_dylib)))": [ + { + "id": "windows_x86_64_msvc 0.48.5", + "target": "windows_x86_64_msvc" + } + ], + "x86_64-pc-windows-gnullvm": [ + { + "id": "windows_x86_64_gnullvm 0.48.5", + "target": "windows_x86_64_gnullvm" + } + ] + } + }, + "edition": "2018", + "version": "0.48.5" + }, + "license": "MIT OR Apache-2.0" + }, + "windows-targets 0.52.5": { + "name": "windows-targets", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows-targets/0.52.5/download", + "sha256": "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_targets", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_targets", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [], + "selects": { + "aarch64-pc-windows-gnullvm": [ + { + "id": "windows_aarch64_gnullvm 0.52.5", + "target": "windows_aarch64_gnullvm" + } + ], + "cfg(all(any(target_arch = \"x86_64\", target_arch = \"arm64ec\"), target_env = \"msvc\", not(windows_raw_dylib)))": [ + { + "id": "windows_x86_64_msvc 0.52.5", + "target": "windows_x86_64_msvc" + } + ], + "cfg(all(target_arch = \"aarch64\", target_env = \"msvc\", not(windows_raw_dylib)))": [ + { + "id": "windows_aarch64_msvc 0.52.5", + "target": "windows_aarch64_msvc" + } + ], + "cfg(all(target_arch = \"x86\", target_env = \"gnu\", not(target_abi = \"llvm\"), not(windows_raw_dylib)))": [ + { + "id": "windows_i686_gnu 0.52.5", + "target": "windows_i686_gnu" + } + ], + "cfg(all(target_arch = \"x86\", target_env = \"msvc\", not(windows_raw_dylib)))": [ + { + "id": "windows_i686_msvc 0.52.5", + "target": "windows_i686_msvc" + } + ], + "cfg(all(target_arch = \"x86_64\", target_env = \"gnu\", not(target_abi = \"llvm\"), not(windows_raw_dylib)))": [ + { + "id": "windows_x86_64_gnu 0.52.5", + "target": "windows_x86_64_gnu" + } + ], + "i686-pc-windows-gnullvm": [ + { + "id": "windows_i686_gnullvm 0.52.5", + "target": "windows_i686_gnullvm" + } + ], + "x86_64-pc-windows-gnullvm": [ + { + "id": "windows_x86_64_gnullvm 0.52.5", + "target": "windows_x86_64_gnullvm" + } + ] + } + }, + "edition": "2021", + "version": "0.52.5" + }, + "license": "MIT OR Apache-2.0" + }, + "windows_aarch64_gnullvm 0.48.5": { + "name": "windows_aarch64_gnullvm", + "version": "0.48.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_aarch64_gnullvm/0.48.5/download", + "sha256": "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_aarch64_gnullvm", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_aarch64_gnullvm", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_aarch64_gnullvm 0.48.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.48.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_aarch64_gnullvm 0.52.5": { + "name": "windows_aarch64_gnullvm", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_aarch64_gnullvm/0.52.5/download", + "sha256": "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_aarch64_gnullvm", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_aarch64_gnullvm", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_aarch64_gnullvm 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_aarch64_msvc 0.48.5": { + "name": "windows_aarch64_msvc", + "version": "0.48.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_aarch64_msvc/0.48.5/download", + "sha256": "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_aarch64_msvc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_aarch64_msvc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_aarch64_msvc 0.48.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.48.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_aarch64_msvc 0.52.5": { + "name": "windows_aarch64_msvc", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_aarch64_msvc/0.52.5/download", + "sha256": "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_aarch64_msvc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_aarch64_msvc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_aarch64_msvc 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_i686_gnu 0.48.5": { + "name": "windows_i686_gnu", + "version": "0.48.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_i686_gnu/0.48.5/download", + "sha256": "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_i686_gnu", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_i686_gnu", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_i686_gnu 0.48.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.48.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_i686_gnu 0.52.5": { + "name": "windows_i686_gnu", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_i686_gnu/0.52.5/download", + "sha256": "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_i686_gnu", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_i686_gnu", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_i686_gnu 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_i686_gnullvm 0.52.5": { + "name": "windows_i686_gnullvm", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_i686_gnullvm/0.52.5/download", + "sha256": "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_i686_gnullvm", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_i686_gnullvm", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_i686_gnullvm 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_i686_msvc 0.48.5": { + "name": "windows_i686_msvc", + "version": "0.48.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_i686_msvc/0.48.5/download", + "sha256": "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_i686_msvc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_i686_msvc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_i686_msvc 0.48.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.48.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_i686_msvc 0.52.5": { + "name": "windows_i686_msvc", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_i686_msvc/0.52.5/download", + "sha256": "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_i686_msvc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_i686_msvc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_i686_msvc 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_x86_64_gnu 0.48.5": { + "name": "windows_x86_64_gnu", + "version": "0.48.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_x86_64_gnu/0.48.5/download", + "sha256": "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_x86_64_gnu", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_x86_64_gnu", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_x86_64_gnu 0.48.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.48.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_x86_64_gnu 0.52.5": { + "name": "windows_x86_64_gnu", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_x86_64_gnu/0.52.5/download", + "sha256": "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_x86_64_gnu", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_x86_64_gnu", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_x86_64_gnu 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_x86_64_gnullvm 0.48.5": { + "name": "windows_x86_64_gnullvm", + "version": "0.48.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_x86_64_gnullvm/0.48.5/download", + "sha256": "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_x86_64_gnullvm", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_x86_64_gnullvm", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_x86_64_gnullvm 0.48.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.48.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_x86_64_gnullvm 0.52.5": { + "name": "windows_x86_64_gnullvm", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_x86_64_gnullvm/0.52.5/download", + "sha256": "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_x86_64_gnullvm", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_x86_64_gnullvm", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_x86_64_gnullvm 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_x86_64_msvc 0.48.5": { + "name": "windows_x86_64_msvc", + "version": "0.48.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_x86_64_msvc/0.48.5/download", + "sha256": "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_x86_64_msvc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_x86_64_msvc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_x86_64_msvc 0.48.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.48.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_x86_64_msvc 0.52.5": { + "name": "windows_x86_64_msvc", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_x86_64_msvc/0.52.5/download", + "sha256": "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_x86_64_msvc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_x86_64_msvc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_x86_64_msvc 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "yansi 0.5.1": { + "name": "yansi", + "version": "0.5.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/yansi/0.5.1/download", + "sha256": "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + } + }, + "targets": [ + { + "Library": { + "crate_name": "yansi", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "yansi", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "0.5.1" + }, + "license": "MIT/Apache-2.0" + } + }, + "binary_crates": [], + "workspace_members": { + "cddl 0.9.4": "third_party" + }, + "conditions": { + "aarch64-apple-darwin": [ + "aarch64-apple-darwin" + ], + "aarch64-apple-ios": [ + "aarch64-apple-ios" + ], + "aarch64-apple-ios-sim": [ + "aarch64-apple-ios-sim" + ], + "aarch64-fuchsia": [ + "aarch64-fuchsia" + ], + "aarch64-linux-android": [ + "aarch64-linux-android" + ], + "aarch64-pc-windows-gnullvm": [], + "aarch64-pc-windows-msvc": [ + "aarch64-pc-windows-msvc" + ], + "aarch64-unknown-linux-gnu": [ + "aarch64-unknown-linux-gnu" + ], + "arm-unknown-linux-gnueabi": [ + "arm-unknown-linux-gnueabi" + ], + "armv7-linux-androideabi": [ + "armv7-linux-androideabi" + ], + "armv7-unknown-linux-gnueabi": [ + "armv7-unknown-linux-gnueabi" + ], + "cfg(all(any(target_arch = \"x86_64\", target_arch = \"arm64ec\"), target_env = \"msvc\", not(windows_raw_dylib)))": [ + "x86_64-pc-windows-msvc" + ], + "cfg(all(target_arch = \"aarch64\", target_env = \"msvc\", not(windows_raw_dylib)))": [ + "aarch64-pc-windows-msvc" + ], + "cfg(all(target_arch = \"aarch64\", target_os = \"linux\"))": [ + "aarch64-unknown-linux-gnu" + ], + "cfg(all(target_arch = \"aarch64\", target_vendor = \"apple\"))": [ + "aarch64-apple-darwin", + "aarch64-apple-ios", + "aarch64-apple-ios-sim" + ], + "cfg(all(target_arch = \"loongarch64\", target_os = \"linux\"))": [], + "cfg(all(target_arch = \"wasm32\", not(any(target_os = \"emscripten\", target_os = \"wasi\"))))": [ + "wasm32-unknown-unknown" + ], + "cfg(all(target_arch = \"x86\", target_env = \"gnu\", not(target_abi = \"llvm\"), not(windows_raw_dylib)))": [ + "i686-unknown-linux-gnu" + ], + "cfg(all(target_arch = \"x86\", target_env = \"gnu\", not(windows_raw_dylib)))": [ + "i686-unknown-linux-gnu" + ], + "cfg(all(target_arch = \"x86\", target_env = \"msvc\", not(windows_raw_dylib)))": [ + "i686-pc-windows-msvc" + ], + "cfg(all(target_arch = \"x86_64\", target_env = \"gnu\", not(target_abi = \"llvm\"), not(windows_raw_dylib)))": [ + "x86_64-unknown-linux-gnu" + ], + "cfg(all(target_arch = \"x86_64\", target_env = \"msvc\", not(windows_raw_dylib)))": [ + "x86_64-pc-windows-msvc" + ], + "cfg(any(target_arch = \"aarch64\", target_arch = \"x86_64\", target_arch = \"x86\"))": [ + "aarch64-apple-darwin", + "aarch64-apple-ios", + "aarch64-apple-ios-sim", + "aarch64-fuchsia", + "aarch64-linux-android", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-gnu", + "i686-apple-darwin", + "i686-linux-android", + "i686-pc-windows-msvc", + "i686-unknown-freebsd", + "i686-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-apple-ios", + "x86_64-fuchsia", + "x86_64-linux-android", + "x86_64-pc-windows-msvc", + "x86_64-unknown-freebsd", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-none" + ], + "cfg(any(target_os = \"macos\", target_os = \"ios\"))": [ + "aarch64-apple-darwin", + "aarch64-apple-ios", + "aarch64-apple-ios-sim", + "i686-apple-darwin", + "x86_64-apple-darwin", + "x86_64-apple-ios" + ], + "cfg(any(target_os = \"macos\", target_os = \"ios\", target_os = \"freebsd\"))": [ + "aarch64-apple-darwin", + "aarch64-apple-ios", + "aarch64-apple-ios-sim", + "i686-apple-darwin", + "i686-unknown-freebsd", + "x86_64-apple-darwin", + "x86_64-apple-ios", + "x86_64-unknown-freebsd" + ], + "cfg(not(target_arch = \"wasm32\"))": [ + "aarch64-apple-darwin", + "aarch64-apple-ios", + "aarch64-apple-ios-sim", + "aarch64-fuchsia", + "aarch64-linux-android", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-gnu", + "arm-unknown-linux-gnueabi", + "armv7-linux-androideabi", + "armv7-unknown-linux-gnueabi", + "i686-apple-darwin", + "i686-linux-android", + "i686-pc-windows-msvc", + "i686-unknown-freebsd", + "i686-unknown-linux-gnu", + "powerpc-unknown-linux-gnu", + "riscv32imc-unknown-none-elf", + "riscv64gc-unknown-none-elf", + "s390x-unknown-linux-gnu", + "thumbv7em-none-eabi", + "thumbv8m.main-none-eabi", + "x86_64-apple-darwin", + "x86_64-apple-ios", + "x86_64-fuchsia", + "x86_64-linux-android", + "x86_64-pc-windows-msvc", + "x86_64-unknown-freebsd", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-none" + ], + "cfg(target_arch = \"spirv\")": [], + "cfg(target_arch = \"wasm32\")": [ + "wasm32-unknown-unknown", + "wasm32-wasi" + ], + "cfg(target_family = \"unix\")": [ + "aarch64-apple-darwin", + "aarch64-apple-ios", + "aarch64-apple-ios-sim", + "aarch64-fuchsia", + "aarch64-linux-android", + "aarch64-unknown-linux-gnu", + "arm-unknown-linux-gnueabi", + "armv7-linux-androideabi", + "armv7-unknown-linux-gnueabi", + "i686-apple-darwin", + "i686-linux-android", + "i686-unknown-freebsd", + "i686-unknown-linux-gnu", + "powerpc-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-apple-ios", + "x86_64-fuchsia", + "x86_64-linux-android", + "x86_64-unknown-freebsd", + "x86_64-unknown-linux-gnu" + ], + "cfg(target_feature = \"atomics\")": [], + "cfg(target_os = \"android\")": [ + "aarch64-linux-android", + "armv7-linux-androideabi", + "i686-linux-android", + "x86_64-linux-android" + ], + "cfg(target_os = \"haiku\")": [], + "cfg(target_os = \"hermit\")": [], + "cfg(target_os = \"redox\")": [], + "cfg(target_os = \"wasi\")": [ + "wasm32-wasi" + ], + "cfg(target_os = \"windows\")": [ + "aarch64-pc-windows-msvc", + "i686-pc-windows-msvc", + "x86_64-pc-windows-msvc" + ], + "cfg(unix)": [ + "aarch64-apple-darwin", + "aarch64-apple-ios", + "aarch64-apple-ios-sim", + "aarch64-fuchsia", + "aarch64-linux-android", + "aarch64-unknown-linux-gnu", + "arm-unknown-linux-gnueabi", + "armv7-linux-androideabi", + "armv7-unknown-linux-gnueabi", + "i686-apple-darwin", + "i686-linux-android", + "i686-unknown-freebsd", + "i686-unknown-linux-gnu", + "powerpc-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-apple-ios", + "x86_64-fuchsia", + "x86_64-linux-android", + "x86_64-unknown-freebsd", + "x86_64-unknown-linux-gnu" + ], + "cfg(windows)": [ + "aarch64-pc-windows-msvc", + "i686-pc-windows-msvc", + "x86_64-pc-windows-msvc" + ], + "i686-apple-darwin": [ + "i686-apple-darwin" + ], + "i686-linux-android": [ + "i686-linux-android" + ], + "i686-pc-windows-gnu": [], + "i686-pc-windows-gnullvm": [], + "i686-pc-windows-msvc": [ + "i686-pc-windows-msvc" + ], + "i686-unknown-freebsd": [ + "i686-unknown-freebsd" + ], + "i686-unknown-linux-gnu": [ + "i686-unknown-linux-gnu" + ], + "powerpc-unknown-linux-gnu": [ + "powerpc-unknown-linux-gnu" + ], + "riscv32imc-unknown-none-elf": [ + "riscv32imc-unknown-none-elf" + ], + "riscv64gc-unknown-none-elf": [ + "riscv64gc-unknown-none-elf" + ], + "s390x-unknown-linux-gnu": [ + "s390x-unknown-linux-gnu" + ], + "thumbv7em-none-eabi": [ + "thumbv7em-none-eabi" + ], + "thumbv8m.main-none-eabi": [ + "thumbv8m.main-none-eabi" + ], + "wasm32-unknown-unknown": [ + "wasm32-unknown-unknown" + ], + "wasm32-wasi": [ + "wasm32-wasi" + ], + "x86_64-apple-darwin": [ + "x86_64-apple-darwin" + ], + "x86_64-apple-ios": [ + "x86_64-apple-ios" + ], + "x86_64-fuchsia": [ + "x86_64-fuchsia" + ], + "x86_64-linux-android": [ + "x86_64-linux-android" + ], + "x86_64-pc-windows-gnu": [], + "x86_64-pc-windows-gnullvm": [], + "x86_64-pc-windows-msvc": [ + "x86_64-pc-windows-msvc" + ], + "x86_64-unknown-freebsd": [ + "x86_64-unknown-freebsd" + ], + "x86_64-unknown-linux-gnu": [ + "x86_64-unknown-linux-gnu" + ], + "x86_64-unknown-none": [ + "x86_64-unknown-none" + ] + } +} diff --git a/third_party/cddl/cddl.BUILD b/third_party/cddl/cddl.BUILD new file mode 100644 index 00000000..832e87f6 --- /dev/null +++ b/third_party/cddl/cddl.BUILD @@ -0,0 +1,57 @@ +load("@cddl_crate_index//:defs.bzl", cddl_aliases = "aliases", cddl_all_create_deps = "all_crate_deps") +load("@rules_rust//rust:defs.bzl", "rust_library", "rust_shared_library") + +package(default_visibility = ["//visibility:private"]) + +licenses(["notice"]) + +exports_files(["LICENSE"]) + +rust_library( + name = "cddl_src_lib", + srcs = glob([ + "src/*.rs", + "src/**/*.rs", + ]), + aliases = cddl_aliases(), + crate_features = [ + "std", + "ast-span", + "ast-comments", + "json", + "cbor", + "additional-controls", + "ast-parent", + ], + proc_macro_deps = cddl_all_create_deps( + package_name = "third_party", + proc_macro = True, + ), + visibility = ["//visibility:public"], + deps = cddl_all_create_deps(package_name = "third_party"), +) + +rust_shared_library( + name = "cddl", + srcs = glob([ + "src/lib.rs", + "src/**/*.rs", + ]), + crate_features = [ + "std", + "ast-span", + "ast-comments", + "json", + "cbor", + "additional-controls", + "ast-parent", + ], + proc_macro_deps = cddl_all_create_deps( + package_name = "third_party", + proc_macro = True, + ), + visibility = ["//visibility:public"], + deps = cddl_all_create_deps(package_name = "third_party") + [ + ":cddl_src_lib", + ], +) diff --git a/third_party/cddl/cddl.h b/third_party/cddl/cddl.h new file mode 100644 index 00000000..4a0f7537 --- /dev/null +++ b/third_party/cddl/cddl.h @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CDDL_H +#define CDDL_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +bool validate_json_str(const char* cddl, const char* json); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif /* CDDL_H */ diff --git a/third_party/cddl/cddl.patch b/third_party/cddl/cddl.patch new file mode 100644 index 00000000..aefa5e47 --- /dev/null +++ b/third_party/cddl/cddl.patch @@ -0,0 +1,60 @@ +From ed9dc15297502ef9d0aeb89d2514e03217e9776a Mon Sep 17 00:00:00 2001 +From: Salman +Date: Thu, 6 Jun 2024 23:01:57 +0000 +Subject: [PATCH] wrapper + +--- + src/lib.rs | 33 +++++++++++++++++++++++++++------ + 1 file changed, 27 insertions(+), 6 deletions(-) + +diff --git a/src/lib.rs b/src/lib.rs +index f55a41c..8b6ded5 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -541,6 +541,9 @@ extern crate uriparse; + #[cfg(feature = "std")] + extern crate base64_url; + ++#[cfg(feature = "std")] ++extern crate libc; ++ + /// Abstract syntax tree representing a CDDL definition + pub mod ast; + /// Static error messages +@@ -575,9 +578,27 @@ pub use self::{ + #[cfg(not(target_arch = "wasm32"))] + pub use self::validator::validate_cbor_from_slice; + +-#[doc(inline)] +-#[cfg(feature = "std")] +-#[cfg(feature = "json")] +-#[cfg(not(feature = "lsp"))] +-#[cfg(not(target_arch = "wasm32"))] +-pub use self::validator::validate_json_from_str; ++ ++use libc::c_char; ++use std::ffi::CStr; ++ ++#[no_mangle] ++/// Validate JSON string from a given CDDL document string ++pub extern fn validate_json_str(cddl: *const c_char, json: *const c_char) -> bool { ++ #[cfg(feature = "std")] ++ #[cfg(feature = "json")] ++ #[cfg(not(feature = "lsp"))] ++ #[cfg(not(target_arch = "wasm32"))] ++ use self::validator::validate_json_from_str; ++ ++ let cddl_str: &CStr = unsafe { CStr::from_ptr(cddl) }; ++ let cddl_str_slice : &str = cddl_str.to_str().unwrap(); ++ let json_str: &CStr = unsafe { CStr::from_ptr(json) }; ++ let json_str_slice : &str = json_str.to_str().unwrap(); ++ ++ #[cfg(feature = "std")] ++ #[cfg(feature = "json")] ++ #[cfg(not(feature = "lsp"))] ++ #[cfg(not(target_arch = "wasm32"))] ++ return validate_json_from_str(cddl_str_slice, json_str_slice, Some(&["json"])).is_ok(); ++} +-- +2.45.2.505.gda0bf45e8d-goog + diff --git a/third_party/deps.bzl b/third_party/deps.bzl new file mode 100644 index 00000000..0abf8a93 --- /dev/null +++ b/third_party/deps.bzl @@ -0,0 +1,28 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#  http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_rust//crate_universe:defs.bzl", "crates_repository") + +def deps(): + # repin deps using: + # EXTRA_DOCKER_RUN_ARGS="--env=CARGO_BAZEL_REPIN=1" builders/tools/bazel-debian sync --only=cddl_crate_index + crates_repository( + name = "cddl_crate_index", + quiet = False, + cargo_lockfile = Label("cddl/Cargo.lock"), + lockfile = Label("cddl/cargo-bazel-lock.json"), + manifests = [ + Label("cddl/Cargo.toml"), + ], + ) diff --git a/tools/debug/start_auction b/tools/debug/start_auction index 1830a2e7..d196e9f3 100755 --- a/tools/debug/start_auction +++ b/tools/debug/start_auction @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Copyright 2024 Google LLC # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -34,18 +35,17 @@ export SERVER_START_CMD=$(cat << END --seller_code_fetch_config='{ "fetchMode": 0, "auctionJsPath": "", - "auctionJsUrl": "https://pubads.g.doubleclick.net/td/sjs", + "auctionJsUrl": "${AUCTION_JS_URL}", "urlFetchPeriodMs": 13000000, "urlFetchTimeoutMs": 30000, "enableSellerDebugUrlGeneration": true, - "enableAdtechCodeLogging": false, "enableReportResultUrlGeneration": true, "enableReportWinUrlGeneration": true, - "buyerReportWinJsUrls": {"https://td.doubleclick.net":"https://td.doubleclick.net/td/bjs"}, - "protectedAppSignalsBuyerReportWinJsUrls": {"https://td.doubleclick.net":"https://storage.googleapis.com/wasm-explorer/PAS/PASreportWin.js"} + "buyerReportWinJsUrls": {"${BUYER_REPORT_WIN_URL}":"${BUYER_REPORT_WIN_SCRIPT}"}, + "protectedAppSignalsBuyerReportWinJsUrls": {"${BUYER_REPORT_WIN_URL}":"https://storage.googleapis.com/wasm-explorer/PAS/PASreportWin.js"} }' \ ---enable_otel_based_logging="false" \ ---consented_debug_token="" \ +--enable_otel_based_logging="true"\ +--consented_debug_token="test_token"\ --ps_verbosity=2 \ --enable_protected_audience="true" \ --auction_tcmalloc_background_release_rate_bytes_per_second=4096 \ diff --git a/tools/debug/start_bfe b/tools/debug/start_bfe index 54dd13b6..a63288b1 100755 --- a/tools/debug/start_bfe +++ b/tools/debug/start_bfe @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Copyright 2024 Google LLC # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,15 +25,15 @@ export SERVER_START_CMD=$(cat << END /server \ --init_config_client="false" --port=50051 \ --bidding_server_addr="127.0.0.1:50057" \ ---buyer_kv_server_addr="https://googleads.g.doubleclick.net/td/bts" \ +--buyer_kv_server_addr="${BUYER_KV_SERVER_ADDR}" \ --enable_buyer_frontend_benchmarking="true" \ --generate_bid_timeout_ms=60000 --bidding_signals_load_timeout_ms 60000 \ --protected_app_signals_generate_bid_timeout_ms=60000 \ --create_new_event_engine="false" \ --test_mode="true" \ --telemetry_config="mode: EXPERIMENT" \ ---enable_otel_based_logging="false" \ ---consented_debug_token="" \ +--enable_otel_based_logging="true"\ +--consented_debug_token="test_token"\ --ps_verbosity=2 \ --enable_protected_audience="true" \ --bfe_tcmalloc_background_release_rate_bytes_per_second=4096 \ diff --git a/tools/debug/start_bidding b/tools/debug/start_bidding index d5525d46..53437b94 100755 --- a/tools/debug/start_bidding +++ b/tools/debug/start_bidding @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Copyright 2024 Google LLC # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,6 +18,7 @@ export PROJECT_ROOT=$(git rev-parse --show-toplevel) source $(dirname "$0")/common declare -a -r DOCKER_RUN_ARGS=( "--volume=${PROJECT_ROOT}/bazel-bin/services/bidding_service/server:/server" + "--volume=${PROJECT_ROOT}/services/bidding_service/egress_cddl_spec:/egress_cddl_spec" "--name=bidding" ) export EXTRA_DOCKER_RUN_ARGS="${COMMON_DOCKER_RUN_ARGS[@]} ${DOCKER_RUN_ARGS[@]}" @@ -35,19 +37,18 @@ export SERVER_START_CMD=$(cat << END --buyer_code_fetch_config='{ "fetchMode": 0, "biddingJsPath": "", - "biddingJsUrl": "https://td.doubleclick.net/td/bjs", - "protectedAppSignalsBiddingJsUrl": "https://td.doubleclick.net/td/bjs", + "biddingJsUrl": "${BIDDING_JS_URL}", + "protectedAppSignalsBiddingJsUrl": "${BIDDING_JS_URL}", "biddingWasmHelperUrl": "", "protectedAppSignalsBiddingWasmHelperUrl": "", "urlFetchPeriodMs": 13000000, "urlFetchTimeoutMs": 30000, "enableBuyerDebugUrlGeneration": true, - "enableAdtechCodeLogging": false, "prepareDataForAdsRetrievalJsUrl": "", "prepareDataForAdsRetrievalWasmHelperUrl": "" }' \ ---enable_otel_based_logging="false" \ ---consented_debug_token="" \ +--enable_otel_based_logging="true"\ +--consented_debug_token="test_token"\ --enable_protected_app_signals="false" \ --enable_protected_audience="true" \ --tee_ad_retrieval_kv_server_addr="localhost:50057" \ diff --git a/tools/debug/start_sfe b/tools/debug/start_sfe index cae62a45..d455a117 100755 --- a/tools/debug/start_sfe +++ b/tools/debug/start_sfe @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Copyright 2024 Google LLC # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,16 +21,15 @@ declare -a -r DOCKER_RUN_ARGS=( ) export EXTRA_DOCKER_RUN_ARGS="${COMMON_DOCKER_RUN_ARGS[@]} ${DOCKER_RUN_ARGS[@]}" - export SERVER_START_CMD=$(cat << END /server \ --init_config_client="false" --port=50053 \ --auction_server_host="127.0.0.1:50061" \ ---key_value_signals_host="https://pubads.g.doubleclick.net/td/sts" \ ---seller_origin_domain="https://securepubads.g.doubleclick.net" \ +--key_value_signals_host="${KEY_VALUE_SIGNALS_HOST}" \ +--seller_origin_domain="${SELLER_ORIGIN_DOMAIN}" \ --seller_cloud_platforms_map='{"component-seller1.com":"GCP", "component-seller2.com":"AWS"}' \ --buyer_server_hosts='{ - "https://td.doubleclick.net": { + "${BUYER_SERVER_HOST}": { "url": "127.0.0.1:50051", "cloudPlatform": "LOCAL" } @@ -40,8 +40,8 @@ export SERVER_START_CMD=$(cat << END --create_new_event_engine="false" \ --test_mode="true" \ --telemetry_config="mode: EXPERIMENT" \ ---enable_otel_based_logging="false" \ ---consented_debug_token="" \ +--enable_otel_based_logging="true"\ +--consented_debug_token="test_token"\ --ps_verbosity=2 \ --enable_protected_audience="true" \ --sfe_tcmalloc_background_release_rate_bytes_per_second=4096 \ diff --git a/tools/secure_invoke/secure_invoke_lib.cc b/tools/secure_invoke/secure_invoke_lib.cc index b72de3b9..8ce13479 100644 --- a/tools/secure_invoke/secure_invoke_lib.cc +++ b/tools/secure_invoke/secure_invoke_lib.cc @@ -215,7 +215,8 @@ absl::Status InvokeBuyerFrontEndWithRawRequest( request_metadata, [onDone = std::move(on_done), ¬ification, start = absl::Now()]( absl::StatusOr> - raw_response) mutable { + raw_response, + ResponseMetadata response_metadata) mutable { ABSL_VLOG(0) << "Received bid response from BFE in " << ((absl::Now() - start) / absl::Milliseconds(1)) << " ms."; @@ -317,12 +318,9 @@ std::string PackagePlainTextGetBidsRequestToJson(const HpkeKeyset& keyset, std::make_unique( keyset.public_key, "unused", std::to_string(keyset.key_id)); auto crypto_client = CreateCryptoClient(); - auto secret_request = - EncryptRequestWithHpke( - std::make_unique( - get_bids_raw_request), - *crypto_client, *key_fetcher_manager, - server_common::CloudPlatform::kGcp); + auto secret_request = EncryptRequestWithHpke( + get_bids_raw_request.SerializeAsString(), *crypto_client, + *key_fetcher_manager, server_common::CloudPlatform::kGcp); CHECK(secret_request.ok()) << secret_request.status(); std::string get_bids_request_json; auto get_bids_request_json_status = diff --git a/tools/secure_invoke/secure_invoke_lib_test.cc b/tools/secure_invoke/secure_invoke_lib_test.cc index 557dbe82..7a21e09c 100644 --- a/tools/secure_invoke/secure_invoke_lib_test.cc +++ b/tools/secure_invoke/secure_invoke_lib_test.cc @@ -67,7 +67,7 @@ constexpr char kSampleGetBidRequest[] = R"JSON({ "generationId" : "hardcoded-uuid" }, "publisherName" : "example.com", - "seller" : "https://securepubads.g.doubleclick.net" + "seller" : "https://sellerorigin.com" })JSON"; constexpr char kSampleComponentGetBidRequest[] = R"JSON({ @@ -93,7 +93,7 @@ constexpr char kSampleComponentGetBidRequest[] = R"JSON({ "generationId" : "hardcoded-uuid" }, "publisherName" : "example.com", - "seller" : "https://securepubads.g.doubleclick.net", + "seller" : "https://sellerorigin.com", "top_level_seller": "https://top-level-seller.com" })JSON"; diff --git a/version.txt b/version.txt index 0be1fc7d..b72ad011 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.8.0 \ No newline at end of file +3.9.0 \ No newline at end of file