diff --git a/data.html b/data.html index c40c9eaa5..446d5ba25 100644 --- a/data.html +++ b/data.html @@ -309,12 +309,23 @@
This is an additional contractual document on top of CONTRIBUTING.
diff --git a/searchindex.js b/searchindex.js index b0a844479..db259f15e 100644 --- a/searchindex.js +++ b/searchindex.js @@ -1 +1 @@ -Object.assign(window.search, {"doc_urls":["intro.html#merino","intro.html#table-of-contents","intro.html#about-the-name","intro.html#architecture","api.html#merino-api-documentation","firefox.html#configuring-firefox-and-merino-environments","firefox.html#environments","firefox.html#production","firefox.html#stage","data.html#data-collection","data.html#logs","data.html#merino-apis","data.html#merino-middleware-logs","data.html#merino-cron-tasks","data.html#merino-feature-flags","data.html#curated-recommendations","data.html#metrics","data.html#accuweather","data.html#curated-recommendations-1","social-contract.html#merino-developer-guidelines-and-social-contract","social-contract.html#foster-a-shared-ownership","social-contract.html#use-adrs-to-record-architectural-decisions","social-contract.html#use-slo-and-error-budget-to-manage-service-risks","social-contract.html#request-rra-for-new-content-integrations","social-contract.html#testing-for-productivity--reliability","social-contract.html#aim-for-simplicity","social-contract.html#blame-free-culture","social-contract.html#have-fun","dev/index.html#developer-documentation-for-working-on-merino","dev/index.html#tldr","dev/index.html#run-the-main-app","dev/index.html#general-commands","dev/index.html#documentation","dev/index.html#local-configuration","dev/index.html#update-the-defaults","dev/index.html#create-a-local-override","dev/content-moderation.html#content-moderation-and-blocklists","dev/content-moderation.html#navigational-suggestions--top-picks","dev/content-moderation.html#wikipedia","dev/content-moderation.html#indexer","dev/content-moderation.html#provider","dev/dependencies.html#development-dependencies","dev/dependencies.html#package-dependencies","dev/dependencies.html#service-dependencies","dev/dependencies.html#dev-helpers","dev/logging-and-metrics.html#logging-and-metrics","dev/logging-and-metrics.html#logging","dev/logging-and-metrics.html#types","dev/logging-and-metrics.html#levels","dev/logging-and-metrics.html#metrics","dev/middlewares.html#middlwares","dev/middlewares.html#caveat","dev/feature_flags.html#feature-flags","dev/feature_flags.html#usage","dev/feature_flags.html#implementation","dev/feature_flags.html#configuration","dev/feature_flags.html#random","dev/feature_flags.html#session","dev/feature_flags.html#fields","dev/feature_flags.html#metrics","dev/feature_flags.html#monitoring-in-grafana","dev/release-process.html#the-release-process","dev/release-process.html#release-best-practices","dev/release-process.html#versioning","dev/release-process.html#load-testing","dev/release-process.html#what-to-do-if-production-breaks","dev/release-process.html#what-to-do-if-tests-fail-during-deployment","dev/profiling.html#profiling","dev/profiling.html#usage","dev/profiling.html#understand-the-outputs","testing/index.html#merino-testing","testing/index.html#test-strategy","testing/unit-tests.html#unit-tests","testing/unit-tests.html#fixtures","testing/unit-tests.html#filtercaplogfixture","testing/unit-tests.html#suggestionrequestfixture","testing/unit-tests.html#scopefixture-receivemockfixture--sendmockfixture","testing/integration-tests.html#integration-tests","testing/integration-tests.html#fixtures","testing/integration-tests.html#filtercaplogfixture","testing/integration-tests.html#testclientfixture","testing/integration-tests.html#testclientwitheventsfixture","testing/integration-tests.html#requestsummarylogdatafixture","testing/integration-tests.html#injectprovidersfixture--providersfixture","testing/integration-tests.html#setupprovidersfixture","testing/integration-tests.html#teardownprovidersfixture","testing/integration-tests.html#testcontainersfixture","testing/load-tests.html#merino-load-locust-tests","testing/load-tests.html#overview","testing/load-tests.html#related-documentation","testing/load-tests.html#local-execution","testing/load-tests.html#setup-environment","testing/load-tests.html#run-test-session","testing/load-tests.html#clean-up-environment","testing/load-tests.html#distributed-gcp-execution---manual-trigger","testing/load-tests.html#setup-environment-1","testing/load-tests.html#run-test-session-1","testing/load-tests.html#clean-up-environment-1","testing/load-tests.html#distributed-gcp-execution---ci-trigger","testing/load-tests.html#run-test-session-2","testing/load-tests.html#calibration","testing/load-tests.html#calibrating-for-users_per_worker","testing/load-tests.html#setup-environment-2","testing/load-tests.html#calibrate","testing/load-tests.html#clean-up-environment-2","testing/load-tests.html#calibrating-for-worker_count","testing/load-tests.html#setup-environment-3","testing/load-tests.html#calibrate-1","testing/load-tests.html#clean-up-environment-3","testing/load-tests.html#maintenance","testing/contract-tests/index.html#merino-contract-tests","testing/contract-tests/index.html#overview","testing/contract-tests/index.html#client","testing/contract-tests/index.html#merino","testing/contract-tests/index.html#kinto-setup","testing/contract-tests/index.html#kinto--kinto-attachments","testing/contract-tests/index.html#local-execution","testing/contract-tests/index.html#maintenance","testing/contract-tests/kinto-setup.html#merino-contract-tests---kinto-setup","testing/contract-tests/kinto-setup.html#overview","testing/contract-tests/kinto-setup.html#local-execution","testing/contract-tests/client.html#merino-contract-tests---client","testing/contract-tests/client.html#overview","testing/contract-tests/client.html#scenarios","testing/contract-tests/client.html#local-execution","operations/index.html#operations","operations/rollback.html#how-to-rollback-changes","operations/blocklist-nav-suggestions.html#navigational-suggestions-job-blocklist","operations/blocklist-nav-suggestions.html#add-to-blocklist","operations/blocklist-nav-suggestions.html#remove-from-blocklist","operations/blocklist-wikipedia.html#how-to-add-to-the-wikipedia-indexer-and-provider-blocklist","operations/blocklist-wikipedia.html#provider---rapid-blocklist-addition","operations/blocklist-wikipedia.html#indexer-job","operations/testfailures.html#what-to-do-with-test-failures-in-ci","operations/configs.html#configuring-merino-operations","operations/configs.html#settings","operations/configs.html#file-organization","operations/configs.html#general","operations/configs.html#caveat","operations/elasticsearch.html#elasticsearch-operations","operations/elasticsearch.html#elasticsearch-index-policy","operations/elasticsearch.html#closed-index-recovery","operations/jobs.html#jobs","operations/jobs/navigational_suggestions.html#merino-jobs-operations","operations/jobs/navigational_suggestions.html#navigational-suggestions","operations/jobs/navigational_suggestions.html#running-the-job-in-airflow","operations/jobs/navigational_suggestions.html#grid-view-tab-airflow-ui","operations/jobs/navigational_suggestions.html#graph-view-tab-airflow-ui---alternative","operations/jobs/dynamic-wiki-indexer.html#merino-jobs-operations","operations/jobs/dynamic-wiki-indexer.html#dynamic-wikipedia-indexer-job","operations/jobs/dynamic-wiki-indexer.html#running-the-job-in-airflow","operations/jobs/dynamic-wiki-indexer.html#grid-view-tab-airflow-ui","operations/jobs/dynamic-wiki-indexer.html#graph-view-tab-airflow-ui---alternative","operations/jobs/csv-remote-settings.html#merino-jobs-operations","operations/jobs/csv-remote-settings.html#csv-remote-settings-uploader-job","operations/jobs/csv-remote-settings.html#uploading-suggestions-step-by-step","operations/jobs/csv-remote-settings.html#running-the-uploader","operations/jobs/csv-remote-settings.html#structure-of-the-remote-settings-data","adr/index.html#merino-adrs","adr/0001-locust-vs-k6-merino-py-performance-test-framework.html#locust-vs-k6-merino-py-performance-test-framework","adr/0001-locust-vs-k6-merino-py-performance-test-framework.html#context-and-problem-statement","adr/0001-locust-vs-k6-merino-py-performance-test-framework.html#decision-drivers","adr/0001-locust-vs-k6-merino-py-performance-test-framework.html#considered-options","adr/0001-locust-vs-k6-merino-py-performance-test-framework.html#decision-outcome","adr/0001-locust-vs-k6-merino-py-performance-test-framework.html#pros-and-cons-of-the-options","adr/0001-locust-vs-k6-merino-py-performance-test-framework.html#a-locust","adr/0001-locust-vs-k6-merino-py-performance-test-framework.html#b-k6","adr/0001-locust-vs-k6-merino-py-performance-test-framework.html#links","adr/0002-merino-general-response.html#merino-suggest-api-response-structure","adr/0002-merino-general-response.html#context-and-problem-statement","adr/0002-merino-general-response.html#decision-drivers","adr/0002-merino-general-response.html#considered-options","adr/0002-merino-general-response.html#decision-outcome","adr/0002-merino-general-response.html#positive-consequences-of-option-b","adr/0002-merino-general-response.html#negative-consequences-of-option-b","adr/0002-merino-general-response.html#positive-consequences-of-not-increasing-api-version","adr/0002-merino-general-response.html#negative-consequences-of-not-increasing-api-version","adr/0002-merino-general-response.html#pros-and-cons-of-the-options","adr/0002-merino-general-response.html#a-continue-to-add-to-top-level-with-optional-fields","adr/0002-merino-general-response.html#b-custom-details-field-for-bespoke-provider-fields","adr/0002-merino-general-response.html#b5-custom-details-field-without-the-provider-nesting","adr/0002-merino-general-response.html#c-custom-details-field-for-a-type","adr/0002-merino-general-response.html#d-component-driven-custom_details","adr/0002-merino-general-response.html#links","adr/0003-test-containers.html#streamline-test-coverage-of-third-party-integrations","adr/0003-test-containers.html#context-and-problem-statement","adr/0003-test-containers.html#decision-drivers","adr/0003-test-containers.html#considered-options","adr/0003-test-containers.html#decision-outcome","adr/0003-test-containers.html#pros-and-cons-of-the-options","adr/0003-test-containers.html#a-yes-expand-the-scope-of-integration-tests-using-dependency-docker-containers-testcontainers","adr/0003-test-containers.html#b-yes-reduce-the-dependency-overhead-in-tests-using-development-and-stage-environments","adr/0003-test-containers.html#c-no-fulfill-the-current-test-strategy-with-contract-test-coverage-status-quo","adr/0003-test-containers.html#open-questions","adr/0003-test-containers.html#links"],"index":{"documentStore":{"docInfo":{"0":{"body":44,"breadcrumbs":2,"title":1},"1":{"body":125,"breadcrumbs":3,"title":2},"10":{"body":30,"breadcrumbs":3,"title":1},"100":{"body":96,"breadcrumbs":8,"title":1},"101":{"body":8,"breadcrumbs":9,"title":2},"102":{"body":123,"breadcrumbs":9,"title":2},"103":{"body":207,"breadcrumbs":8,"title":1},"104":{"body":9,"breadcrumbs":10,"title":3},"105":{"body":14,"breadcrumbs":9,"title":2},"106":{"body":28,"breadcrumbs":9,"title":2},"107":{"body":162,"breadcrumbs":8,"title":1},"108":{"body":9,"breadcrumbs":10,"title":3},"109":{"body":46,"breadcrumbs":8,"title":1},"11":{"body":187,"breadcrumbs":4,"title":2},"110":{"body":6,"breadcrumbs":10,"title":3},"111":{"body":92,"breadcrumbs":8,"title":1},"112":{"body":34,"breadcrumbs":8,"title":1},"113":{"body":12,"breadcrumbs":8,"title":1},"114":{"body":27,"breadcrumbs":9,"title":2},"115":{"body":22,"breadcrumbs":10,"title":3},"116":{"body":52,"breadcrumbs":9,"title":2},"117":{"body":39,"breadcrumbs":8,"title":1},"118":{"body":0,"breadcrumbs":14,"title":5},"119":{"body":27,"breadcrumbs":10,"title":1},"12":{"body":6,"breadcrumbs":5,"title":3},"120":{"body":137,"breadcrumbs":11,"title":2},"121":{"body":0,"breadcrumbs":12,"title":4},"122":{"body":33,"breadcrumbs":9,"title":1},"123":{"body":287,"breadcrumbs":9,"title":1},"124":{"body":204,"breadcrumbs":10,"title":2},"125":{"body":4,"breadcrumbs":2,"title":1},"126":{"body":41,"breadcrumbs":4,"title":2},"127":{"body":17,"breadcrumbs":9,"title":4},"128":{"body":16,"breadcrumbs":7,"title":2},"129":{"body":32,"breadcrumbs":7,"title":2},"13":{"body":6,"breadcrumbs":5,"title":3},"130":{"body":0,"breadcrumbs":10,"title":5},"131":{"body":52,"breadcrumbs":9,"title":4},"132":{"body":57,"breadcrumbs":7,"title":2},"133":{"body":90,"breadcrumbs":7,"title":3},"134":{"body":11,"breadcrumbs":5,"title":3},"135":{"body":51,"breadcrumbs":3,"title":1},"136":{"body":82,"breadcrumbs":4,"title":2},"137":{"body":116,"breadcrumbs":3,"title":1},"138":{"body":49,"breadcrumbs":3,"title":1},"139":{"body":12,"breadcrumbs":4,"title":2},"14":{"body":10,"breadcrumbs":5,"title":3},"140":{"body":26,"breadcrumbs":5,"title":3},"141":{"body":41,"breadcrumbs":5,"title":3},"142":{"body":0,"breadcrumbs":3,"title":1},"143":{"body":0,"breadcrumbs":7,"title":3},"144":{"body":77,"breadcrumbs":6,"title":2},"145":{"body":17,"breadcrumbs":7,"title":3},"146":{"body":22,"breadcrumbs":9,"title":5},"147":{"body":96,"breadcrumbs":10,"title":6},"148":{"body":0,"breadcrumbs":8,"title":3},"149":{"body":66,"breadcrumbs":9,"title":4},"15":{"body":60,"breadcrumbs":4,"title":2},"150":{"body":17,"breadcrumbs":8,"title":3},"151":{"body":22,"breadcrumbs":10,"title":5},"152":{"body":79,"breadcrumbs":11,"title":6},"153":{"body":0,"breadcrumbs":9,"title":3},"154":{"body":57,"breadcrumbs":11,"title":5},"155":{"body":460,"breadcrumbs":10,"title":4},"156":{"body":203,"breadcrumbs":8,"title":2},"157":{"body":34,"breadcrumbs":10,"title":4},"158":{"body":7,"breadcrumbs":3,"title":2},"159":{"body":13,"breadcrumbs":15,"title":8},"16":{"body":120,"breadcrumbs":3,"title":1},"160":{"body":38,"breadcrumbs":10,"title":3},"161":{"body":66,"breadcrumbs":9,"title":2},"162":{"body":3,"breadcrumbs":9,"title":2},"163":{"body":55,"breadcrumbs":9,"title":2},"164":{"body":0,"breadcrumbs":10,"title":3},"165":{"body":89,"breadcrumbs":8,"title":1},"166":{"body":138,"breadcrumbs":9,"title":2},"167":{"body":9,"breadcrumbs":8,"title":1},"168":{"body":21,"breadcrumbs":9,"title":5},"169":{"body":76,"breadcrumbs":7,"title":3},"17":{"body":151,"breadcrumbs":3,"title":1},"170":{"body":24,"breadcrumbs":6,"title":2},"171":{"body":29,"breadcrumbs":6,"title":2},"172":{"body":45,"breadcrumbs":6,"title":2},"173":{"body":21,"breadcrumbs":8,"title":4},"174":{"body":13,"breadcrumbs":8,"title":4},"175":{"body":30,"breadcrumbs":9,"title":5},"176":{"body":24,"breadcrumbs":9,"title":5},"177":{"body":0,"breadcrumbs":7,"title":3},"178":{"body":145,"breadcrumbs":10,"title":6},"179":{"body":165,"breadcrumbs":11,"title":7},"18":{"body":74,"breadcrumbs":4,"title":2},"180":{"body":153,"breadcrumbs":11,"title":7},"181":{"body":89,"breadcrumbs":9,"title":5},"182":{"body":86,"breadcrumbs":8,"title":4},"183":{"body":3,"breadcrumbs":5,"title":1},"184":{"body":11,"breadcrumbs":11,"title":6},"185":{"body":135,"breadcrumbs":8,"title":3},"186":{"body":92,"breadcrumbs":7,"title":2},"187":{"body":30,"breadcrumbs":7,"title":2},"188":{"body":105,"breadcrumbs":7,"title":2},"189":{"body":0,"breadcrumbs":8,"title":3},"19":{"body":5,"breadcrumbs":7,"title":5},"190":{"body":278,"breadcrumbs":15,"title":10},"191":{"body":111,"breadcrumbs":15,"title":10},"192":{"body":112,"breadcrumbs":15,"title":10},"193":{"body":31,"breadcrumbs":7,"title":2},"194":{"body":5,"breadcrumbs":6,"title":1},"2":{"body":32,"breadcrumbs":2,"title":1},"20":{"body":216,"breadcrumbs":5,"title":3},"21":{"body":40,"breadcrumbs":7,"title":5},"22":{"body":56,"breadcrumbs":9,"title":7},"23":{"body":49,"breadcrumbs":7,"title":5},"24":{"body":60,"breadcrumbs":5,"title":3},"25":{"body":41,"breadcrumbs":4,"title":2},"26":{"body":78,"breadcrumbs":5,"title":3},"27":{"body":7,"breadcrumbs":3,"title":1},"28":{"body":0,"breadcrumbs":6,"title":4},"29":{"body":5,"breadcrumbs":3,"title":1},"3":{"body":197,"breadcrumbs":2,"title":1},"30":{"body":25,"breadcrumbs":5,"title":3},"31":{"body":196,"breadcrumbs":4,"title":2},"32":{"body":32,"breadcrumbs":3,"title":1},"33":{"body":31,"breadcrumbs":4,"title":2},"34":{"body":87,"breadcrumbs":4,"title":2},"35":{"body":58,"breadcrumbs":5,"title":3},"36":{"body":90,"breadcrumbs":7,"title":3},"37":{"body":52,"breadcrumbs":8,"title":4},"38":{"body":21,"breadcrumbs":5,"title":1},"39":{"body":55,"breadcrumbs":5,"title":1},"4":{"body":11,"breadcrumbs":6,"title":3},"40":{"body":36,"breadcrumbs":5,"title":1},"41":{"body":0,"breadcrumbs":5,"title":2},"42":{"body":71,"breadcrumbs":5,"title":2},"43":{"body":83,"breadcrumbs":5,"title":2},"44":{"body":16,"breadcrumbs":5,"title":2},"45":{"body":46,"breadcrumbs":6,"title":2},"46":{"body":30,"breadcrumbs":5,"title":1},"47":{"body":48,"breadcrumbs":5,"title":1},"48":{"body":127,"breadcrumbs":5,"title":1},"49":{"body":31,"breadcrumbs":5,"title":1},"5":{"body":42,"breadcrumbs":8,"title":4},"50":{"body":19,"breadcrumbs":4,"title":1},"51":{"body":59,"breadcrumbs":4,"title":1},"52":{"body":0,"breadcrumbs":6,"title":2},"53":{"body":65,"breadcrumbs":5,"title":1},"54":{"body":45,"breadcrumbs":5,"title":1},"55":{"body":7,"breadcrumbs":5,"title":1},"56":{"body":8,"breadcrumbs":5,"title":1},"57":{"body":15,"breadcrumbs":5,"title":1},"58":{"body":28,"breadcrumbs":5,"title":1},"59":{"body":29,"breadcrumbs":5,"title":1},"6":{"body":0,"breadcrumbs":5,"title":1},"60":{"body":28,"breadcrumbs":6,"title":2},"61":{"body":97,"breadcrumbs":6,"title":2},"62":{"body":17,"breadcrumbs":7,"title":3},"63":{"body":16,"breadcrumbs":5,"title":1},"64":{"body":67,"breadcrumbs":6,"title":2},"65":{"body":13,"breadcrumbs":6,"title":2},"66":{"body":5,"breadcrumbs":8,"title":4},"67":{"body":52,"breadcrumbs":4,"title":1},"68":{"body":38,"breadcrumbs":4,"title":1},"69":{"body":69,"breadcrumbs":5,"title":2},"7":{"body":11,"breadcrumbs":5,"title":1},"70":{"body":0,"breadcrumbs":7,"title":2},"71":{"body":66,"breadcrumbs":7,"title":2},"72":{"body":68,"breadcrumbs":9,"title":2},"73":{"body":3,"breadcrumbs":8,"title":1},"74":{"body":30,"breadcrumbs":8,"title":1},"75":{"body":21,"breadcrumbs":8,"title":1},"76":{"body":22,"breadcrumbs":10,"title":3},"77":{"body":104,"breadcrumbs":9,"title":2},"78":{"body":3,"breadcrumbs":8,"title":1},"79":{"body":5,"breadcrumbs":8,"title":1},"8":{"body":21,"breadcrumbs":5,"title":1},"80":{"body":15,"breadcrumbs":8,"title":1},"81":{"body":21,"breadcrumbs":8,"title":1},"82":{"body":34,"breadcrumbs":8,"title":1},"83":{"body":28,"breadcrumbs":9,"title":2},"84":{"body":17,"breadcrumbs":8,"title":1},"85":{"body":16,"breadcrumbs":8,"title":1},"86":{"body":43,"breadcrumbs":8,"title":1},"87":{"body":13,"breadcrumbs":11,"title":4},"88":{"body":26,"breadcrumbs":8,"title":1},"89":{"body":12,"breadcrumbs":9,"title":2},"9":{"body":11,"breadcrumbs":4,"title":2},"90":{"body":25,"breadcrumbs":9,"title":2},"91":{"body":174,"breadcrumbs":9,"title":2},"92":{"body":101,"breadcrumbs":10,"title":3},"93":{"body":14,"breadcrumbs":10,"title":3},"94":{"body":10,"breadcrumbs":12,"title":5},"95":{"body":185,"breadcrumbs":9,"title":2},"96":{"body":306,"breadcrumbs":10,"title":3},"97":{"body":11,"breadcrumbs":10,"title":3},"98":{"body":22,"breadcrumbs":12,"title":5},"99":{"body":269,"breadcrumbs":10,"title":3}},"docs":{"0":{"body":"Merino is a service that provides address bar suggestions and curated recommendations to Firefox. Some of this content comes from third party providers. In this case, Merino serves as a privacy preserving buffer. User input in the address bar is handled by Merino and any clicked impression will be delegated to a Mozilla-controlled service which will then send an interaction ping if defined in the request and not to a provider directly. See API documentation for more details.","breadcrumbs":"Intro » Merino","id":"0","title":"Merino"},"1":{"body":"api.md - API Documentation describes endpoints, query parameters, request and response headers, response objects and details on the suggestion objects. firefox.md - Firefox and Merino Environments describes how to enable Merino in Firefox and lists the endpoints for the service in Production, State and Dev. data.md - Data, Metrics, Logging describes all metrics and logs. dev/index.md - Basic Developer Docs describes basics of working on Merino. dev/dependencies.md - Development Dependencies describes the development dependencies required for Merino. dev/logging-and-metrics.md - Logging and Metrics describes metrics, logging, and telemetry. dev/release-process.md - Release Process describes the release process of Merino in detail. dev/testing.md - Testing describes unit, integration, contract and load tests for Merino. dev/profiling.md - Profiling describes how to profile Merino to address performance issues. operations/configs.md - Configuring Merino describes configuration management of the project, Dynaconf setup, and the configuration of the HTTP server, logging, metrics, Remote Settings, and Sentry. operations/elasticsearch.md - Elasticsearch Operations describes some functionality and operations that we do on the Elasticsearch cluster. operations/jobs.md - Merino Jobs describes the jobs that are configured in Merino. Indicate where the jobs exist and link to the details for how the jobs are run.","breadcrumbs":"Intro » Table of Contents","id":"1","title":"Table of Contents"},"10":{"body":"This list does not include any DEBUG level events, since those are not logged by default in production. The level and type of the log is listed. Any log containing sensitive data must include a boolean field sensitive that is set to true to exempt it from flowing to the generally accessible log inspection interfaces.","breadcrumbs":"Data collection » Logs","id":"10","title":"Logs"},"100":{"body":"Following the addition of new features, such as a Locust Task or Locust User, or environmental changes, such as node size or the upgrade of a major dependency like the python version image, it may be necessary to re-establish the recommended parameters of the performance test. Parameter Description [WAIT TIME][13] - Changing this cadence will increase or decrease the number of channel subscriptions and notifications sent by a MerinoUser. - The default is currently in use for the MerinoUser class. [TASK WEIGHT][14] - Changing this weight impacts the probability of a task being chosen for execution. - This value is hardcoded in the task decorators of the MerinoUser class. USERS_PER_WORKER - This value should be set to the maximum number of users a Locust worker can support given CPU and memory constraints. - This value is hardcoded in the LoadTestShape classes. WORKER_COUNT - This value is derived by dividing the total number of users needed for the performance test by the USERS_PER_WORKER. - This value is hardcoded in the LoadTestShape classes and the setup_k8s.sh script.","breadcrumbs":"Working on the Code » Testing & Test Strategy » Load Tests » Calibration","id":"100","title":"Calibration"},"101":{"body":"This process is used to determine the number of users that a Locust worker can support.","breadcrumbs":"Working on the Code » Testing & Test Strategy » Load Tests » Calibrating for USERS_PER_WORKER","id":"101","title":"Calibrating for USERS_PER_WORKER"},"102":{"body":"1. Start a GCP Cloud Shell The load tests can be executed from the contextual-services-test-eng cloud shell . If executing a load test for the first time, the git merino-py repository will need to be cloned locally. 2. Configure the Bash Script The setup_k8s.sh file, located in the tests\\load directory, contains shell commands to create a GKE cluster, setup an existing GKE cluster or delete a GKE cluster Execute the following from the root directory, to make the file executable: chmod +x tests/load/setup_k8s.sh 3. Create the GCP Cluster In the setup_k8s.sh script, modify the WORKER_COUNT variable to equal 1 Execute the setup_k8s.sh file from the root directory and select the create option, in order to initiate the process of creating a cluster, setting up the env variables and building the docker image. Choose smoke or average depending on the type of load test required. ./tests/load/setup_k8s.sh create [smoke|average] The cluster creation process will take some time. It is considered complete, once an external IP is assigned to the locust_master node. Monitor the assignment via a watch loop: kubectl get svc locust-master --watch","breadcrumbs":"Working on the Code » Testing & Test Strategy » Load Tests » Setup Environment","id":"102","title":"Setup Environment"},"103":{"body":"Repeat steps 1 to 3, using a process of elimination, such as the bisection method, to determine the maximum USERS_PER_WORKER. The load tests are considered optimized when CPU and memory resources are maximally utilized. This step is meant to determine the maximum user count that a node can accommodate by observing CPU and memory usage while steadily increasing or decreasing the user count. You can monitor the CPU percentage in the Locust UI but also in the Kubernetes engine Workloads tab where both memory and CPU are visualized on charts. 1. Start Load Test In a browser navigate to http://$EXTERNAL_IP:8089 This url can be generated via command EXTERNAL_IP=$(kubectl get svc locust-master -o jsonpath=\"{.status.loadBalancer.ingress[0].ip}\")\necho http://$EXTERNAL_IP:8089 Set up the load test parameters: ShapeClass: Default UserClasses: MerinoUser Number of users: USERS_PER_WORKER (Consult the Merino_spreadsheet to determine a starting point) Ramp up: RAMP_UP (RAMP_UP = 5/USERS_PER_WORKER) Host: 'https://stagepy.merino.nonprod.cloudops.mozgcp.net' Duration (Optional): 600s Select \"Start Swarm\" 2. Stop Load Test Select the 'Stop' button in the top right hand corner of the Locust UI, after the desired test duration has elapsed. If the 'Run time' or 'Duration' is set in step 1, the load test will stop automatically. 3. Analyse Results CPU and Memory Resource Graphs CPU and Memory usage should be less than 90% of the available capacity CPU and Memory Resources can be observed in Google Cloud > Kubernetes Engine > Workloads Log Errors or Warnings Locust will emit errors or warnings if high CPU or memory usage occurs during the execution of a load test. The presence of these logs is a strong indication that the USERS_PER_WORKER count is too high 4. Report Results See Distributed GCP Execution (Manual Trigger) - Analyse Results 5. Update Shape and Script Values WORKER_COUNT = MAX_USERS/USERS_PER_WORKER If MAX_USERS is unknown, calibrate to determine WORKER_COUNT Update the USERS_PER_WORKER and WORKER_COUNT values in the following files: \\tests\\load\\locustfiles\\smoke_load.py or \\tests\\load\\locustfiles\\average_load.py \\tests\\load\\setup_k8s.sh","breadcrumbs":"Working on the Code » Testing & Test Strategy » Load Tests » Calibrate","id":"103","title":"Calibrate"},"104":{"body":"See Distributed GCP Execution (Manual Trigger) - Clean-up Environment","breadcrumbs":"Working on the Code » Testing & Test Strategy » Load Tests » Clean-up Environment","id":"104","title":"Clean-up Environment"},"105":{"body":"This process is used to determine the number of Locust workers required in order to generate sufficient load for a test given a SHAPE_CLASS.","breadcrumbs":"Working on the Code » Testing & Test Strategy » Load Tests » Calibrating for WORKER_COUNT","id":"105","title":"Calibrating for WORKER_COUNT"},"106":{"body":"See Distributed GCP Execution (Manual Trigger) - Setup Environment Note that in the setup_k8s.sh the maximum number of nodes is set using the total-max-nodes google cloud option. It may need to be increased if the number of workers can't be supported by the cluster.","breadcrumbs":"Working on the Code » Testing & Test Strategy » Load Tests » Setup Environment","id":"106","title":"Setup Environment"},"107":{"body":"Repeat steps 1 to 4, using a process of elimination, such as the bisection method, to determine the maximum WORKER_COUNT. The tests are considered optimized when they generate the minimum load required to cause node scaling in the the Merino-py Stage environment. You can monitor the Merino-py pod counts on Grafana . 1. Update Shape and Script Values Update the WORKER_COUNT values in the following files: \\tests\\load\\locustfiles\\smoke_load.py or \\tests\\load\\locustfiles\\average_load.py \\tests\\load\\setup_k8s.sh Using Git, commit the changes locally 2. Start Load Test In a browser navigate to http://$EXTERNAL_IP:8089 This url can be generated via command EXTERNAL_IP=$(kubectl get svc locust-master -o jsonpath=\"{.status.loadBalancer.ingress[0].ip}\")\necho http://$EXTERNAL_IP:8089 Set up the load test parameters: ShapeClass: SHAPE_CLASS Host: 'https://stagepy.merino.nonprod.cloudops.mozgcp.net' Select \"Start Swarm\" 3. Stop Load Test Select the 'Stop' button in the top right hand corner of the Locust UI, after the desired test duration has elapsed. If the 'Run time', 'Duration' or 'ShapeClass' are set in step 1, the load test will stop automatically. 4. Analyse Results Stage Environment Pod Counts The 'Merino-py Pod Count' should demonstrate scaling during the execution of the load test The pod counts can be observed in Grafana CPU and Memory Resources CPU and Memory usage should be less than 90% of the available capacity in the cluster CPU and Memory Resources can be observed in Google Cloud > Kubernetes Engine > Workloads 5. Report Results See Distributed GCP Execution (Manual Trigger) - Report Results","breadcrumbs":"Working on the Code » Testing & Test Strategy » Load Tests » Calibrate","id":"107","title":"Calibrate"},"108":{"body":"See Distributed GCP Execution (Manual Trigger) - Clean-up Environment","breadcrumbs":"Working on the Code » Testing & Test Strategy » Load Tests » Clean-up Environment","id":"108","title":"Clean-up Environment"},"109":{"body":"The load test maintenance schedule cadence is once a quarter and should include updating the following: poetry version and python dependencies pyproject.toml poetry.lock Docker artifacts Dockerfile docker-compose.yml Distributed GCP execution scripts and Kubernetes configurations setup_k8s.sh locust-master-controller.yml locust-master-service.yml locust-worker-controller.yml CircleCI contract test jobs config.yml Documentation load test docs","breadcrumbs":"Working on the Code » Testing & Test Strategy » Load Tests » Maintenance","id":"109","title":"Maintenance"},"11":{"body":"INFO web.suggest.request - A suggestion request is being processed. This event will include fields for all relevant details of the request. Fields: sensitive - Always set to true to ensure proper routing. query - If query logging is enabled, the text the user typed. Otherwise an empty string. country - The country the request came from. region - The first country subdivision the request came from. city - The city the request came from. dma - A US-only location description that is larger than city and smaller than states, but does not align to political borders. agent - The original user agent. os_family - Parsed from the user agent. One of \"windows\", \"macos\", \"linux\", \"ios\", \"android\", \"chrome os\", \"blackberry\", or \"other\". form_factor - Parsed from the user agent. One of \"desktop\", \"phone\", \"tablet\", or \"other\" browser - The browser and possibly version detected. Either \"Firefox(XX)\" where XX is the version, or \"Other\". rid - The request ID. WIP accepts_english - True if the user's Accept-Language header includes an English locale, false otherwise. requested_providers - A comma separated list of providers requested via the query string, or an empty string if none were requested (in which case the default values would be used). client_variants - Any client variants sent to Merino in the query string. session_id - A UUID generated by the client for each search session. sequence_no - A client-side event counter (0-based) that records the query sequence within each search session. INFO request.summary - The application request summary that follows the MozLog convention. This log is recorded for all incoming HTTP requests except for the suggest API endpoint. ERROR dockerflow.error_endpoint - The __error__ endpoint of the server was called. This is used to test our error reporting system. It is not a cause for concern, unless we receive a large amount of these records, in which case some outside service is likely malicious or misconfigured.","breadcrumbs":"Data collection » Merino APIs","id":"11","title":"Merino APIs"},"110":{"body":"This documentation describes the automated contract tests for Merino.","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » Merino Contract Tests","id":"110","title":"Merino Contract Tests"},"111":{"body":"The tests in the tests/contract directory consume Merino's APIs using more opaque techniques. These tests run against a Docker container of the service, specify settings via environment variables, and operate on the HTTP API layer only and as such are more concerned with external contracts and behavior. The contract tests cannot configure the server per-test. The contract test suite is designed to be set up as a docker-compose CI workflow. To simulate common use cases, the suite utilizes 6 docker containers: client, merino, kinto-setup, kinto, kinto-attachments, and redis. The following sequence diagram depicts container interactions during the remote_settings__coffee test scenario. Test Scenario: remote_settings__coffee Sequence diagram of the integration tests Notes: The interactions between kinto and kinto-attachments are not depicted. The diagram was composed using Miro ,","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » Overview","id":"111","title":"Overview"},"112":{"body":"The client container consists of a Python-based test framework that executes the contract tests. The HTTP client used in the framework can be instructed to prepare Remote Settings data through requests to kinto and can verify Merino functionality through requests to the Merino service. For more details, see the client documentation .","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » client","id":"112","title":"client"},"113":{"body":"The merino container encapsulates the Merino service under test. For more details, see the Merino documentation .","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » merino","id":"113","title":"merino"},"114":{"body":"The kinto-setup container consists of a Python-based program responsible for defining the Remote Settings bucket, \"main\", and collection, \"quicksuggest\", prior to the merino container startup, a pre-requisite. For more details, see the kinto-setup documentation .","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » kinto-setup","id":"114","title":"kinto-setup"},"115":{"body":"The kinto container holds a minimalist storage service with synchronisation and sharing abilities. It uses the kinto-attachments container to store data locally. For more details, see the Remote Settings documentation .","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » kinto & kinto-attachments","id":"115","title":"kinto & kinto-attachments"},"116":{"body":"Local execution can be expedited by simply running make contract-tests, from the repository root. This creates the Docker containers with kinto, Merino and the test client and runs the test scenarios against them. make contract-tests To remove contract test containers and network artifacts, execute the following from the repository root: make contract-tests-clean Failing to run this clean command between code changes may result in your changes not being reflected. See Makefile for details.","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » Local Execution","id":"116","title":"Local Execution"},"117":{"body":"The contract test maintenance schedule cadence is once a quarter and should include updating the following: poetry version and python dependencies pyproject.toml poetry.lock Docker artifacts client Dockerfile kinto-setup Dockerfile docker-compose.yml CircleCI contract test jobs config.yml Documentation client documentation kinto-setup documentation contract test documentation","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » Maintenance","id":"117","title":"Maintenance"},"118":{"body":"","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » Kinto Setup » Merino Contract Tests - Kinto-Setup","id":"118","title":"Merino Contract Tests - Kinto-Setup"},"119":{"body":"This documentation describes setting up Kinto for contract tests. Specifically, it is responsible for the creation of the Remote Settings bucket and collection, a pre-requisite for the Merino service. For more details on contract test design, refer to the contract-tests documentation .","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » Kinto Setup » Overview","id":"119","title":"Overview"},"12":{"body":"Geolocation WARNING merino.middleware.geolocation - There was an error with a geolocation lookup.","breadcrumbs":"Data collection » Merino Middleware Logs","id":"12","title":"Merino Middleware Logs"},"120":{"body":"To execute the kinto-setup outside the Docker container, create a Python virtual environment, set environment variables, expose the Kinto API port in the docker-compose.yml and use a Python command. It is recommended to execute the setup within a Python virtual environment to prevent dependency cross contamination. Create a Virtual Environment The developer documentation on dependences for working on Merino dependencies, provides instruction on creating a virtual environment via pyenv, and installing all requirements via poetry. Setup Environment Variables The following environment variables are set in docker-compose.yml, but will require local setup via command line, pytest.ini file or IDE configuration: KINTO_URL: The URL of the Kinto service Example: KINTO_URL=http://localhost:8888 KINTO_BUCKET: The ID of the Kinto bucket to create Example: KINTO_BUCKET=main KINTO_COLLECTION: The ID of the Kinto collection to create Example: KINTO_COLLECTION=quicksuggest Modify tests/contract/docker-compose.yml In the kinto definition, expose port 8888 by adding the following: ports: - \"8888:8888\" Run kinto and kinto-attachment docker containers. Execute the following from the project root: docker-compose \\ -f tests/contract/docker-compose.yml \\ -p merino-py-contract-tests \\ up kinto Run the kinto-setup service Execute the following from the project root: python tests/contract/kinto-setup/main.py","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » Kinto Setup » Local Execution","id":"120","title":"Local Execution"},"121":{"body":"","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » Client » Merino Contract Tests - Client","id":"121","title":"Merino Contract Tests - Client"},"122":{"body":"This documentation describes a Python-based test framework for the contract tests. The HTTP client used in the framework supports: Requests to Kinto (Remote Settings) for record population Requests for suggestions from Merino, with response checks For more details on contract test design, refer to the contract-tests documentation .","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » Client » Overview","id":"122","title":"Overview"},"123":{"body":"The client is instructed on request and response check actions via scenarios, recorded in the scenarios.yml file. A scenario is defined by a name, a description and steps. Name A test name should identify the use case under test. Names are written in snake case with double-underscores __ for scenario and behavior delineation. Example: remote_settings__refresh Description A test description should outline, in greater detail, the purpose of a test, meaning the feature or common user interaction being verified. Example: Test that Merino successfully returns refreshed output in the cases of suggestion content updates and additions Steps Kinto Service The Kinto service scenario step populates the records used by the Merino service Kinto request fields: service - Set the value to kinto, to direct requests to the Kinto service delay - (optional) Set seconds to pause before execution of request record_id - Set the ID that will correspond with the records specified in filename filename - Set the file with records to upload. The files are located in ..\\volumes\\kinto data_type - Set to data or offline-expansion-data A Kinto service scenario step will not check a response defined in the scenario, but will raise an error in the event of an unexpected HTTP response All records populated by Kinto service scenario steps are deleted as part of the scenario teardown Example: - request: service: kinto record_id: \"data-01\" filename: \"data-01.json\" data_type: \"data\" Merino Service The Merino service scenario step sends queries to merino and checks the validity of the responses Merino request fields: service - Set the value to merino, to direct requests to the Merino service delay - (optional) Set seconds to pause before execution of request method - Set the HTTP request method path - Set the query and parameters for the request method headers - Set a list of HTTP request headers Merino response fields: status_code - Set the expected HTTP response status code content - Set a list of expected merino suggestion content The request_id is excluded from verification and can be set to null in the scenario Example: - name: suggest__apple description: Test that Merino successfully returns a suggestion steps: - request: service: merino method: GET path: '/api/v1/suggest?q=apple' headers: - name: User-Agent value: 'Mozilla/5.0 (Windows NT 10.0; rv:10.0) Gecko/20100101 Firefox/91.0' - name: Accept-Language value: 'en-US' response: status_code: 200 content: client_variants: [] server_variants: [] request_id: null suggestions: - block_id: 1 full_keyword: 'apple' title: 'Wikipedia - Apple' url: 'https://en.wikipedia.org/wiki/Apple' impression_url: 'https://127.0.0.1/' click_url: 'https://127.0.0.1/' provider: 'test_provider' advertiser: 'test_advertiser' is_sponsored: false icon: 'https://en.wikipedia.org/favicon.ico' score: 0.0","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » Client » Scenarios","id":"123","title":"Scenarios"},"124":{"body":"To execute the test scenarios outside the client Docker container, create a Python virtual environment, set environment variables, expose the Merino and Kinto API ports in the docker-compose.yml and use a pytest command. It is recommended to execute the tests within a Python virtual environment to prevent dependency cross contamination. Create a Virtual Environment The developer documentation on dependences for working on Merino dependencies, provides instruction on creating a virtual environment via pyenv, and installing all requirements via poetry. Setup Environment Variables The following environment variables are set in docker-compose.yml, but will require local setup via command line, pytest.ini file or IDE configuration: MERINO_URL: The URL of the Merino service Example: MERINO_URL=http://localhost:8000 KINTO_URL: The URL of the Kinto service Example: KINTO_URL=http://localhost:8888 KINTO_ATTACHMENTS_URL: The URL of the Kinto Attachments service Example: KINTO_ATTACHMENTS_URL=http://localhost:80 KINTO_BUCKET: The ID of the Kinto bucket to create Example: KINTO_BUCKET=main KINTO_COLLECTION: The ID of the Kinto collection to create Example: KINTO_COLLECTION=quicksuggest KINTO_DATA_DIR: The directory containing advertiser data Example: KINTO_DATA_DIR=tests/contract/volumes/kinto SCENARIOS_FILE: The directory containing contract test scenario files Example: SCENARIOS_FILE=tests/contract/volumes/client/scenarios.yml Modify tests/contract/docker-compose.yml In the merino definition, expose port 8000 by adding the following: ports: - \"8000:8000\" In the kinto definition, expose port 8888 by adding the following: ports: - \"8888:8888\" In the kinto-attachments definition, expose port 80 by adding the following: ports: - \"80:80\" Run merino, kinto-setup, kinto and kinto-attachment docker containers Execute the following from the project root: docker-compose \\ -f tests/contract/docker-compose.yml \\ -p merino-py-contract-tests \\ up merino Run the contract tests Execute the following from the project root: pytest tests/contract/client/tests/test_merino.py -vv Tests can be run individually using -k expr . Example executing the remote_settings__refresh scenario: pytest tests/contract/client/tests/test_merino.py -vv -k remote_settings__refresh","breadcrumbs":"Working on the Code » Testing & Test Strategy » Contract Tests » Client » Local Execution","id":"124","title":"Local Execution"},"125":{"body":"This is where we put operational documentation for Merino.","breadcrumbs":"Operations » Operations","id":"125","title":"Operations"},"126":{"body":"Note: We use \"roll-forward\" strategy for rolling back changes in production. Depending on the severity of the problem, decide if this warrants kicking off an incident ; Identify the problematic commit (it may not be the latest commit) and create a revert PR. If it is the latest commit, you can revert the change with: git revert HEAD~1 Create a revert PR and go through normal review process to merge PR.","breadcrumbs":"Operations » Rollback » How to Rollback Changes","id":"126","title":"How to Rollback Changes"},"127":{"body":"The Navigational Suggestions Job blocklist is contained in merino/utils/blocklists.py. The TOP_PICKS_BLOCKLIST variable is used when running the indexing job and prevents the included domains from being added.","breadcrumbs":"Operations » Modify Navigational Suggestions Blocklist » Navigational Suggestions Job Blocklist","id":"127","title":"Navigational Suggestions Job Blocklist"},"128":{"body":"Go to merino/utils/blocklists.py . Add the second-level-domain to the TOP_PICKS_BLOCKLIST set. Open a PR and merge in the changes to block this domain from being indexed.","breadcrumbs":"Operations » Modify Navigational Suggestions Blocklist » Add to Blocklist","id":"128","title":"Add to Blocklist"},"129":{"body":"Repeat as above, just remove the domain from the TOP_PICKS_BLOCKLIST set. Note: removing from the blocklist means that the domain was likely not created during the Airflow job, so if you wish to see it re-added, supposing it is still in the top 1000 domains, you have to re-run the airflow job. See the instructions for this in the jobs/navigational_suggestions docs .","breadcrumbs":"Operations » Modify Navigational Suggestions Blocklist » Remove from Blocklist","id":"129","title":"Remove from Blocklist"},"13":{"body":"WARNING merino.cron - There was an error while executing a cron task.","breadcrumbs":"Data collection » Merino Cron Tasks","id":"13","title":"Merino Cron Tasks"},"130":{"body":"","breadcrumbs":"Operations » Modify Wikipedia Suggestions Blocklist » How to Add to the Wikipedia Indexer and Provider Blocklist","id":"130","title":"How to Add to the Wikipedia Indexer and Provider Blocklist"},"131":{"body":"These steps define how to rapidly add and therefore block a Wikipedia article by its title. In /merino/utils/blocklists.py, add the matching title to TITLE_BLOCK_LIST. NOTE: Ensure the title field is added as it appears with correct spacing between the words. In adding to the list, enter the title as it appears in Wikipedia. Membership checks of the block list are not case sensitive and any underscores in the titles should instead be spaces. Check in the changes to source control, merge a pull request with the new block list and deploy Merino.","breadcrumbs":"Operations » Modify Wikipedia Suggestions Blocklist » Provider - Rapid Blocklist Addition","id":"131","title":"Provider - Rapid Blocklist Addition"},"132":{"body":"Since the indexer runs at a regular cadence, you do not need to re-run the Airflow job. Adding to the blocklist using the steps above is sufficient to rapidly block a title. The next time the Wikipedia indexer job runs, this title will be excluded during the indexer job. NOTE: There are two blocklists referenced by the Wikipedia Indexer Job: blocklist_file_url: a key contained in the merino/configs/default.toml file that points to a remote block list which encapsulates blocked categories. WIKIPEDIA_TITLE_BLOCKLIST: an application-level list of titles found at /merino/utils/blocklists.py as explained above.","breadcrumbs":"Operations » Modify Wikipedia Suggestions Blocklist » Indexer Job","id":"132","title":"Indexer Job"},"133":{"body":"Investigate the cause of the test failure For functional tests (unit, integration or contract), logs can be found on CircleCI For performance tests (load), insights can be found on Grafana and in the Locust logs. To access the Locust logs see the Distributed GCP Exection - CI Trigger section of the load test documentation. Fix or mitigate the failure If a fix can be identified in a relatively short time, then submit a fix If the failure is caused by a flaky or intermittent functional test and the risk to the end-user experience is low, then the test can be \"skipped\", using the pytestxfail decorator during continued investigation. Example: @pytest.mark.xfail(reason=\"Test Flake Detected (ref: DISCO-####)\") Re-Deploy A fix or mitigation will most likely require a PR merge to the main branch that will automatically trigger the deployment process. If this is not possible, a re-deployment can be initiated manually by triggering the CI pipeline in CircleCI .","breadcrumbs":"Operations » Test Failures in CI » What to do with test failures in CI?","id":"133","title":"What to do with test failures in CI?"},"134":{"body":"To manage configurations and view all documentation for individual config values, please view the default.toml file.","breadcrumbs":"Operations » Configs » Configuring Merino (Operations)","id":"134","title":"Configuring Merino (Operations)"},"135":{"body":"Merino's settings are managed via Dynaconf and can be specified in two ways: a TOML file in the merino/configs/ directory . via environment variables. Environment variables take precedence over the values set in the TOML files. Production environment variables are managed by SRE and defined in the relevant merino-py repo. TOML files set with the same environment name that is currently activated also automatically override defaults. Any config file that is pointed to will override the merino/configs/default.toml file.","breadcrumbs":"Operations » Configs » Settings","id":"135","title":"Settings"},"136":{"body":"These are the settings sources, with later sources overriding earlier ones. A config.py file establishes a Dynaconf instance and environment-specific values are pulled in from the corresponding TOML files and environment variables. Other configurations are established by files that are prefixed with config_*.py, such as config_sentry.py or config_logging.py. Per-environment configuration files are in the configs directory . The environment is selected using the environment variable MERINO_ENV. The settings for that environment are then loaded from configs/${env}.toml, if the file/env exists. The default environment is \"development\". A \"production\" environment is also provided. Local configuration files are not checked into the repository, but if created should be named configs/development.local.toml, following the format of