derived_data_service: make the number of failure retries configurable…

… using a JK when enqueuing items into the derivation queue

Summary: The number of retries we do for failures encountered when enqueuing items into the derivation queue is currently hardcoded to 3. Let's turn it into a JK instead in case we want to tune it.

Reviewed By: singhsrb

Differential Revision: D64830451

fbshipit-source-id: 10723547ac5847be889f06957e07afb190781e4a

eden/mononoke/mononoke_macros/just_knobs_defaults/just_knobs.json

@@ -81,6 +81,7 @@
     "scm/mononoke_memcache_sitevers:mutable_renames": 0,
     "scm/mononoke_memcache_sitevers:sql": 0,
     "scm/mononoke_memcache_sitevers:synced_commit_mapping": 0,
-    "scm/mononoke:pushredirection_config_cache_ttl_secs": 0
+    "scm/mononoke:pushredirection_config_cache_ttl_secs": 0,
+    "scm/mononoke:build_underived_batched_graph_max_failed_attempts": 3
   }
 }

eden/mononoke/repo_attributes/repo_derivation_queues/Cargo.toml

@@ -23,6 +23,7 @@ ephemeral_blobstore = { version = "0.1.0", path = "../../blobstore/ephemeral_blo
 facet = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
 fbthrift = { version = "0.0.1+unstable", git = "https://github.com/facebook/fbthrift.git", branch = "main" }
 futures = { version = "0.3.30", features = ["async-await", "compat"] }
+justknobs = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
 mononoke_types = { version = "0.1.0", path = "../../mononoke_types" }
 parking_lot = { version = "0.12.1", features = ["send_guard"] }
 slog = { version = "2.7", features = ["max_level_trace", "nested-values"] }

eden/mononoke/repo_attributes/repo_derivation_queues/TARGETS

@@ -16,6 +16,7 @@ rust_library(
         "fbsource//third-party/rust:tokio",
         "//common/rust/shed/cloned:cloned",
         "//common/rust/shed/facet:facet",
+        "//common/rust/shed/justknobs_stub:justknobs",
         "//common/rust/zeus:zeus_client",
         "//eden/mononoke/blobstore:ephemeral_blobstore",
         "//eden/mononoke/common/bounded_traversal:bounded_traversal",

eden/mononoke/repo_attributes/repo_derivation_queues/src/underived.rs

@@ -31,8 +31,6 @@ use crate::DerivationQueue;
 use crate::EnqueueResponse;
 use crate::InternalError;
 
-const MAX_FAILED_ATTEMPTS: u64 = 3;
-
 // Generation number starts with 1, so we need to account for it by offsetting
 // We also need to multiply index additionally by (batch size)
 // to get the generation number of root for each bat
@@ -124,13 +122,16 @@ pub async fn build_underived_batched_graph<'a>(
                     deps.collect(),
                     ctx.metadata().client_info(),
                 )?;
+
+                let max_failed_attemps = justknobs::get_as::<u64>("scm/mononoke:build_underived_batched_graph_max_failed_attempts", None)?;
+
                 let mut cur_item = Some(item.clone());
                 // Upstream batch will depend on this cs
                 let mut upstream_dep = item.id().clone();
                 let mut failed_attempt = 0;
                 let mut err_msg = None;
                 while let Some(item) = cur_item {
-                    if failed_attempt >= MAX_FAILED_ATTEMPTS {
+                    if failed_attempt >= max_failed_attemps {
                         return Err(anyhow!(
                             "Couldn't enqueue item {:?} into zeus after {} attempts. Last err: {:?}",
                             item,