pass FetchCause with backingstore requests

Summary: This diff sets up a struct to pass metadata to the backingstore, this diff now is for passing FetchCause, but we'd probably want to pass FetchMode here, as well as CRI

Reviewed By: jdelliot

Differential Revision: D55106072

fbshipit-source-id: 7ac4fc76246a56861947fda574ce7baff9e043cd

eden/fs/store/ObjectFetchContext.h

@@ -61,7 +61,15 @@ class ObjectFetchContext : public RefCounted {
   /**
    * Why did EdenFS fetch these objects?
    */
-  enum Cause : uint8_t { Unknown, Fs, Thrift, Prefetch };
+  enum Cause : uint8_t {
+    Unknown,
+    /** The request originated from FUSE/NFS/PrjFS */
+    Fs,
+    /** The request originated from a Thrift endpoint */
+    Thrift,
+    /** The request originated from a Thrift prefetch endpoint */
+    Prefetch
+  };
 
   ObjectFetchContext() = default;
 
@@ -130,4 +138,7 @@ class ObjectFetchContext : public RefCounted {
   ObjectFetchContext& operator=(const ObjectFetchContext&) = delete;
 };
 
+// For fbcode/eden/scm/lib/backingstore/src/ffi.rs
+using FetchCause = ObjectFetchContext::Cause;
+
 } // namespace facebook::eden

eden/fs/store/hg/HgQueuedBackingStore.cpp

@@ -474,15 +474,15 @@ void HgQueuedBackingStore::getBlobBatch(
           XLOGF(
               DBG6,
               "Failed to import node={} from EdenAPI (batch {}/{}): {}",
-              folly::hexlify(requests[index]),
+              folly::hexlify(requests[index].node),
               index,
               requests.size(),
               content.exception().what().toStdString());
         } else {
           XLOGF(
               DBG6,
               "Imported node={} from EdenAPI (batch: {}/{})",
-              folly::hexlify(requests[index]),
+              folly::hexlify(requests[index].node),
               index,
               requests.size());
         }
@@ -499,8 +499,8 @@ void HgQueuedBackingStore::getBlobBatch(
           return;
         }
 
-        XLOGF(DBG9, "Imported Blob node={}", folly::hexlify(requests[index]));
-        const auto& nodeId = requests[index];
+        const auto& nodeId = requests[index].node;
+        XLOGF(DBG9, "Imported Blob node={}", folly::hexlify(nodeId));
         auto& [importRequestList, watch] = importRequestsMap[nodeId];
         auto result = content.hasException()
             ? folly::Try<BlobPtr>{content.exception()}
@@ -615,15 +615,15 @@ void HgQueuedBackingStore::getTreeBatch(
           XLOGF(
               DBG6,
               "Failed to import node={} from EdenAPI (batch tree {}/{}): {}",
-              folly::hexlify(requests[index]),
+              folly::hexlify(requests[index].node),
               index,
               requests.size(),
               content.exception().what().toStdString());
         } else {
           XLOGF(
               DBG6,
               "Imported node={} from EdenAPI (batch tree: {}/{})",
-              folly::hexlify(requests[index]),
+              folly::hexlify(requests[index].node),
               index,
               requests.size());
         }
@@ -640,8 +640,8 @@ void HgQueuedBackingStore::getTreeBatch(
           return;
         }
 
-        XLOGF(DBG9, "Imported Tree node={}", folly::hexlify(requests[index]));
-        const auto& nodeId = requests[index];
+        const auto& nodeId = requests[index].node;
+        XLOGF(DBG9, "Imported Tree node={}", folly::hexlify(nodeId));
         auto& [importRequestList, watch] = importRequestsMap[nodeId];
         for (auto& importRequest : importRequestList) {
           auto* treeRequest =
@@ -667,7 +667,9 @@ void HgQueuedBackingStore::getTreeBatch(
 }
 
 template <typename T>
-std::pair<HgQueuedBackingStore::ImportRequestsMap, std::vector<sapling::NodeId>>
+std::pair<
+    HgQueuedBackingStore::ImportRequestsMap,
+    std::vector<sapling::SaplingRequest>>
 HgQueuedBackingStore::prepareRequests(
     const ImportRequestsList& importRequests,
     const std::string& requestType) {
@@ -726,13 +728,27 @@ HgQueuedBackingStore::prepareRequests(
   }
 
   // Indexable vector of nodeIds - required by SaplingNativeBackingStore API.
-  std::vector<sapling::NodeId> requests;
-  requests.reserve(importRequestsMap.size());
-  std::transform(
-      importRequestsMap.begin(),
-      importRequestsMap.end(),
-      std::back_inserter(requests),
-      [](auto& pair) { return pair.first; });
+  // With the current implementation, we can't efficiently deduplicate the
+  // requests only based on nodeId since multiple requests for the same nodeId
+  // can have different FetchCauses, which might trigger different behaviors in
+  // the backingstore.
+  std::vector<sapling::SaplingRequest> requests;
+  for (const auto& importRequestsIdPair : importRequestsMap) {
+    // Deduplicate the requests for a given nodeId based on the FetchCause.
+    std::set<ObjectFetchContext::Cause> seenCausesForId;
+    const ImportRequestsList& importRequestsForId =
+        importRequestsIdPair.second.first;
+    for (const auto& request : importRequestsForId) {
+      if (request &&
+          (seenCausesForId.find(request->getCause()) ==
+           seenCausesForId.end())) {
+        requests.push_back(sapling::SaplingRequest{
+            importRequestsIdPair.first, request->getCause()});
+        // Mark this cause as seen
+        seenCausesForId.insert(request->getCause());
+      }
+    }
+  }
 
   return std::make_pair(std::move(importRequestsMap), std::move(requests));
 }
@@ -796,15 +812,15 @@ void HgQueuedBackingStore::getBlobMetadataBatch(
           XLOGF(
               DBG6,
               "Failed to import metadata node={} from EdenAPI (batch {}/{}): {}",
-              folly::hexlify(requests[index]),
+              folly::hexlify(requests[index].node),
               index,
               requests.size(),
               auxTry.exception().what().toStdString());
         } else {
           XLOGF(
               DBG6,
               "Imported metadata node={} from EdenAPI (batch: {}/{})",
-              folly::hexlify(requests[index]),
+              folly::hexlify(requests[index].node),
               index,
               requests.size());
         }
@@ -821,9 +837,8 @@ void HgQueuedBackingStore::getBlobMetadataBatch(
           return;
         }
 
-        XLOGF(
-            DBG9, "Imported BlobMetadata={}", folly::hexlify(requests[index]));
-        const auto& nodeId = requests[index];
+        const auto& nodeId = requests[index].node;
+        XLOGF(DBG9, "Imported BlobMetadata={}", folly::hexlify(nodeId));
         auto& [importRequestList, watch] = importRequestsMap[nodeId];
         folly::Try<BlobMetadataPtr> result;
         if (auxTry.hasException()) {

eden/fs/store/hg/HgQueuedBackingStore.h

@@ -455,7 +455,8 @@ class HgQueuedBackingStore final : public BackingStore {
       HgImportObject object) const;
 
   template <typename T>
-  std::pair<ImportRequestsMap, std::vector<sapling::NodeId>> prepareRequests(
+  std::pair<ImportRequestsMap, std::vector<sapling::SaplingRequest>>
+  prepareRequests(
       const ImportRequestsList& importRequests,
       const std::string& requestType);
 

eden/scm/lib/backingstore/TARGETS

@@ -10,6 +10,7 @@ cpp_library(
     headers = glob(["include/**/*.h"]),
     exported_deps = [
         "fbsource//third-party/rust:cxx-core",
+        "//eden/fs/store:context",
         "//folly:function",
         "//folly:range",
         "//folly:try",
@@ -28,6 +29,7 @@ cpp_library(
         "fbsource//third-party/rust:cxx-core",
         ":backingstore_bridge@header",
         ":sapling_native_backingstore-include",
+        "//eden/fs/store:context",
         "//folly:range",
         "//folly:string",
         "//folly/io:iobuf",
@@ -38,7 +40,10 @@ cpp_library(
 rust_cxx_bridge(
     name = "backingstore_bridge",
     src = "src/ffi.rs",
-    deps = [":sapling_native_backingstore-include"],
+    deps = [
+        ":sapling_native_backingstore-include",
+        "//eden/fs/store:context",
+    ],
 )
 
 rust_library(

eden/scm/lib/backingstore/include/SaplingNativeBackingStore.h

@@ -14,6 +14,7 @@
 #include <optional>
 #include <string_view>
 
+#include "eden/fs/store/ObjectFetchContext.h"
 #include "eden/scm/lib/backingstore/src/ffi.rs.h" // @manual
 
 namespace folly {
@@ -29,11 +30,19 @@ namespace sapling {
  * object ID again, this can be made into a struct.
  */
 using NodeId = folly::ByteRange;
+using FetchCause = facebook::eden::ObjectFetchContext::Cause;
+
+struct SaplingRequest {
+  NodeId node;
+  FetchCause cause;
+  // TODO: sapling::FetchMode mode;
+  // TODO: sapling::ClientRequestInfo cri;
+};
 
 /**
- * List of NodeIds used in batch requests.
+ * List of SaplingRequests used in batch requests.
  */
-using NodeIdRange = folly::Range<const NodeId*>;
+using SaplingRequestRange = folly::Range<const SaplingRequest*>;
 
 /**
  * Storage for a 20-byte hg manifest id.
@@ -70,7 +79,7 @@ class SaplingNativeBackingStore {
       sapling::FetchMode fetch_mode);
 
   void getTreeBatch(
-      NodeIdRange requests,
+      SaplingRequestRange requests,
       bool local,
       folly::FunctionRef<void(size_t, folly::Try<std::shared_ptr<Tree>>)>
           resolve);
@@ -80,7 +89,7 @@ class SaplingNativeBackingStore {
       sapling::FetchMode fetchMode);
 
   void getBlobBatch(
-      NodeIdRange requests,
+      SaplingRequestRange requests,
       bool local,
       folly::FunctionRef<
           void(size_t, folly::Try<std::unique_ptr<folly::IOBuf>>)> resolve);
@@ -90,7 +99,7 @@ class SaplingNativeBackingStore {
       bool local);
 
   void getBlobMetadataBatch(
-      NodeIdRange requests,
+      SaplingRequestRange requests,
       bool local,
       folly::FunctionRef<void(size_t, folly::Try<std::shared_ptr<FileAuxData>>)>
           resolve);

eden/scm/lib/backingstore/src/SaplingNativeBackingStore.cpp

@@ -78,7 +78,7 @@ folly::Try<std::shared_ptr<Tree>> SaplingNativeBackingStore::getTree(
 }
 
 void SaplingNativeBackingStore::getTreeBatch(
-    NodeIdRange requests,
+    SaplingRequestRange requests,
     bool local,
     folly::FunctionRef<void(size_t, folly::Try<std::shared_ptr<Tree>>)>
         resolve) {
@@ -94,9 +94,10 @@ void SaplingNativeBackingStore::getTreeBatch(
 
   std::vector<Request> raw_requests;
   raw_requests.reserve(count);
-  for (auto& node : requests) {
+  for (auto& request : requests) {
     raw_requests.push_back(Request{
-        node.data(),
+        request.node.data(),
+        request.cause,
     });
   }
 
@@ -132,7 +133,7 @@ folly::Try<std::unique_ptr<folly::IOBuf>> SaplingNativeBackingStore::getBlob(
 }
 
 void SaplingNativeBackingStore::getBlobBatch(
-    NodeIdRange requests,
+    SaplingRequestRange requests,
     bool local,
     folly::FunctionRef<void(size_t, folly::Try<std::unique_ptr<folly::IOBuf>>)>
         resolve) {
@@ -147,9 +148,10 @@ void SaplingNativeBackingStore::getBlobBatch(
 
   std::vector<Request> raw_requests;
   raw_requests.reserve(count);
-  for (auto& node : requests) {
+  for (auto& request : requests) {
     raw_requests.push_back(Request{
-        node.data(),
+        request.node.data(),
+        request.cause,
     });
   }
 
@@ -181,7 +183,7 @@ SaplingNativeBackingStore::getBlobMetadata(NodeId node, bool local) {
 }
 
 void SaplingNativeBackingStore::getBlobMetadataBatch(
-    NodeIdRange requests,
+    SaplingRequestRange requests,
     bool local,
     folly::FunctionRef<void(size_t, folly::Try<std::shared_ptr<FileAuxData>>)>
         resolve) {
@@ -196,9 +198,10 @@ void SaplingNativeBackingStore::getBlobMetadataBatch(
 
   std::vector<Request> raw_requests;
   raw_requests.reserve(count);
-  for (auto& node : requests) {
+  for (auto& request : requests) {
     raw_requests.push_back(Request{
-        node.data(),
+        request.node.data(),
+        request.cause,
     });
   }
 

eden/scm/lib/backingstore/src/ffi.rs

@@ -21,6 +21,27 @@ use crate::backingstore::BackingStore;
 
 #[cxx::bridge(namespace = sapling)]
 pub(crate) mod ffi {
+    // see https://cxx.rs/shared.html#extern-enums
+    #[namespace = "facebook::eden"]
+    #[repr(u8)]
+    pub enum FetchCause {
+        Unknown,
+        // The request originated from FUSE/NFS/PrjFS
+        Fs,
+        // The request originated from a Thrift endpoint
+        Thrift,
+        // The request originated from a Thrift prefetch endpoint
+        Prefetch,
+    }
+
+    #[namespace = "facebook::eden"]
+    unsafe extern "C++" {
+        include!("eden/fs/store/ObjectFetchContext.h");
+
+        // The above enum
+        type FetchCause;
+    }
+
     pub struct SaplingNativeBackingStoreOptions {
         allow_retries: bool,
     }
@@ -61,6 +82,9 @@ pub(crate) mod ffi {
 
     pub struct Request {
         node: *const u8,
+        cause: FetchCause,
+        // TODO: mode: FetchMode
+        // TODO: cri: ClientRequestInfo
     }
 
     pub struct Blob {
@@ -215,6 +239,7 @@ pub fn sapling_backingstore_get_tree_batch(
     fetch_mode: ffi::FetchMode,
     resolver: SharedPtr<ffi::GetTreeBatchResolver>,
 ) {
+    // TODO: pass FetchCause along with they keys to the backingstore
     let keys: Vec<Key> = requests.iter().map(|req| req.key()).collect();
 
     store.get_tree_batch(keys, FetchMode::from(fetch_mode), |idx, result| {