diff --git a/eden/fs/store/ObjectFetchContext.h b/eden/fs/store/ObjectFetchContext.h index ff5e90ebe8e6c..ce6f22c8dd645 100644 --- a/eden/fs/store/ObjectFetchContext.h +++ b/eden/fs/store/ObjectFetchContext.h @@ -61,7 +61,15 @@ class ObjectFetchContext : public RefCounted { /** * Why did EdenFS fetch these objects? */ - enum Cause : uint8_t { Unknown, Fs, Thrift, Prefetch }; + enum Cause : uint8_t { + Unknown, + /** The request originated from FUSE/NFS/PrjFS */ + Fs, + /** The request originated from a Thrift endpoint */ + Thrift, + /** The request originated from a Thrift prefetch endpoint */ + Prefetch + }; ObjectFetchContext() = default; @@ -130,4 +138,7 @@ class ObjectFetchContext : public RefCounted { ObjectFetchContext& operator=(const ObjectFetchContext&) = delete; }; +// For fbcode/eden/scm/lib/backingstore/src/ffi.rs +using FetchCause = ObjectFetchContext::Cause; + } // namespace facebook::eden diff --git a/eden/fs/store/hg/HgQueuedBackingStore.cpp b/eden/fs/store/hg/HgQueuedBackingStore.cpp index 16cec1e6e0dea..07bbefce79573 100644 --- a/eden/fs/store/hg/HgQueuedBackingStore.cpp +++ b/eden/fs/store/hg/HgQueuedBackingStore.cpp @@ -474,7 +474,7 @@ void HgQueuedBackingStore::getBlobBatch( XLOGF( DBG6, "Failed to import node={} from EdenAPI (batch {}/{}): {}", - folly::hexlify(requests[index]), + folly::hexlify(requests[index].node), index, requests.size(), content.exception().what().toStdString()); @@ -482,7 +482,7 @@ void HgQueuedBackingStore::getBlobBatch( XLOGF( DBG6, "Imported node={} from EdenAPI (batch: {}/{})", - folly::hexlify(requests[index]), + folly::hexlify(requests[index].node), index, requests.size()); } @@ -499,8 +499,8 @@ void HgQueuedBackingStore::getBlobBatch( return; } - XLOGF(DBG9, "Imported Blob node={}", folly::hexlify(requests[index])); - const auto& nodeId = requests[index]; + const auto& nodeId = requests[index].node; + XLOGF(DBG9, "Imported Blob node={}", folly::hexlify(nodeId)); auto& [importRequestList, watch] = importRequestsMap[nodeId]; auto result = content.hasException() ? folly::Try{content.exception()} @@ -615,7 +615,7 @@ void HgQueuedBackingStore::getTreeBatch( XLOGF( DBG6, "Failed to import node={} from EdenAPI (batch tree {}/{}): {}", - folly::hexlify(requests[index]), + folly::hexlify(requests[index].node), index, requests.size(), content.exception().what().toStdString()); @@ -623,7 +623,7 @@ void HgQueuedBackingStore::getTreeBatch( XLOGF( DBG6, "Imported node={} from EdenAPI (batch tree: {}/{})", - folly::hexlify(requests[index]), + folly::hexlify(requests[index].node), index, requests.size()); } @@ -640,8 +640,8 @@ void HgQueuedBackingStore::getTreeBatch( return; } - XLOGF(DBG9, "Imported Tree node={}", folly::hexlify(requests[index])); - const auto& nodeId = requests[index]; + const auto& nodeId = requests[index].node; + XLOGF(DBG9, "Imported Tree node={}", folly::hexlify(nodeId)); auto& [importRequestList, watch] = importRequestsMap[nodeId]; for (auto& importRequest : importRequestList) { auto* treeRequest = @@ -667,7 +667,9 @@ void HgQueuedBackingStore::getTreeBatch( } template -std::pair> +std::pair< + HgQueuedBackingStore::ImportRequestsMap, + std::vector> HgQueuedBackingStore::prepareRequests( const ImportRequestsList& importRequests, const std::string& requestType) { @@ -726,13 +728,27 @@ HgQueuedBackingStore::prepareRequests( } // Indexable vector of nodeIds - required by SaplingNativeBackingStore API. - std::vector requests; - requests.reserve(importRequestsMap.size()); - std::transform( - importRequestsMap.begin(), - importRequestsMap.end(), - std::back_inserter(requests), - [](auto& pair) { return pair.first; }); + // With the current implementation, we can't efficiently deduplicate the + // requests only based on nodeId since multiple requests for the same nodeId + // can have different FetchCauses, which might trigger different behaviors in + // the backingstore. + std::vector requests; + for (const auto& importRequestsIdPair : importRequestsMap) { + // Deduplicate the requests for a given nodeId based on the FetchCause. + std::set seenCausesForId; + const ImportRequestsList& importRequestsForId = + importRequestsIdPair.second.first; + for (const auto& request : importRequestsForId) { + if (request && + (seenCausesForId.find(request->getCause()) == + seenCausesForId.end())) { + requests.push_back(sapling::SaplingRequest{ + importRequestsIdPair.first, request->getCause()}); + // Mark this cause as seen + seenCausesForId.insert(request->getCause()); + } + } + } return std::make_pair(std::move(importRequestsMap), std::move(requests)); } @@ -796,7 +812,7 @@ void HgQueuedBackingStore::getBlobMetadataBatch( XLOGF( DBG6, "Failed to import metadata node={} from EdenAPI (batch {}/{}): {}", - folly::hexlify(requests[index]), + folly::hexlify(requests[index].node), index, requests.size(), auxTry.exception().what().toStdString()); @@ -804,7 +820,7 @@ void HgQueuedBackingStore::getBlobMetadataBatch( XLOGF( DBG6, "Imported metadata node={} from EdenAPI (batch: {}/{})", - folly::hexlify(requests[index]), + folly::hexlify(requests[index].node), index, requests.size()); } @@ -821,9 +837,8 @@ void HgQueuedBackingStore::getBlobMetadataBatch( return; } - XLOGF( - DBG9, "Imported BlobMetadata={}", folly::hexlify(requests[index])); - const auto& nodeId = requests[index]; + const auto& nodeId = requests[index].node; + XLOGF(DBG9, "Imported BlobMetadata={}", folly::hexlify(nodeId)); auto& [importRequestList, watch] = importRequestsMap[nodeId]; folly::Try result; if (auxTry.hasException()) { diff --git a/eden/fs/store/hg/HgQueuedBackingStore.h b/eden/fs/store/hg/HgQueuedBackingStore.h index 93db936276852..375f7b2ca5578 100644 --- a/eden/fs/store/hg/HgQueuedBackingStore.h +++ b/eden/fs/store/hg/HgQueuedBackingStore.h @@ -455,7 +455,8 @@ class HgQueuedBackingStore final : public BackingStore { HgImportObject object) const; template - std::pair> prepareRequests( + std::pair> + prepareRequests( const ImportRequestsList& importRequests, const std::string& requestType); diff --git a/eden/scm/lib/backingstore/TARGETS b/eden/scm/lib/backingstore/TARGETS index 57c70f368aec0..99a30b480a646 100644 --- a/eden/scm/lib/backingstore/TARGETS +++ b/eden/scm/lib/backingstore/TARGETS @@ -10,6 +10,7 @@ cpp_library( headers = glob(["include/**/*.h"]), exported_deps = [ "fbsource//third-party/rust:cxx-core", + "//eden/fs/store:context", "//folly:function", "//folly:range", "//folly:try", @@ -28,6 +29,7 @@ cpp_library( "fbsource//third-party/rust:cxx-core", ":backingstore_bridge@header", ":sapling_native_backingstore-include", + "//eden/fs/store:context", "//folly:range", "//folly:string", "//folly/io:iobuf", @@ -38,7 +40,10 @@ cpp_library( rust_cxx_bridge( name = "backingstore_bridge", src = "src/ffi.rs", - deps = [":sapling_native_backingstore-include"], + deps = [ + ":sapling_native_backingstore-include", + "//eden/fs/store:context", + ], ) rust_library( diff --git a/eden/scm/lib/backingstore/include/SaplingNativeBackingStore.h b/eden/scm/lib/backingstore/include/SaplingNativeBackingStore.h index 5e4b470d7b235..3dd12068ecf86 100644 --- a/eden/scm/lib/backingstore/include/SaplingNativeBackingStore.h +++ b/eden/scm/lib/backingstore/include/SaplingNativeBackingStore.h @@ -14,6 +14,7 @@ #include #include +#include "eden/fs/store/ObjectFetchContext.h" #include "eden/scm/lib/backingstore/src/ffi.rs.h" // @manual namespace folly { @@ -29,11 +30,19 @@ namespace sapling { * object ID again, this can be made into a struct. */ using NodeId = folly::ByteRange; +using FetchCause = facebook::eden::ObjectFetchContext::Cause; + +struct SaplingRequest { + NodeId node; + FetchCause cause; + // TODO: sapling::FetchMode mode; + // TODO: sapling::ClientRequestInfo cri; +}; /** - * List of NodeIds used in batch requests. + * List of SaplingRequests used in batch requests. */ -using NodeIdRange = folly::Range; +using SaplingRequestRange = folly::Range; /** * Storage for a 20-byte hg manifest id. @@ -70,7 +79,7 @@ class SaplingNativeBackingStore { sapling::FetchMode fetch_mode); void getTreeBatch( - NodeIdRange requests, + SaplingRequestRange requests, bool local, folly::FunctionRef>)> resolve); @@ -80,7 +89,7 @@ class SaplingNativeBackingStore { sapling::FetchMode fetchMode); void getBlobBatch( - NodeIdRange requests, + SaplingRequestRange requests, bool local, folly::FunctionRef< void(size_t, folly::Try>)> resolve); @@ -90,7 +99,7 @@ class SaplingNativeBackingStore { bool local); void getBlobMetadataBatch( - NodeIdRange requests, + SaplingRequestRange requests, bool local, folly::FunctionRef>)> resolve); diff --git a/eden/scm/lib/backingstore/src/SaplingNativeBackingStore.cpp b/eden/scm/lib/backingstore/src/SaplingNativeBackingStore.cpp index 941318e3cfc79..999464e513e97 100644 --- a/eden/scm/lib/backingstore/src/SaplingNativeBackingStore.cpp +++ b/eden/scm/lib/backingstore/src/SaplingNativeBackingStore.cpp @@ -78,7 +78,7 @@ folly::Try> SaplingNativeBackingStore::getTree( } void SaplingNativeBackingStore::getTreeBatch( - NodeIdRange requests, + SaplingRequestRange requests, bool local, folly::FunctionRef>)> resolve) { @@ -94,9 +94,10 @@ void SaplingNativeBackingStore::getTreeBatch( std::vector raw_requests; raw_requests.reserve(count); - for (auto& node : requests) { + for (auto& request : requests) { raw_requests.push_back(Request{ - node.data(), + request.node.data(), + request.cause, }); } @@ -132,7 +133,7 @@ folly::Try> SaplingNativeBackingStore::getBlob( } void SaplingNativeBackingStore::getBlobBatch( - NodeIdRange requests, + SaplingRequestRange requests, bool local, folly::FunctionRef>)> resolve) { @@ -147,9 +148,10 @@ void SaplingNativeBackingStore::getBlobBatch( std::vector raw_requests; raw_requests.reserve(count); - for (auto& node : requests) { + for (auto& request : requests) { raw_requests.push_back(Request{ - node.data(), + request.node.data(), + request.cause, }); } @@ -181,7 +183,7 @@ SaplingNativeBackingStore::getBlobMetadata(NodeId node, bool local) { } void SaplingNativeBackingStore::getBlobMetadataBatch( - NodeIdRange requests, + SaplingRequestRange requests, bool local, folly::FunctionRef>)> resolve) { @@ -196,9 +198,10 @@ void SaplingNativeBackingStore::getBlobMetadataBatch( std::vector raw_requests; raw_requests.reserve(count); - for (auto& node : requests) { + for (auto& request : requests) { raw_requests.push_back(Request{ - node.data(), + request.node.data(), + request.cause, }); } diff --git a/eden/scm/lib/backingstore/src/ffi.rs b/eden/scm/lib/backingstore/src/ffi.rs index 54d046502c060..561a545e44e3f 100644 --- a/eden/scm/lib/backingstore/src/ffi.rs +++ b/eden/scm/lib/backingstore/src/ffi.rs @@ -21,6 +21,27 @@ use crate::backingstore::BackingStore; #[cxx::bridge(namespace = sapling)] pub(crate) mod ffi { + // see https://cxx.rs/shared.html#extern-enums + #[namespace = "facebook::eden"] + #[repr(u8)] + pub enum FetchCause { + Unknown, + // The request originated from FUSE/NFS/PrjFS + Fs, + // The request originated from a Thrift endpoint + Thrift, + // The request originated from a Thrift prefetch endpoint + Prefetch, + } + + #[namespace = "facebook::eden"] + unsafe extern "C++" { + include!("eden/fs/store/ObjectFetchContext.h"); + + // The above enum + type FetchCause; + } + pub struct SaplingNativeBackingStoreOptions { allow_retries: bool, } @@ -61,6 +82,9 @@ pub(crate) mod ffi { pub struct Request { node: *const u8, + cause: FetchCause, + // TODO: mode: FetchMode + // TODO: cri: ClientRequestInfo } pub struct Blob { @@ -215,6 +239,7 @@ pub fn sapling_backingstore_get_tree_batch( fetch_mode: ffi::FetchMode, resolver: SharedPtr, ) { + // TODO: pass FetchCause along with they keys to the backingstore let keys: Vec = requests.iter().map(|req| req.key()).collect(); store.get_tree_batch(keys, FetchMode::from(fetch_mode), |idx, result| {