diff --git a/cloud/blockstore/config/disk.proto b/cloud/blockstore/config/disk.proto index e6585e26984..c56bdad4676 100644 --- a/cloud/blockstore/config/disk.proto +++ b/cloud/blockstore/config/disk.proto @@ -201,7 +201,7 @@ message TDiskAgentConfig optional TSpdkEnvConfig SpdkEnvConfig = 19; // RDMA target config. - optional TRdmaTarget RdmaTarget = 20; + optional TRdmaTarget RdmaTarget = 20; // deprecated // Use flock to lock devices on agent initialization. optional bool DeviceLockingEnabled = 21; diff --git a/cloud/blockstore/config/rdma.proto b/cloud/blockstore/config/rdma.proto index f6cca1decfe..e1010d9abf2 100644 --- a/cloud/blockstore/config/rdma.proto +++ b/cloud/blockstore/config/rdma.proto @@ -20,12 +20,12 @@ message TRdmaServer uint32 Backlog = 1; uint32 QueueSize = 2; uint32 MaxBufferSize = 3; - uint64 KeepAliveTimeout = 4; // in milliseconds + uint64 KeepAliveTimeout = 4; // in milliseconds EWaitMode WaitMode = 5; uint32 PollerThreads = 6; - uint32 MaxInflightBytes = 7; // per client - uint64 AdaptiveWaitSleepDelay = 8; // in microseconds - uint64 AdaptiveWaitSleepDuration = 9; // in microseconds + uint32 MaxInflightBytes = 7; // per client + uint64 AdaptiveWaitSleepDelay = 8; // in microseconds + uint64 AdaptiveWaitSleepDuration = 9; // in microseconds bool AlignedDataEnabled = 10; } @@ -35,24 +35,26 @@ message TRdmaClient uint32 MaxBufferSize = 2; uint32 PollerThreads = 3; EWaitMode WaitMode = 4; - uint64 AdaptiveWaitSleepDelay = 5; // in microseconds - uint64 AdaptiveWaitSleepDuration = 6; // in microseconds + uint64 AdaptiveWaitSleepDelay = 5; // in microseconds + uint64 AdaptiveWaitSleepDuration = 6; // in microseconds bool AlignedDataEnabled = 7; } message TRdmaTarget { TRdmaEndpoint Endpoint = 1; - TRdmaServer Server = 2; + TRdmaServer Server = 2; // deprecated + uint32 WorkerThreads = 3; } message TRdmaConfig { bool ClientEnabled = 1; - TRdmaClient Client = 2; bool ServerEnabled = 3; - TRdmaServer Server = 4; + + bool DiskAgentTargetEnabled = 5; + TRdmaTarget DiskAgentTarget = 6; } diff --git a/cloud/blockstore/libs/disk_agent/config_initializer.cpp b/cloud/blockstore/libs/disk_agent/config_initializer.cpp index be4ebb03afb..f50e1d29b63 100644 --- a/cloud/blockstore/libs/disk_agent/config_initializer.cpp +++ b/cloud/blockstore/libs/disk_agent/config_initializer.cpp @@ -218,12 +218,23 @@ void TConfigInitializer::InitRdmaConfig() if (Options->RdmaConfig) { ParseProtoTextFromFileRobust(Options->RdmaConfig, rdmaConfig); + + // inherit the target from DiskAgentConfig to smooth out the transition + if (DiskAgentConfig->DeprecatedHasRdmaTarget()) { + rdmaConfig.SetDiskAgentTargetEnabled(true); + } } else { // no rdma config file is given fallback to legacy config if (DiskAgentConfig->DeprecatedHasRdmaTarget()) { + const auto& oldTarget = DiskAgentConfig->DeprecatedGetRdmaTarget(); + auto* newTarget = rdmaConfig.MutableDiskAgentTarget(); + rdmaConfig.SetServerEnabled(true); - const auto& rdmaTarget = DiskAgentConfig->DeprecatedGetRdmaTarget(); - rdmaConfig.MutableServer()->CopyFrom(rdmaTarget.GetServer()); + rdmaConfig.MutableServer()->CopyFrom(oldTarget.GetServer()); + + rdmaConfig.SetDiskAgentTargetEnabled(true); + newTarget->MutableEndpoint()->CopyFrom(oldTarget.GetEndpoint()); + newTarget->SetWorkerThreads(oldTarget.GetWorkerThreads()); } } diff --git a/cloud/blockstore/libs/rdma/iface/config.h b/cloud/blockstore/libs/rdma/iface/config.h index b929c9b06dd..141adfc538c 100644 --- a/cloud/blockstore/libs/rdma/iface/config.h +++ b/cloud/blockstore/libs/rdma/iface/config.h @@ -35,6 +35,16 @@ class TRdmaConfig { return Config.GetServer(); } + + bool GetDiskAgentTargetEnabled() const + { + return Config.GetDiskAgentTargetEnabled(); + } + + const auto& GetDiskAgentTarget() const + { + return Config.GetDiskAgentTarget(); + } }; } // namespace NCloud::NBlockStore::NRdma diff --git a/cloud/blockstore/libs/rdma_test/rdma_test_environment.cpp b/cloud/blockstore/libs/rdma_test/rdma_test_environment.cpp index 61c68520bca..239e2de2a3d 100644 --- a/cloud/blockstore/libs/rdma_test/rdma_test_environment.cpp +++ b/cloud/blockstore/libs/rdma_test/rdma_test_environment.cpp @@ -39,21 +39,25 @@ TRdmaTestEnvironment::TRdmaTestEnvironment(size_t deviceSize, ui32 poolSize) "vol0", 0); - NProto::TRdmaEndpoint config; - config.SetHost(Host); - config.SetPort(Port); - - TRdmaTargetConfig rdmaTargetConfig{ - true, - TOldRequestCounters{ - Counters->GetCounter("Delayed"), - Counters->GetCounter("Rejected"), - Counters->GetCounter("Already")}, - poolSize}; + NProto::TRdmaTarget target; + target.MutableEndpoint()->SetHost(Host); + target.MutableEndpoint()->SetPort(Port); + target.SetWorkerThreads(poolSize); + + constexpr bool rejectLateRequests = true; + + auto rdmaTargetConfig = std::make_shared( + rejectLateRequests, + target); + + TOldRequestCounters oldRequestCounters{ + Counters->GetCounter("Delayed"), + Counters->GetCounter("Rejected"), + Counters->GetCounter("Already")}; RdmaTarget = CreateRdmaTarget( - std::move(config), std::move(rdmaTargetConfig), + std::move(oldRequestCounters), Logging, Server, std::move(deviceClient), diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor.h b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor.h index 02f594b2952..6e04430b694 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor.h +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_init.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_init.cpp index 1fc21a5ffc7..4688c44c416 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_init.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_init.cpp @@ -22,10 +22,20 @@ using namespace NActors; void TDiskAgentActor::InitAgent(const TActorContext& ctx) { + Y_DEBUG_ABORT_UNLESS( + OldRequestCounters.Delayed && OldRequestCounters.Rejected && + OldRequestCounters.Already); + + TRdmaTargetConfigPtr rdmaTargetConfig = nullptr; + if (RdmaConfig && RdmaConfig->GetDiskAgentTargetEnabled()) { + rdmaTargetConfig = std::make_shared( + Config->GetRejectLateRequestsAtDiskAgentEnabled(), + RdmaConfig->GetDiskAgentTarget()); + } + State = std::make_unique( Config, AgentConfig, - RdmaConfig, Spdk, Allocator, StorageProvider, @@ -33,21 +43,14 @@ void TDiskAgentActor::InitAgent(const TActorContext& ctx) BlockDigestGenerator, Logging, RdmaServer, - NvmeManager); - - Y_DEBUG_ABORT_UNLESS( - OldRequestCounters.Delayed && OldRequestCounters.Rejected && - OldRequestCounters.Already); - TRdmaTargetConfig rdmaTargetConfig{ - RejectLateRequestsAtDiskAgentEnabled, - OldRequestCounters}; - - auto result = State->Initialize(std::move(rdmaTargetConfig)); + NvmeManager, + std::move(rdmaTargetConfig), + OldRequestCounters); auto* actorSystem = ctx.ActorSystem(); auto replyTo = ctx.SelfID; - result.Subscribe([=] (auto future) { + State->Initialize().Subscribe([=] (auto future) { using TCompletionEvent = TEvDiskAgentPrivate::TEvInitAgentCompleted; NProto::TError error; diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.cpp index 2659266e5c5..e452bfe8da3 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.cpp @@ -239,7 +239,6 @@ TVector ComputeDigest( TDiskAgentState::TDiskAgentState( TStorageConfigPtr storageConfig, TDiskAgentConfigPtr agentConfig, - NRdma::TRdmaConfigPtr rdmaConfig, NSpdk::ISpdkEnvPtr spdk, ICachingAllocatorPtr allocator, IStorageProviderPtr storageProvider, @@ -247,10 +246,11 @@ TDiskAgentState::TDiskAgentState( IBlockDigestGeneratorPtr blockDigestGenerator, ILoggingServicePtr logging, NRdma::IServerPtr rdmaServer, - NNvme::INvmeManagerPtr nvmeManager) + NNvme::INvmeManagerPtr nvmeManager, + TRdmaTargetConfigPtr rdmaTargetConfig, + TOldRequestCounters oldRequestCounters) : StorageConfig(std::move(storageConfig)) , AgentConfig(std::move(agentConfig)) - , RdmaConfig(std::move(rdmaConfig)) , Spdk(std::move(spdk)) , Allocator(std::move(allocator)) , StorageProvider(std::move(storageProvider)) @@ -260,6 +260,8 @@ TDiskAgentState::TDiskAgentState( , Log(Logging->CreateLog("BLOCKSTORE_DISK_AGENT")) , RdmaServer(std::move(rdmaServer)) , NvmeManager(std::move(nvmeManager)) + , RdmaTargetConfig(std::move(rdmaTargetConfig)) + , OldRequestCounters(std::move(oldRequestCounters)) { } @@ -419,25 +421,21 @@ TFuture TDiskAgentState::InitAioStorage() }); } -void TDiskAgentState::InitRdmaTarget(TRdmaTargetConfig rdmaTargetConfig) +void TDiskAgentState::InitRdmaTarget() { - if (RdmaServer) { + if (RdmaServer && RdmaTargetConfig) { THashMap devices; - auto endpoint = AgentConfig->GetRdmaEndpoint(); - - if (endpoint.GetHost().empty()) { - endpoint.SetHost(FQDNHostName()); - } - for (auto& [uuid, state]: Devices) { - state.Config.MutableRdmaEndpoint()->CopyFrom(endpoint); + auto* endpoint = state.Config.MutableRdmaEndpoint(); + endpoint->SetHost(RdmaTargetConfig->Host); + endpoint->SetPort(RdmaTargetConfig->Port); devices.emplace(uuid, state.StorageAdapter); } RdmaTarget = CreateRdmaTarget( - endpoint, - std::move(rdmaTargetConfig), + RdmaTargetConfig, + OldRequestCounters, Logging, RdmaServer, DeviceClient, @@ -447,13 +445,12 @@ void TDiskAgentState::InitRdmaTarget(TRdmaTargetConfig rdmaTargetConfig) } } -TFuture TDiskAgentState::Initialize( - TRdmaTargetConfig rdmaTargetConfig) +TFuture TDiskAgentState::Initialize() { auto future = Spdk ? InitSpdkStorage() : InitAioStorage(); return future.Subscribe( - [this, rdmaTargetConfig = std::move(rdmaTargetConfig)](auto) mutable + [this](auto) mutable { TVector uuids(Reserve(Devices.size())); for (const auto& x: Devices) { @@ -465,7 +462,7 @@ TFuture TDiskAgentState::Initialize( std::move(uuids), Logging->CreateLog("BLOCKSTORE_DISK_AGENT")); - InitRdmaTarget(std::move(rdmaTargetConfig)); + InitRdmaTarget(); RestoreSessions(*DeviceClient); }); diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.h b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.h index 7637e57ec13..3cb018e3309 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.h +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.h @@ -44,7 +44,6 @@ class TDiskAgentState private: const TStorageConfigPtr StorageConfig; const TDiskAgentConfigPtr AgentConfig; - const NRdma::TRdmaConfigPtr RdmaConfig; const NSpdk::ISpdkEnvPtr Spdk; const ICachingAllocatorPtr Allocator; const IStorageProviderPtr StorageProvider; @@ -64,11 +63,13 @@ class TDiskAgentState ui32 InitErrorsCount = 0; bool PartiallySuspended = false; + TRdmaTargetConfigPtr RdmaTargetConfig; + TOldRequestCounters OldRequestCounters; + public: TDiskAgentState( TStorageConfigPtr storageConfig, TDiskAgentConfigPtr agentConfig, - NRdma::TRdmaConfigPtr rdmaConfig, NSpdk::ISpdkEnvPtr spdk, ICachingAllocatorPtr allocator, IStorageProviderPtr storageProvider, @@ -76,7 +77,9 @@ class TDiskAgentState IBlockDigestGeneratorPtr blockDigestGenerator, ILoggingServicePtr logging, NRdma::IServerPtr rdmaServer, - NNvme::INvmeManagerPtr nvmeManager); + NNvme::INvmeManagerPtr nvmeManager, + TRdmaTargetConfigPtr rdmaTargetConfig, + TOldRequestCounters oldRequestCounters); struct TInitializeResult { @@ -86,8 +89,7 @@ class TDiskAgentState TDeviceGuard Guard; }; - NThreading::TFuture Initialize( - TRdmaTargetConfig rdmaTargetConfig); + NThreading::TFuture Initialize(); NThreading::TFuture CollectStats(); @@ -178,7 +180,7 @@ class TDiskAgentState NThreading::TFuture InitSpdkStorage(); NThreading::TFuture InitAioStorage(); - void InitRdmaTarget(TRdmaTargetConfig rdmaTargetConfig); + void InitRdmaTarget(); void RestoreSessions(TDeviceClient& client) const; }; diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state_ut.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state_ut.cpp index 4c6cfcdb368..2486186853b 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state_ut.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state_ut.cpp @@ -152,7 +152,6 @@ auto CreateDiskAgentStateSpdk(TDiskAgentConfigPtr config) return std::make_unique( CreateStorageConfig(), std::move(config), - nullptr, // rdmaConfig NSpdk::CreateEnvStub(), CreateTestAllocator(), nullptr, // storageProvider @@ -160,7 +159,9 @@ auto CreateDiskAgentStateSpdk(TDiskAgentConfigPtr config) CreateBlockDigestGeneratorStub(), CreateLoggingService("console"), nullptr, // rdmaServer - nullptr); // nvmeManager + nullptr, // nvmeManager + nullptr, // rdmaTargetConfig + TOldRequestCounters()); } //////////////////////////////////////////////////////////////////////////////// @@ -342,7 +343,6 @@ struct TFiles return std::make_unique( CreateStorageConfig(), std::move(config), - nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), NServer::CreateNullStorageProvider(), @@ -350,7 +350,9 @@ struct TFiles CreateBlockDigestGeneratorStub(), CreateLoggingService("console"), nullptr, // rdmaServer - NvmeManager); + NvmeManager, + nullptr, // rdmaTargetConfig + TOldRequestCounters()); } }; @@ -383,7 +385,7 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) { void ShouldInitialize(TDiskAgentState& state, bool checkSerialNumbers) { - auto future = state.Initialize({}); + auto future = state.Initialize(); const auto& r = future.GetValue(WaitTimeout); UNIT_ASSERT(r.Errors.empty()); @@ -512,7 +514,6 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) TDiskAgentState state( CreateStorageConfig(), config, - nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), std::make_shared(THashSet{ @@ -523,9 +524,11 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) CreateBlockDigestGeneratorStub(), CreateLoggingService("console"), nullptr, // rdmaServer - NvmeManager); + NvmeManager, + nullptr, // rdmaTargetConfig + TOldRequestCounters()); - auto future = state.Initialize({}); + auto future = state.Initialize(); const auto& r = future.GetValue(WaitTimeout); UNIT_ASSERT_VALUES_EQUAL(1, r.Errors.size()); @@ -724,7 +727,6 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) TDiskAgentState state( CreateStorageConfig(), config, - nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), std::make_shared(THashSet{ @@ -736,9 +738,11 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) CreateBlockDigestGeneratorStub(), CreateLoggingService("console"), nullptr, // rdmaServer - NvmeManager); + NvmeManager, + nullptr, // rdmaTargetConfig + TOldRequestCounters()); - auto future = state.Initialize({}); + auto future = state.Initialize(); const auto& r = future.GetValue(WaitTimeout); if (checkLockedDevices) { @@ -795,7 +799,6 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) TDiskAgentState state( CreateStorageConfig(), std::make_shared(std::move(config), "rack"), - nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), NServer::CreateNullStorageProvider(), @@ -803,9 +806,11 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) CreateBlockDigestGeneratorStub(), CreateLoggingService("console"), nullptr, // rdmaServer - std::make_shared()); + std::make_shared(), + nullptr, // rdmaTargetConfig + TOldRequestCounters()); - auto future = state.Initialize({}); + auto future = state.Initialize(); const auto& r = future.GetValue(WaitTimeout); UNIT_ASSERT_VALUES_EQUAL(3, r.Errors.size()); @@ -823,7 +828,7 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) CreateNullConfig({ .Files = Nvme3s, .AcquireRequired = true }) ); - auto future = state->Initialize({}); + auto future = state->Initialize(); const auto& r = future.GetValue(WaitTimeout); UNIT_ASSERT(r.Errors.empty()); @@ -1215,7 +1220,6 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) TDiskAgentState state( CreateStorageConfig(), config, - nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), std::make_shared(storageState), @@ -1223,9 +1227,11 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) CreateBlockDigestGeneratorStub(), CreateLoggingService("console"), nullptr, // rdmaServer - NvmeManager); + NvmeManager, + nullptr, // rdmaTargetConfig + TOldRequestCounters()); - auto future = state.Initialize({}); + auto future = state.Initialize(); const auto& r = future.GetValue(WaitTimeout); UNIT_ASSERT_VALUES_EQUAL(0, r.Errors.size()); @@ -1373,7 +1379,6 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) auto state = std::make_unique( CreateStorageConfig(), config, - nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), std::make_shared(storageState), @@ -1381,9 +1386,11 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) CreateBlockDigestGeneratorStub(), CreateLoggingService("console"), nullptr, // rdmaServer - NvmeManager); + NvmeManager, + nullptr, // rdmaTargetConfig + TOldRequestCounters()); - auto future = state->Initialize({}); + auto future = state->Initialize(); const auto& result = future.GetValue(WaitTimeout); UNIT_ASSERT_VALUES_EQUAL_C(0, result.Errors.size(), JoinSeq(", ", result.Errors)); @@ -1453,7 +1460,6 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) auto state = std::make_unique( CreateStorageConfig(), config, - nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), std::make_shared(storageState), @@ -1461,9 +1467,11 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) CreateBlockDigestGeneratorStub(), CreateLoggingService("console"), nullptr, // rdmaServer - NvmeManager); + NvmeManager, + nullptr, // rdmaTargetConfig + TOldRequestCounters()); - auto future = state->Initialize({}); + auto future = state->Initialize(); const auto& result = future.GetValue(WaitTimeout); UNIT_ASSERT_VALUES_EQUAL(1, result.Errors.size()); @@ -1498,7 +1506,7 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) TFileOutput(file).Write("<<< garbage >>>"); } - auto future = state->Initialize({}); + auto future = state->Initialize(); UNIT_ASSERT_EXCEPTION_CONTAINS( future.GetValue(), @@ -1533,7 +1541,6 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) .DiscoveryConfig = discoveryConfig, .CachedConfigPath = CachedConfigPath }), - nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), std::make_shared(storageState), @@ -1541,13 +1548,15 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) CreateBlockDigestGeneratorStub(), CreateLoggingService("console"), nullptr, // rdmaServer - NvmeManager); + NvmeManager, + nullptr, // rdmaTargetConfig + TOldRequestCounters()); }; { auto state = newState(discovery); - auto future = state->Initialize({}); + auto future = state->Initialize(); const auto& result = future.GetValue(WaitTimeout); UNIT_ASSERT_VALUES_EQUAL_C(0, result.Errors.size(), JoinSeq(", ", result.Errors)); @@ -1574,7 +1583,7 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) { auto state = newState(discovery); - auto future = state->Initialize({}); + auto future = state->Initialize(); const auto& result = future.GetValue(WaitTimeout); UNIT_ASSERT_VALUES_EQUAL_C(0, result.Errors.size(), JoinSeq(", ", result.Errors)); @@ -1627,7 +1636,7 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) }) ); - auto future = state->Initialize({}); + auto future = state->Initialize(); const auto& r = future.GetValue(WaitTimeout); UNIT_ASSERT_VALUES_EQUAL_C(0, r.Errors.size(), r.Errors[0]); diff --git a/cloud/blockstore/libs/storage/disk_agent/model/config.h b/cloud/blockstore/libs/storage/disk_agent/model/config.h index ccccb94219d..c937c7d7147 100644 --- a/cloud/blockstore/libs/storage/disk_agent/model/config.h +++ b/cloud/blockstore/libs/storage/disk_agent/model/config.h @@ -55,11 +55,6 @@ class TDiskAgentConfig return Config.GetNvmeTarget(); } - const auto& GetRdmaEndpoint() const - { - return Config.GetRdmaTarget().GetEndpoint(); - } - const auto& DeprecatedGetRdmaTarget() const { return Config.GetRdmaTarget(); diff --git a/cloud/blockstore/libs/storage/disk_agent/rdma_target.cpp b/cloud/blockstore/libs/storage/disk_agent/rdma_target.cpp index 768d1eefa38..2be9327d053 100644 --- a/cloud/blockstore/libs/storage/disk_agent/rdma_target.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/rdma_target.cpp @@ -105,13 +105,13 @@ struct TDeviceData THashMap MakeDevices( THashMap devices, - const TRdmaTargetConfig& rdmaTargetConfig) + TOldRequestCounters oldRequestCounters) { THashMap result; for (auto& [deviceUUID, storageAdapter]: devices) { TSynchronizedData synchronizedData{ TRecentBlocksTracker{deviceUUID}, - rdmaTargetConfig.OldRequestCounters}; + oldRequestCounters}; TDeviceData device{ std::move(storageAdapter), @@ -144,19 +144,20 @@ class TRequestHandler final const NRdma::TProtoMessageSerializer* Serializer = TBlockStoreProtocol::Serializer(); - const bool RejectLateRequestsAtDiskAgentEnabled; + const bool RejectLateRequests; public: TRequestHandler( THashMap devices, ITaskQueuePtr taskQueue, TDeviceClientPtr deviceClient, - TRdmaTargetConfig rdmaTargetConfig) - : Devices(MakeDevices(std::move(devices), rdmaTargetConfig)) + TOldRequestCounters oldRequestCounters, + bool rejectLateRequests) + : Devices( + MakeDevices(std::move(devices), std::move(oldRequestCounters))) , TaskQueue(std::move(taskQueue)) , DeviceClient(std::move(deviceClient)) - , RejectLateRequestsAtDiskAgentEnabled( - rdmaTargetConfig.RejectLateRequestsAtDiskAgentEnabled) + , RejectLateRequests(rejectLateRequests) {} void Init(NRdma::IServerEndpointPtr endpoint, TLog log) @@ -393,7 +394,7 @@ class TRequestHandler final requestDetails.Range); if (overlapsWithInflightRequests) { synchronizedData.OldRequestCounters.Delayed->Inc(); - if (!RejectLateRequestsAtDiskAgentEnabled) { + if (!RejectLateRequests) { // Monitoring mode. Don't change the behavior. return ECheckRange::NotOverlapped; } @@ -415,7 +416,7 @@ class TRequestHandler final Y_DEBUG_ABORT_UNLESS(false); } - if (!RejectLateRequestsAtDiskAgentEnabled) { + if (!RejectLateRequests) { // Monitoring mode. Don't change the behavior. return ECheckRange::NotOverlapped; } @@ -961,7 +962,7 @@ class TRdmaTarget final : public IRdmaTarget { private: - const NProto::TRdmaEndpoint Config; + const TRdmaTargetConfigPtr Config; std::shared_ptr Handler; ILoggingServicePtr Logging; @@ -972,13 +973,13 @@ class TRdmaTarget final public: TRdmaTarget( - NProto::TRdmaEndpoint config, + TRdmaTargetConfigPtr config, + TOldRequestCounters oldRequestCounters, ILoggingServicePtr logging, NRdma::IServerPtr server, TDeviceClientPtr deviceClient, THashMap devices, - ITaskQueuePtr taskQueue, - TRdmaTargetConfig rdmaTargetConfig) + ITaskQueuePtr taskQueue) : Config(std::move(config)) , Logging(std::move(logging)) , Server(std::move(server)) @@ -988,7 +989,8 @@ class TRdmaTarget final std::move(devices), std::move(taskQueue), std::move(deviceClient), - std::move(rdmaTargetConfig)); + std::move(oldRequestCounters), + Config->RejectLateRequests); } void Start() override @@ -996,8 +998,8 @@ class TRdmaTarget final Log = Logging->CreateLog("BLOCKSTORE_DISK_AGENT"); auto endpoint = Server->StartEndpoint( - Config.GetHost(), - Config.GetPort(), + Config->Host, + Config->Port, Handler); if (endpoint == nullptr) { @@ -1030,24 +1032,24 @@ class TRdmaTarget final } // namespace IRdmaTargetPtr CreateRdmaTarget( - NProto::TRdmaEndpoint config, - TRdmaTargetConfig rdmaTargetConfig, + TRdmaTargetConfigPtr config, + TOldRequestCounters oldRequestCounters, ILoggingServicePtr logging, NRdma::IServerPtr server, TDeviceClientPtr deviceClient, THashMap devices) { - auto threadPool = CreateThreadPool("RDMA", rdmaTargetConfig.PoolSize); + auto threadPool = CreateThreadPool("RDMA", config->WorkerThreads); threadPool->Start(); return std::make_shared( std::move(config), + std::move(oldRequestCounters), std::move(logging), std::move(server), std::move(deviceClient), std::move(devices), - std::move(threadPool), - std::move(rdmaTargetConfig)); + std::move(threadPool)); } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_agent/rdma_target.h b/cloud/blockstore/libs/storage/disk_agent/rdma_target.h index ab2551a0246..9160024fc35 100644 --- a/cloud/blockstore/libs/storage/disk_agent/rdma_target.h +++ b/cloud/blockstore/libs/storage/disk_agent/rdma_target.h @@ -10,6 +10,8 @@ #include +#include + namespace NCloud::NBlockStore::NStorage { //////////////////////////////////////////////////////////////////////////////// @@ -23,11 +25,32 @@ struct TOldRequestCounters struct TRdmaTargetConfig { - bool RejectLateRequestsAtDiskAgentEnabled = false; - TOldRequestCounters OldRequestCounters = {}; - ui32 PoolSize = 1; + bool RejectLateRequests = false; + TString Host = "::"; + ui32 Port = 10020; + ui32 WorkerThreads = 1; + + TRdmaTargetConfig(bool rejectLateRequests, NProto::TRdmaTarget target) + : RejectLateRequests(rejectLateRequests) + { + auto& endpoint = target.GetEndpoint(); + + if (auto& host = endpoint.GetHost()) { + Host = host; + } + + if (auto port = endpoint.GetPort()) { + Port = port; + } + + if (auto threads = target.GetWorkerThreads()) { + WorkerThreads = threads; + } + } }; +using TRdmaTargetConfigPtr = std::shared_ptr; + struct IRdmaTarget: IStartable { virtual NProto::TError DeviceSecureEraseStart( @@ -41,8 +64,8 @@ using TStorageAdapterPtr = std::shared_ptr; using IRdmaTargetPtr = std::shared_ptr; IRdmaTargetPtr CreateRdmaTarget( - NProto::TRdmaEndpoint config, - TRdmaTargetConfig rdmaTargetConfig, + TRdmaTargetConfigPtr rdmaTargetConfig, + TOldRequestCounters OldRequestCounters, ILoggingServicePtr logging, NRdma::IServerPtr server, TDeviceClientPtr deviceClient,