diff --git a/cloud/filestore/config/storage.proto b/cloud/filestore/config/storage.proto index 76ec722279d..92594cda71f 100644 --- a/cloud/filestore/config/storage.proto +++ b/cloud/filestore/config/storage.proto @@ -387,4 +387,7 @@ message TStorageConfig // If the number of blocks marked for deletion via large deletion markers // exceeds this threshold, Cleanup will be triggered. optional uint64 LargeDeletionMarkersCleanupThreshold = 388; + + // Throttle DescribeData and GenerateBlobIds requests + optional bool MultipleStageRequestThrottlingEnabled = 389; } diff --git a/cloud/filestore/libs/storage/core/config.cpp b/cloud/filestore/libs/storage/core/config.cpp index 89aff58c48a..95cbca49c2d 100644 --- a/cloud/filestore/libs/storage/core/config.cpp +++ b/cloud/filestore/libs/storage/core/config.cpp @@ -203,6 +203,8 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases; xxx(NodeRegistrationMaxAttempts, ui32, 10 )\ xxx(NodeRegistrationTimeout, TDuration, TDuration::Seconds(5) )\ xxx(NodeRegistrationErrorTimeout, TDuration, TDuration::Seconds(1) )\ + \ + xxx(MultipleStageRequestThrottlingEnabled, bool, false )\ // FILESTORE_STORAGE_CONFIG #define FILESTORE_STORAGE_CONFIG_REF(xxx) \ diff --git a/cloud/filestore/libs/storage/core/config.h b/cloud/filestore/libs/storage/core/config.h index 96215d85678..aa1f0770f77 100644 --- a/cloud/filestore/libs/storage/core/config.h +++ b/cloud/filestore/libs/storage/core/config.h @@ -264,6 +264,8 @@ class TStorageConfig ui64 GetLargeDeletionMarkerBlocks() const; ui64 GetLargeDeletionMarkersThreshold() const; ui64 GetLargeDeletionMarkersCleanupThreshold() const; + + bool GetMultipleStageRequestThrottlingEnabled() const; }; } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/service/service_actor_readdata.cpp b/cloud/filestore/libs/storage/service/service_actor_readdata.cpp index 42f2cace9b1..e60aa9bd0c3 100644 --- a/cloud/filestore/libs/storage/service/service_actor_readdata.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_readdata.cpp @@ -503,6 +503,7 @@ void TReadDataActor::ReadData( auto request = std::make_unique(); request->Record = std::move(ReadRequest); + request->Record.MutableHeaders()->SetThrottlingDisabled(true); // forward request through tablet proxy ctx.Send(MakeIndexTabletProxyServiceId(), request.release()); diff --git a/cloud/filestore/libs/storage/service/service_actor_writedata.cpp b/cloud/filestore/libs/storage/service/service_actor_writedata.cpp index 6d6cf54f65b..50d98563b3a 100644 --- a/cloud/filestore/libs/storage/service/service_actor_writedata.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_writedata.cpp @@ -379,6 +379,7 @@ class TWriteDataActor final: public TActorBootstrapped auto request = std::make_unique(); request->Record = std::move(WriteRequest); + request->Record.MutableHeaders()->SetThrottlingDisabled(true); // forward request through tablet proxy ctx.Send(MakeIndexTabletProxyServiceId(), request.release()); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp index 0c7e013c0e6..787ecf57e1d 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp @@ -253,6 +253,12 @@ void TIndexTabletActor::HandleGenerateBlobIds( return; } + if (Config->GetMultipleStageRequestThrottlingEnabled() && + ThrottleIfNeeded(ev, ctx)) + { + return; + } + // We schedule this event for the case if the client does not call AddData. // Thus we ensure that the collect barrier will be released eventually. ctx.Schedule( diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp index 0a92b3edc7b..026425dc612 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp @@ -617,6 +617,12 @@ void TIndexTabletActor::HandleDescribeData( return; } + if (Config->GetMultipleStageRequestThrottlingEnabled() && + ThrottleIfNeeded(ev, ctx)) + { + return; + } + auto* msg = ev->Get(); const TByteRange byteRange( msg->Record.GetOffset(), diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_throttling.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_throttling.cpp index ff9c3dc9297..71dfb5ff02c 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_throttling.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_throttling.cpp @@ -61,6 +61,30 @@ TThrottlingRequestInfo BuildRequestInfo( }; } +template <> +TThrottlingRequestInfo BuildRequestInfo( + const TEvIndexTablet::TEvDescribeDataRequest& request, + ui32 policyVersion) +{ + return { + static_cast(CalculateByteCount(request.Record)), + static_cast(TThrottlingPolicy::EOpType::Read), + policyVersion + }; +} + +template <> +TThrottlingRequestInfo BuildRequestInfo( + const TEvIndexTablet::TEvGenerateBlobIdsRequest& request, + ui32 policyVersion) +{ + return { + static_cast(CalculateByteCount(request.Record)), + static_cast(TThrottlingPolicy::EOpType::Write), + policyVersion + }; +} + } // namespace //////////////////////////////////////////////////////////////////////////////// @@ -146,7 +170,8 @@ bool TIndexTabletActor::ThrottleIfNeeded( const NActors::TActorContext& ctx) { if (!Config->GetThrottlingEnabled() || - !GetPerformanceProfile().GetThrottlingEnabled()) + !GetPerformanceProfile().GetThrottlingEnabled() || + ev->Get()->Record.GetHeaders().GetThrottlingDisabled()) { return false; } @@ -185,6 +210,8 @@ template bool TIndexTabletActor::ThrottleIfNeeded( \ GENERATE_IMPL(ReadData, TEvService) GENERATE_IMPL(WriteData, TEvService) +GENERATE_IMPL(DescribeData, TEvIndexTablet) +GENERATE_IMPL(GenerateBlobIds, TEvIndexTablet) #undef GENERATE_IMPL diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp index 04af8f86bb4..520c9aa327f 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp @@ -22,6 +22,7 @@ struct TTablet std::unique_ptr Env = nullptr; std::unique_ptr Tablet = nullptr; ui64 Handle = 0; + ui64 NodeId = 0; public: void SetUp(NUnitTest::TTestContext& /*context*/) override @@ -45,6 +46,8 @@ struct TTablet NProto::TStorageConfig storageConfig; if (throttlingEnabled.Defined()) { storageConfig.SetThrottlingEnabled(throttlingEnabled.GetRef()); + storageConfig.SetMultipleStageRequestThrottlingEnabled( + throttlingEnabled.GetRef()); } Env = std::make_unique(envConfig, storageConfig); @@ -58,9 +61,8 @@ struct TTablet tabletId); Tablet->InitSession("client", "session"); - const auto nodeId = - CreateNode(*Tablet, TCreateNodeArgs::File(RootNodeId, "test")); - Handle = CreateHandle(*Tablet, nodeId); + NodeId = CreateNode(*Tablet, TCreateNodeArgs::File(RootNodeId, "test")); + Handle = CreateHandle(*Tablet, NodeId); } auto UpdateConfig(const TFileSystemConfig& config) @@ -78,6 +80,16 @@ struct TTablet Tablet->SendReadDataRequest(Handle, offset, len); } + void DescribeData(ui64 offset, ui32 len) + { + Tablet->SendDescribeDataRequest(Handle, offset, len); + } + + void GenerateBlobIds(ui64 offset, ui32 len) + { + Tablet->SendGenerateBlobIdsRequest(NodeId, Handle, offset, len); + } + void Tick(TDuration duration) { Tablet->AdvanceTime(duration); @@ -108,6 +120,8 @@ struct TTablet TEST_CLIENT_DECLARE_METHOD(ReadData); TEST_CLIENT_DECLARE_METHOD(WriteData); + TEST_CLIENT_DECLARE_METHOD(DescribeData); + TEST_CLIENT_DECLARE_METHOD(GenerateBlobIds); #undef TEST_CLIENT_DECLARE_METHOD @@ -219,7 +233,7 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Throttling) // 2. Testing that we start rejecting requests after // our postponed limit saturates. - WriteData(0, 1, 'y'); // Event 1 byte request must be rejected. + WriteData(0, 1, 'y'); // Even 1 byte request must be rejected. AssertWriteDataQuickResponse(E_FS_THROTTLED); ReadData(0, 1_KB); AssertReadDataQuickResponse(E_FS_THROTTLED); @@ -281,6 +295,94 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Throttling) // TODO: 7. Test max count (NBS-2278). } + Y_UNIT_TEST_F(ShouldThrottleMultipleStage, TTablet) + { + const auto config = MakeThrottlerConfig( + true, // throttlingEnabled + 2, // maxReadIops + 2, // maxWriteIops + 8_KB, // maxReadBandwidth + 8_KB, // maxWriteBandwidth + TDuration::Minutes(30).MilliSeconds(), // boostTime + TDuration::Hours(12).MilliSeconds(), // boostRefillTime + 10, // boostPercentage + 32_KB, // maxPostponedWeight + 10, // maxWriteCostMultiplier + TDuration::Seconds(25).MilliSeconds(), // maxPostponedTime + 64, // maxPostponedCount + 100, // burstPercentage + 1_KB // defaultPostponedRequestWeight + ); + UpdateConfig(config); + + // 0. Testing that at 1rps nothing is throttled. + for (size_t i = 0; i < 10; ++i) { + Tick(TDuration::Seconds(1)); + DescribeData(4_KB * i, 4_KB); + AssertDescribeDataQuickResponse(S_OK); + Tick(TDuration::Seconds(1)); + GenerateBlobIds(4_KB * (i + 1), 4_KB); + AssertGenerateBlobIdsQuickResponse(S_OK); + } + + // 1. Testing that excess requests are postponed. + for (size_t i = 0; i < 20; ++i) { + DescribeData(4_KB * i, 4_KB); + AssertDescribeDataNoResponse(); + } + + // Now we have 20_KB in PostponeQueue. + + for (size_t i = 0; i < 3; ++i) { + GenerateBlobIds(0, 4_KB); + AssertGenerateBlobIdsNoResponse(); + } + + // Now we have 32_KB in PostponedQueue. Equal to MaxPostponedWeight. + + // 2. Testing that we start rejecting requests after + // our postponed limit saturates. + + GenerateBlobIds(0, config.BlockSize); // Request must be rejected. + AssertGenerateBlobIdsQuickResponse(E_FS_THROTTLED); + DescribeData(0, config.BlockSize); + AssertDescribeDataQuickResponse(E_FS_THROTTLED); + + // 3. Testing that after some time passes our postponed requests + // are successfully processed. + // Test actor runtime will automatically advance the timer for us. + for (ui32 i = 0; i < 20; ++i) { + Tick(TDuration::Seconds(1)); + AssertDescribeDataResponse(S_OK); + } + + for (ui32 i = 0; i < 3; ++i) { + Tick(TDuration::Seconds(1)); + AssertGenerateBlobIdsResponse(S_OK); + } + + // Now PostponedQueue is empty. + + // 4. Testing that bursts actually work. + Tick(TDuration::Seconds(2)); + DescribeData(0, 12_KB); + { + // Postponed WriteDataRequest must write 4_KB of 'z'. + AssertDescribeDataQuickResponse(S_OK); + } + DescribeData(12_KB, 4_KB); + AssertDescribeDataNoResponse(); + { + AssertDescribeDataResponse(S_OK); + Tick(TDuration::Seconds(1)); + } + + // 5. Requests of any size should work, but not immediately. + GenerateBlobIds(0, 32_KB); + Tick(TDuration::Seconds(5)); + AssertGenerateBlobIdsResponse(S_OK); + } + Y_UNIT_TEST_F(ShouldNotThrottleIfThrottlerIsDisabled, TTablet) { auto config = MakeThrottlerConfig( diff --git a/cloud/filestore/public/api/protos/headers.proto b/cloud/filestore/public/api/protos/headers.proto index e16ba5df327..b908129d9f3 100644 --- a/cloud/filestore/public/api/protos/headers.proto +++ b/cloud/filestore/public/api/protos/headers.proto @@ -56,6 +56,9 @@ message THeaders // Explicitly disables multitablet forwarding. bool DisableMultiTabletForwarding = 11; + + // Throttle already performed + bool ThrottlingDisabled = 12; } ////////////////////////////////////////////////////////////////////////////////