From f4fefe9af1d117cfd5ab14a2f6102303473e3fc7 Mon Sep 17 00:00:00 2001 From: Egor Masharskii Date: Mon, 23 Dec 2024 18:52:11 +0000 Subject: [PATCH 1/6] add fs and shard counters --- cloud/filestore/libs/storage/tablet/tablet_actor.h | 5 ++++- .../libs/storage/tablet/tablet_actor_counters.cpp | 14 +++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.h b/cloud/filestore/libs/storage/tablet/tablet_actor.h index 4f55d62109..8007bf8b29 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.h +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.h @@ -80,6 +80,8 @@ class TIndexTabletActor final { bool Initialized{false}; + std::atomic FsCount{0}; + std::atomic FsShardCount{0}; std::atomic TotalBytesCount{0}; std::atomic UsedBytesCount{0}; std::atomic AggregateUsedBytesCount{0}; @@ -323,7 +325,8 @@ class TIndexTabletActor final const TNodeIndexCacheStats& nodeIndexCacheStats, const TNodeToSessionCounters& nodeToSessionCounters, const TMiscNodeStats& miscNodeStats, - const TInMemoryIndexStateStats& inMemoryIndexStateStats); + const TInMemoryIndexStateStats& inMemoryIndexStateStats, + bool isShard); void UpdatePerformanceMetrics( TInstant now, const TDiagnosticsConfig& diagConfig, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp index 58ed66d079..46ab35fe95 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp @@ -275,6 +275,7 @@ void TIndexTabletActor::TMetrics::Register( metrType) \ // REGISTER_LOCAL + REGISTER_AGGREGATABLE_SUM(FsCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(TotalBytesCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(UsedBytesCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM( @@ -476,8 +477,13 @@ void TIndexTabletActor::TMetrics::Update( const TNodeIndexCacheStats& nodeIndexCacheStats, const TNodeToSessionCounters& nodeToSessionCounters, const TMiscNodeStats& miscNodeStats, - const TInMemoryIndexStateStats& inMemoryIndexStateStats) + const TInMemoryIndexStateStats& inMemoryIndexStateStats, + bool isShard) { + auto shardCnt = isShard ? 1 : 0; + NMetrics::Store(FsCount, 1 - shardCnt); + NMetrics::Store(FsShardCount, shardCnt); + const ui32 blockSize = fileSystem.GetBlockSize(); Store(TotalBytesCount, fileSystem.GetBlocksCount() * blockSize); @@ -695,7 +701,8 @@ void TIndexTabletActor::RegisterStatCounters(TInstant now) CalculateNodeIndexCacheStats(), GetNodeToSessionCounters(), GetMiscNodeStats(), - GetInMemoryIndexStateStats()); + GetInMemoryIndexStateStats(), + IsShard()); Metrics.Register(fsId, storageMediaKind); } @@ -745,7 +752,8 @@ void TIndexTabletActor::HandleUpdateCounters( CalculateNodeIndexCacheStats(), GetNodeToSessionCounters(), GetMiscNodeStats(), - GetInMemoryIndexStateStats()); + GetInMemoryIndexStateStats(), + IsShard()); SendMetricsToExecutor(ctx); UpdateCountersScheduled = false; From 9dd28564ef83e0f91eaaa4428f54dc5cc74ceb3b Mon Sep 17 00:00:00 2001 From: Egor Masharskii Date: Mon, 23 Dec 2024 18:52:11 +0000 Subject: [PATCH 2/6] add fs and shard counters --- .../filestore/libs/storage/tablet/tablet_actor.h | 5 ++++- .../libs/storage/tablet/tablet_actor_counters.cpp | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.h b/cloud/filestore/libs/storage/tablet/tablet_actor.h index 4f55d62109..8007bf8b29 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.h +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.h @@ -80,6 +80,8 @@ class TIndexTabletActor final { bool Initialized{false}; + std::atomic FsCount{0}; + std::atomic FsShardCount{0}; std::atomic TotalBytesCount{0}; std::atomic UsedBytesCount{0}; std::atomic AggregateUsedBytesCount{0}; @@ -323,7 +325,8 @@ class TIndexTabletActor final const TNodeIndexCacheStats& nodeIndexCacheStats, const TNodeToSessionCounters& nodeToSessionCounters, const TMiscNodeStats& miscNodeStats, - const TInMemoryIndexStateStats& inMemoryIndexStateStats); + const TInMemoryIndexStateStats& inMemoryIndexStateStats, + bool isShard); void UpdatePerformanceMetrics( TInstant now, const TDiagnosticsConfig& diagConfig, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp index 58ed66d079..93da4c8c5f 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp @@ -275,6 +275,8 @@ void TIndexTabletActor::TMetrics::Register( metrType) \ // REGISTER_LOCAL + REGISTER_AGGREGATABLE_SUM(FsCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM(FsShardCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(TotalBytesCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(UsedBytesCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM( @@ -476,8 +478,13 @@ void TIndexTabletActor::TMetrics::Update( const TNodeIndexCacheStats& nodeIndexCacheStats, const TNodeToSessionCounters& nodeToSessionCounters, const TMiscNodeStats& miscNodeStats, - const TInMemoryIndexStateStats& inMemoryIndexStateStats) + const TInMemoryIndexStateStats& inMemoryIndexStateStats, + bool isShard) { + auto shardCnt = isShard ? 1 : 0; + NMetrics::Store(FsCount, 1 - shardCnt); + NMetrics::Store(FsShardCount, shardCnt); + const ui32 blockSize = fileSystem.GetBlockSize(); Store(TotalBytesCount, fileSystem.GetBlocksCount() * blockSize); @@ -695,7 +702,8 @@ void TIndexTabletActor::RegisterStatCounters(TInstant now) CalculateNodeIndexCacheStats(), GetNodeToSessionCounters(), GetMiscNodeStats(), - GetInMemoryIndexStateStats()); + GetInMemoryIndexStateStats(), + IsShard()); Metrics.Register(fsId, storageMediaKind); } @@ -745,7 +753,8 @@ void TIndexTabletActor::HandleUpdateCounters( CalculateNodeIndexCacheStats(), GetNodeToSessionCounters(), GetMiscNodeStats(), - GetInMemoryIndexStateStats()); + GetInMemoryIndexStateStats(), + IsShard()); SendMetricsToExecutor(ctx); UpdateCountersScheduled = false; From ba9362f7d154b50cea59c5d09509261c54f40648 Mon Sep 17 00:00:00 2001 From: Egor Masharskii Date: Mon, 23 Dec 2024 19:25:25 +0000 Subject: [PATCH 3/6] update --- .../libs/storage/tablet/tablet_actor_counters.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp index 93da4c8c5f..11a2587a8b 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp @@ -276,7 +276,6 @@ void TIndexTabletActor::TMetrics::Register( // REGISTER_LOCAL REGISTER_AGGREGATABLE_SUM(FsCount, EMetricType::MT_ABSOLUTE); - REGISTER_AGGREGATABLE_SUM(FsShardCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(TotalBytesCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(UsedBytesCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM( @@ -481,9 +480,9 @@ void TIndexTabletActor::TMetrics::Update( const TInMemoryIndexStateStats& inMemoryIndexStateStats, bool isShard) { - auto shardCnt = isShard ? 1 : 0; - NMetrics::Store(FsCount, 1 - shardCnt); - NMetrics::Store(FsShardCount, shardCnt); + auto fsCounter = isShard ? 0 : 1; + NMetrics::Store(FsCount, fsCounter); + NMetrics::Store(FsShardCount, 1 - fsCounter); const ui32 blockSize = fileSystem.GetBlockSize(); From 0816f0c603260157d3ebe57b986bab4014c82d3c Mon Sep 17 00:00:00 2001 From: Egor Masharskii Date: Tue, 24 Dec 2024 16:55:14 +0000 Subject: [PATCH 4/6] update --- .../libs/storage/tablet/tablet_actor.h | 7 +++-- .../storage/tablet/tablet_actor_counters.cpp | 30 +++++++++++-------- .../storage/tablet/tablet_actor_loadstate.cpp | 3 ++ .../tablet/tablet_actor_updateconfig.cpp | 3 ++ .../storage/tablet/tablet_ut_counters.cpp | 21 +++++++++++++ 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.h b/cloud/filestore/libs/storage/tablet/tablet_actor.h index 8007bf8b29..bc3ca6a441 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.h +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.h @@ -81,7 +81,8 @@ class TIndexTabletActor final bool Initialized{false}; std::atomic FsCount{0}; - std::atomic FsShardCount{0}; + std::atomic TabletCount{1}; + std::atomic TotalBytesCount{0}; std::atomic UsedBytesCount{0}; std::atomic AggregateUsedBytesCount{0}; @@ -308,6 +309,7 @@ class TIndexTabletActor final NMetrics::IMetricsRegistryPtr FsRegistry; NMetrics::IMetricsRegistryPtr AggregatableFsRegistry; + NMetrics::IMetricsRegistryPtr AggregatableRegistry; explicit TMetrics(NMetrics::IMetricsRegistryPtr metricsRegistry); @@ -325,8 +327,7 @@ class TIndexTabletActor final const TNodeIndexCacheStats& nodeIndexCacheStats, const TNodeToSessionCounters& nodeToSessionCounters, const TMiscNodeStats& miscNodeStats, - const TInMemoryIndexStateStats& inMemoryIndexStateStats, - bool isShard); + const TInMemoryIndexStateStats& inMemoryIndexStateStats); void UpdatePerformanceMetrics( TInstant now, const TDiagnosticsConfig& diagConfig, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp index 11a2587a8b..c44e458548 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp @@ -253,7 +253,10 @@ void TIndexTabletActor::TMetrics::Register( StorageFsRegistry); AggregatableFsRegistry = CreateScopedMetricsRegistry( {}, - {std::move(totalKindRegistry), FsRegistry}); + {totalKindRegistry, FsRegistry}); + AggregatableRegistry = CreateScopedMetricsRegistry( + {}, + {std::move(totalKindRegistry)}); #define REGISTER(registry, name, aggrType, metrType) \ RegisterSensor(registry, #name, name, aggrType, metrType) \ @@ -267,6 +270,14 @@ void TIndexTabletActor::TMetrics::Register( metrType) \ // REGISTER_AGGREGATABLE_SUM +#define REGISTER_AGGREGATABLE_ONLY_SUM(name, metrType) \ + REGISTER( \ + AggregatableRegistry, \ + name, \ + EAggregationType::AT_SUM, \ + metrType) \ +// REGISTER_AGGREGATABLE_ONLY_SUM + #define REGISTER_LOCAL(name, metrType) \ REGISTER( \ FsRegistry, \ @@ -275,7 +286,9 @@ void TIndexTabletActor::TMetrics::Register( metrType) \ // REGISTER_LOCAL - REGISTER_AGGREGATABLE_SUM(FsCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_ONLY_SUM(FsCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_ONLY_SUM(TabletCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM(TotalBytesCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(UsedBytesCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM( @@ -477,13 +490,8 @@ void TIndexTabletActor::TMetrics::Update( const TNodeIndexCacheStats& nodeIndexCacheStats, const TNodeToSessionCounters& nodeToSessionCounters, const TMiscNodeStats& miscNodeStats, - const TInMemoryIndexStateStats& inMemoryIndexStateStats, - bool isShard) + const TInMemoryIndexStateStats& inMemoryIndexStateStats) { - auto fsCounter = isShard ? 0 : 1; - NMetrics::Store(FsCount, fsCounter); - NMetrics::Store(FsShardCount, 1 - fsCounter); - const ui32 blockSize = fileSystem.GetBlockSize(); Store(TotalBytesCount, fileSystem.GetBlocksCount() * blockSize); @@ -701,8 +709,7 @@ void TIndexTabletActor::RegisterStatCounters(TInstant now) CalculateNodeIndexCacheStats(), GetNodeToSessionCounters(), GetMiscNodeStats(), - GetInMemoryIndexStateStats(), - IsShard()); + GetInMemoryIndexStateStats()); Metrics.Register(fsId, storageMediaKind); } @@ -752,8 +759,7 @@ void TIndexTabletActor::HandleUpdateCounters( CalculateNodeIndexCacheStats(), GetNodeToSessionCounters(), GetMiscNodeStats(), - GetInMemoryIndexStateStats(), - IsShard()); + GetInMemoryIndexStateStats()); SendMetricsToExecutor(ctx); UpdateCountersScheduled = false; diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_loadstate.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_loadstate.cpp index fef2ebc559..3d04c0cb24 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_loadstate.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_loadstate.cpp @@ -3,6 +3,7 @@ #include "helpers.h" #include +#include namespace NCloud::NFileStore::NStorage { @@ -332,6 +333,8 @@ void TIndexTabletActor::CompleteTx_LoadState( LOG_INFO_S(ctx, TFileStoreComponents::TABLET, LogTag << " Load state completed"); + + NMetrics::Store(Metrics.FsCount, IsShard() ? 0 : 1); } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp index 037531a42c..95263dd784 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp @@ -1,6 +1,7 @@ #include "tablet_actor.h" #include +#include #include @@ -230,6 +231,8 @@ void TIndexTabletActor::ExecuteTx_UpdateConfig( Convert(args.FileSystem.GetPerformanceProfile(), config); UpdateConfig(db, args.FileSystem, config); + + NMetrics::Store(Metrics.FsCount, IsShard() ? 0 : 1); } void TIndexTabletActor::CompleteTx_UpdateConfig( diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp index 2589d66ef3..5d31595eac 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp @@ -110,6 +110,10 @@ struct TEnv void TearDown(NUnitTest::TTestContext& /*context*/) override {} + const TTestEnv& GetTestEnv() const + { + return Env; + } }; } // namespace @@ -896,6 +900,23 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Counters) }, }); } + + Y_UNIT_TEST_F(ShouldReportFsAndTabletCountSensors, TEnv) + { + auto registry = Env.GetRegistry(); + + Tablet->AdvanceTime(TDuration::Seconds(15)); + Env.GetRuntime().DispatchEvents({}, TDuration::Seconds(5)); + + registry->Visit(TInstant::Zero(), Visitor); + Visitor.ValidateExpectedCounters({ + {{{"sensor", "FsCount"}}, 1}, + {{{"sensor", "TabletCount"}}, 1}, + }); + + //auto counters = Env.GetCounters(); + //counters-> + } } } // namespace NCloud::NFileStore::NStorage From 960aa75a58217a4954f8db08e848f45d70236ff0 Mon Sep 17 00:00:00 2001 From: Egor Masharskii Date: Tue, 24 Dec 2024 17:01:10 +0000 Subject: [PATCH 5/6] update --- cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp index 5d31595eac..3d536e5745 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp @@ -109,11 +109,6 @@ struct TEnv void TearDown(NUnitTest::TTestContext& /*context*/) override {} - - const TTestEnv& GetTestEnv() const - { - return Env; - } }; } // namespace From 316e3ea53c678b12075c2c9e52118de551b3f0a4 Mon Sep 17 00:00:00 2001 From: Egor Masharskii Date: Tue, 24 Dec 2024 22:07:36 +0000 Subject: [PATCH 6/6] update --- cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp index 3d536e5745..533aad8672 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp @@ -908,9 +908,6 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Counters) {{{"sensor", "FsCount"}}, 1}, {{{"sensor", "TabletCount"}}, 1}, }); - - //auto counters = Env.GetCounters(); - //counters-> } }