From 54a9cb464dca91deede6a2bfaadd990341356bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gra=CC=88tzer?= Date: Thu, 8 Apr 2021 22:50:33 +0200 Subject: [PATCH] Create DB properties for internal table cache --- db/db_properties_test.cc | 49 +++++++++++++++++++++++++++++++++++++--- db/internal_stats.cc | 24 ++++++++++++++++++++ db/internal_stats.h | 23 +++++++++++-------- include/rocksdb/db.h | 9 ++++++++ 4 files changed, 92 insertions(+), 13 deletions(-) diff --git a/db/db_properties_test.cc b/db/db_properties_test.cc index d3333fa93..8f748c5b9 100644 --- a/db/db_properties_test.cc +++ b/db/db_properties_test.cc @@ -245,7 +245,8 @@ void GetExpectedTableProperties( const int kDeletionCount = kTableCount * kDeletionsPerTable; const int kMergeCount = kTableCount * kMergeOperandsPerTable; const int kRangeDeletionCount = kTableCount * kRangeDeletionsPerTable; - const int kKeyCount = kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount; + const int kKeyCount = + kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount; const int kAvgSuccessorSize = kKeySize / 5; const int kEncodingSavePerKey = kKeySize / 4; expected_tp->raw_key_size = kKeyCount * (kKeySize + 8); @@ -256,7 +257,8 @@ void GetExpectedTableProperties( expected_tp->num_merge_operands = kMergeCount; expected_tp->num_range_deletions = kRangeDeletionCount; expected_tp->num_data_blocks = - kTableCount * (kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) / + kTableCount * + (kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) / kBlockSize; expected_tp->data_size = kTableCount * (kKeysPerTable * (kKeySize + 8 + kValueSize)); @@ -1090,7 +1092,8 @@ class CountingUserTblPropCollector : public TablePropertiesCollector { std::string encoded; PutVarint32(&encoded, count_); *properties = UserCollectedProperties{ - {"CountingUserTblPropCollector", message_}, {"Count", encoded}, + {"CountingUserTblPropCollector", message_}, + {"Count", encoded}, }; return Status::OK(); } @@ -1713,6 +1716,46 @@ TEST_F(DBPropertiesTest, BlockCacheProperties) { ASSERT_EQ(0, value); } +TEST_F(DBPropertiesTest, TableCacheProperties) { + Options options; + uint64_t value, new_value; + + options.env = CurrentOptions().env; + + Reopen(options); + + // + // test table_cache access is "live" + // TableCacheCapacity originally comes from DBOptions::max_open_files + // and can vary by system. Get its current value. + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kTableCacheCapacity, &value)); + + // now, change max_open_files to prove we are really accessing the value of + // interest + new_value = value / 2; + std::unordered_map new_options; + new_options.insert(std::pair( + "max_open_files", std::to_string(new_value))); + ASSERT_OK(db_->SetDBOptions(new_options)); + + // did the value we are reading update. NOTE: rocksdb internally reduces + // the value we pass by 10. + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kTableCacheCapacity, &value)); + ASSERT_EQ(new_value - 10, value); + + // + // TableCacheUsage is a count of open .sst files. Force the creation of a + // a new table file. First add a record via Put(). Then force that + // record from write buffer to new .sst via Flush(). New .sst + // automatically opens and gets position in table cache ... raising usage + // count + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kTableCacheUsage, &value)); + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kTableCacheUsage, &new_value)); + ASSERT_EQ(new_value, value + 1); +} #endif // ROCKSDB_LITE } // namespace ROCKSDB_NAMESPACE diff --git a/db/internal_stats.cc b/db/internal_stats.cc index 512bc1b01..b1169a7ff 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -260,6 +260,8 @@ static const std::string estimate_oldest_key_time = "estimate-oldest-key-time"; static const std::string block_cache_capacity = "block-cache-capacity"; static const std::string block_cache_usage = "block-cache-usage"; static const std::string block_cache_pinned_usage = "block-cache-pinned-usage"; +static const std::string tablecache_capacity = "table-cache-capacity"; +static const std::string tablecache_usage = "table-cache-usage"; static const std::string options_statistics = "options-statistics"; const std::string DB::Properties::kNumFilesAtLevelPrefix = @@ -348,6 +350,10 @@ const std::string DB::Properties::kBlockCacheUsage = rocksdb_prefix + block_cache_usage; const std::string DB::Properties::kBlockCachePinnedUsage = rocksdb_prefix + block_cache_pinned_usage; +const std::string DB::Properties::kTableCacheCapacity = + rocksdb_prefix + tablecache_capacity; +const std::string DB::Properties::kTableCacheUsage = + rocksdb_prefix + tablecache_usage; const std::string DB::Properties::kOptionsStatistics = rocksdb_prefix + options_statistics; @@ -487,6 +493,12 @@ const std::unordered_map {DB::Properties::kBlockCachePinnedUsage, {false, nullptr, &InternalStats::HandleBlockCachePinnedUsage, nullptr, nullptr}}, + {DB::Properties::kTableCacheCapacity, + {false, nullptr, &InternalStats::HandleTableCacheCapacity, nullptr, + nullptr}}, + {DB::Properties::kTableCacheUsage, + {false, nullptr, &InternalStats::HandleTableCacheUsage, nullptr, + nullptr}}, {DB::Properties::kOptionsStatistics, {false, nullptr, nullptr, nullptr, &DBImpl::GetPropertyHandleOptionsStatistics}}, @@ -987,6 +999,18 @@ bool InternalStats::HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* /*db*/, return true; } +bool InternalStats::HandleTableCacheCapacity(uint64_t* value, DBImpl* db, + Version* /*version*/) { + *value = static_cast(db->table_cache_->GetCapacity()); + return true; +} + +bool InternalStats::HandleTableCacheUsage(uint64_t* value, DBImpl* db, + Version* /*version*/) { + *value = static_cast(db->table_cache_->GetUsage()); + return true; +} + void InternalStats::DumpDBStats(std::string* value) { char buf[1000]; // DB-level stats, only available from default column family diff --git a/db/internal_stats.h b/db/internal_stats.h index 056719c5c..d63ef0a82 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -296,7 +296,7 @@ class InternalStats { this->num_dropped_records += c.num_dropped_records; this->count += c.count; int num_of_reasons = static_cast(CompactionReason::kNumOfReasons); - for (int i = 0; i< num_of_reasons; i++) { + for (int i = 0; i < num_of_reasons; i++) { counts[i] += c.counts[i]; } } @@ -437,8 +437,8 @@ class InternalStats { struct CFStatsSnapshot { // ColumnFamily-level stats CompactionStats comp_stats; - uint64_t ingest_bytes_flush; // Bytes written to L0 (Flush) - uint64_t stall_count; // Stall count + uint64_t ingest_bytes_flush; // Bytes written to L0 (Flush) + uint64_t stall_count; // Stall count // Stats from compaction jobs - bytes written, bytes read, duration. uint64_t compact_bytes_write; uint64_t compact_bytes_read; @@ -480,10 +480,10 @@ class InternalStats { struct DBStatsSnapshot { // DB-level stats - uint64_t ingest_bytes; // Bytes written by user - uint64_t wal_bytes; // Bytes written to WAL - uint64_t wal_synced; // Number of times WAL is synced - uint64_t write_with_wal; // Number of writes that request WAL + uint64_t ingest_bytes; // Bytes written by user + uint64_t wal_bytes; // Bytes written to WAL + uint64_t wal_synced; // Number of times WAL is synced + uint64_t write_with_wal; // Number of writes that request WAL // These count the number of writes processed by the calling thread or // another thread. uint64_t write_other; @@ -594,6 +594,8 @@ class InternalStats { bool HandleBlockCacheUsage(uint64_t* value, DBImpl* db, Version* version); bool HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* db, Version* version); + bool HandleTableCacheCapacity(uint64_t* value, DBImpl* db, Version* version); + bool HandleTableCacheUsage(uint64_t* value, DBImpl* db, Version* version); // Total number of background errors encountered. Every time a flush task // or compaction task fails, this counter is incremented. The failure can // be caused by any possible reason, including file system errors, out of @@ -697,13 +699,14 @@ class InternalStats { return false; } - bool GetIntProperty(const DBPropertyInfo& /*property_info*/, uint64_t* /*value*/, - DBImpl* /*db*/) const { + bool GetIntProperty(const DBPropertyInfo& /*property_info*/, + uint64_t* /*value*/, DBImpl* /*db*/) const { return false; } bool GetIntPropertyOutOfMutex(const DBPropertyInfo& /*property_info*/, - Version* /*version*/, uint64_t* /*value*/) const { + Version* /*version*/, + uint64_t* /*value*/) const { return false; } }; diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 995d9f0f1..9a4cd0a0d 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -923,6 +923,13 @@ class DB { // entries being pinned. static const std::string kBlockCachePinnedUsage; + // "rocksdb.table-cache-capacity" - returns table cache capacity. + static const std::string kTableCacheCapacity; + + // "rocksdb.table-cache-usage" - returns the memory size for the entries + // residing in table cache. + static const std::string kTableCacheUsage; + // "rocksdb.options-statistics" - returns multi-line string // of options.statistics static const std::string kOptionsStatistics; @@ -986,6 +993,8 @@ class DB { // "rocksdb.block-cache-capacity" // "rocksdb.block-cache-usage" // "rocksdb.block-cache-pinned-usage" + // "rocksdb.table-cache-capacity" + // "rocksdb.table-cache-usage" virtual bool GetIntProperty(ColumnFamilyHandle* column_family, const Slice& property, uint64_t* value) = 0; virtual bool GetIntProperty(const Slice& property, uint64_t* value) {