Skip to content

Commit

Permalink
Create DB properties for internal table cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@graetzer
graetzer committed Apr 16, 2021
1 parent f7ef236 commit 54a9cb4
Show file tree
Hide file tree
Showing 4 changed files with 92 additions and 13 deletions.
49 changes: 46 additions & 3 deletions db/db_properties_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,8 @@ void GetExpectedTableProperties(
const int kDeletionCount = kTableCount * kDeletionsPerTable;
const int kMergeCount = kTableCount * kMergeOperandsPerTable;
const int kRangeDeletionCount = kTableCount * kRangeDeletionsPerTable;
const int kKeyCount = kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount;
const int kKeyCount =
kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount;
const int kAvgSuccessorSize = kKeySize / 5;
const int kEncodingSavePerKey = kKeySize / 4;
expected_tp->raw_key_size = kKeyCount * (kKeySize + 8);
Expand All @@ -256,7 +257,8 @@ void GetExpectedTableProperties(
expected_tp->num_merge_operands = kMergeCount;
expected_tp->num_range_deletions = kRangeDeletionCount;
expected_tp->num_data_blocks =
kTableCount * (kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) /
kTableCount *
(kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) /
kBlockSize;
expected_tp->data_size =
kTableCount * (kKeysPerTable * (kKeySize + 8 + kValueSize));
Expand Down Expand Up @@ -1090,7 +1092,8 @@ class CountingUserTblPropCollector : public TablePropertiesCollector {
std::string encoded;
PutVarint32(&encoded, count_);
*properties = UserCollectedProperties{
{"CountingUserTblPropCollector", message_}, {"Count", encoded},
{"CountingUserTblPropCollector", message_},
{"Count", encoded},
};
return Status::OK();
}
Expand Down Expand Up @@ -1713,6 +1716,46 @@ TEST_F(DBPropertiesTest, BlockCacheProperties) {
ASSERT_EQ(0, value);
}

TEST_F(DBPropertiesTest, TableCacheProperties) {
Options options;
uint64_t value, new_value;

options.env = CurrentOptions().env;

Reopen(options);

//
// test table_cache access is "live"
// TableCacheCapacity originally comes from DBOptions::max_open_files
// and can vary by system. Get its current value.
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kTableCacheCapacity, &value));

// now, change max_open_files to prove we are really accessing the value of
// interest
new_value = value / 2;
std::unordered_map<std::string, std::string> new_options;
new_options.insert(std::pair<std::string, std::string>(
"max_open_files", std::to_string(new_value)));
ASSERT_OK(db_->SetDBOptions(new_options));

// did the value we are reading update. NOTE: rocksdb internally reduces
// the value we pass by 10.
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kTableCacheCapacity, &value));
ASSERT_EQ(new_value - 10, value);

//
// TableCacheUsage is a count of open .sst files. Force the creation of a
// a new table file. First add a record via Put(). Then force that
// record from write buffer to new .sst via Flush(). New .sst
// automatically opens and gets position in table cache ... raising usage
// count
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kTableCacheUsage, &value));
ASSERT_OK(Put("foo", "v1"));
ASSERT_OK(db_->Flush(FlushOptions()));
ASSERT_TRUE(
db_->GetIntProperty(DB::Properties::kTableCacheUsage, &new_value));
ASSERT_EQ(new_value, value + 1);
}
#endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE

Expand Down
24 changes: 24 additions & 0 deletions db/internal_stats.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,8 @@ static const std::string estimate_oldest_key_time = "estimate-oldest-key-time";
static const std::string block_cache_capacity = "block-cache-capacity";
static const std::string block_cache_usage = "block-cache-usage";
static const std::string block_cache_pinned_usage = "block-cache-pinned-usage";
static const std::string tablecache_capacity = "table-cache-capacity";
static const std::string tablecache_usage = "table-cache-usage";
static const std::string options_statistics = "options-statistics";

const std::string DB::Properties::kNumFilesAtLevelPrefix =
Expand Down Expand Up @@ -348,6 +350,10 @@ const std::string DB::Properties::kBlockCacheUsage =
rocksdb_prefix + block_cache_usage;
const std::string DB::Properties::kBlockCachePinnedUsage =
rocksdb_prefix + block_cache_pinned_usage;
const std::string DB::Properties::kTableCacheCapacity =
rocksdb_prefix + tablecache_capacity;
const std::string DB::Properties::kTableCacheUsage =
rocksdb_prefix + tablecache_usage;
const std::string DB::Properties::kOptionsStatistics =
rocksdb_prefix + options_statistics;

Expand Down Expand Up @@ -487,6 +493,12 @@ const std::unordered_map<std::string, DBPropertyInfo>
{DB::Properties::kBlockCachePinnedUsage,
{false, nullptr, &InternalStats::HandleBlockCachePinnedUsage, nullptr,
nullptr}},
{DB::Properties::kTableCacheCapacity,
{false, nullptr, &InternalStats::HandleTableCacheCapacity, nullptr,
nullptr}},
{DB::Properties::kTableCacheUsage,
{false, nullptr, &InternalStats::HandleTableCacheUsage, nullptr,
nullptr}},
{DB::Properties::kOptionsStatistics,
{false, nullptr, nullptr, nullptr,
&DBImpl::GetPropertyHandleOptionsStatistics}},
Expand Down Expand Up @@ -987,6 +999,18 @@ bool InternalStats::HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* /*db*/,
return true;
}

bool InternalStats::HandleTableCacheCapacity(uint64_t* value, DBImpl* db,
Version* /*version*/) {
*value = static_cast<uint64_t>(db->table_cache_->GetCapacity());
return true;
}

bool InternalStats::HandleTableCacheUsage(uint64_t* value, DBImpl* db,
Version* /*version*/) {
*value = static_cast<uint64_t>(db->table_cache_->GetUsage());
return true;
}

void InternalStats::DumpDBStats(std::string* value) {
char buf[1000];
// DB-level stats, only available from default column family
Expand Down
23 changes: 13 additions & 10 deletions db/internal_stats.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ class InternalStats {
this->num_dropped_records += c.num_dropped_records;
this->count += c.count;
int num_of_reasons = static_cast<int>(CompactionReason::kNumOfReasons);
for (int i = 0; i< num_of_reasons; i++) {
for (int i = 0; i < num_of_reasons; i++) {
counts[i] += c.counts[i];
}
}
Expand Down Expand Up @@ -437,8 +437,8 @@ class InternalStats {
struct CFStatsSnapshot {
// ColumnFamily-level stats
CompactionStats comp_stats;
uint64_t ingest_bytes_flush; // Bytes written to L0 (Flush)
uint64_t stall_count; // Stall count
uint64_t ingest_bytes_flush; // Bytes written to L0 (Flush)
uint64_t stall_count; // Stall count
// Stats from compaction jobs - bytes written, bytes read, duration.
uint64_t compact_bytes_write;
uint64_t compact_bytes_read;
Expand Down Expand Up @@ -480,10 +480,10 @@ class InternalStats {

struct DBStatsSnapshot {
// DB-level stats
uint64_t ingest_bytes; // Bytes written by user
uint64_t wal_bytes; // Bytes written to WAL
uint64_t wal_synced; // Number of times WAL is synced
uint64_t write_with_wal; // Number of writes that request WAL
uint64_t ingest_bytes; // Bytes written by user
uint64_t wal_bytes; // Bytes written to WAL
uint64_t wal_synced; // Number of times WAL is synced
uint64_t write_with_wal; // Number of writes that request WAL
// These count the number of writes processed by the calling thread or
// another thread.
uint64_t write_other;
Expand Down Expand Up @@ -594,6 +594,8 @@ class InternalStats {
bool HandleBlockCacheUsage(uint64_t* value, DBImpl* db, Version* version);
bool HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* db,
Version* version);
bool HandleTableCacheCapacity(uint64_t* value, DBImpl* db, Version* version);
bool HandleTableCacheUsage(uint64_t* value, DBImpl* db, Version* version);
// Total number of background errors encountered. Every time a flush task
// or compaction task fails, this counter is incremented. The failure can
// be caused by any possible reason, including file system errors, out of
Expand Down Expand Up @@ -697,13 +699,14 @@ class InternalStats {
return false;
}

bool GetIntProperty(const DBPropertyInfo& /*property_info*/, uint64_t* /*value*/,
DBImpl* /*db*/) const {
bool GetIntProperty(const DBPropertyInfo& /*property_info*/,
uint64_t* /*value*/, DBImpl* /*db*/) const {
return false;
}

bool GetIntPropertyOutOfMutex(const DBPropertyInfo& /*property_info*/,
Version* /*version*/, uint64_t* /*value*/) const {
Version* /*version*/,
uint64_t* /*value*/) const {
return false;
}
};
Expand Down
9 changes: 9 additions & 0 deletions include/rocksdb/db.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -923,6 +923,13 @@ class DB {
// entries being pinned.
static const std::string kBlockCachePinnedUsage;

// "rocksdb.table-cache-capacity" - returns table cache capacity.
static const std::string kTableCacheCapacity;

// "rocksdb.table-cache-usage" - returns the memory size for the entries
// residing in table cache.
static const std::string kTableCacheUsage;

// "rocksdb.options-statistics" - returns multi-line string
// of options.statistics
static const std::string kOptionsStatistics;
Expand Down Expand Up @@ -986,6 +993,8 @@ class DB {
// "rocksdb.block-cache-capacity"
// "rocksdb.block-cache-usage"
// "rocksdb.block-cache-pinned-usage"
// "rocksdb.table-cache-capacity"
// "rocksdb.table-cache-usage"
virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
const Slice& property, uint64_t* value) = 0;
virtual bool GetIntProperty(const Slice& property, uint64_t* value) {
Expand Down

0 comments on commit 54a9cb4

Please sign in to comment.