diff --git a/conf/pika.conf b/conf/pika.conf index 3fcb5d5158..391ed24825 100644 --- a/conf/pika.conf +++ b/conf/pika.conf @@ -337,6 +337,12 @@ level0-slowdown-writes-trigger : 20 # rocksdb level0_file_num_compaction_trigger level0-file-num-compaction-trigger : 4 +# rocksdb level0_file_num_compaction_trigger +level0-file-num-compaction-trigger : 4 + +#rocksdb statistics tickers +open_rocksdb_statistics_tickers : no + # The maximum size of the response package to client to prevent memory # exhaustion caused by commands like 'keys *' and 'Scan' which can generate huge response. # Supported Units [K|M|G]. The default unit is in [bytes]. diff --git a/include/pika_conf.h b/include/pika_conf.h index e93a5e7e5b..5ba69c5297 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -170,6 +170,9 @@ class PikaConf : public pstd::BaseConf { std::shared_lock l(rwlock_); return max_total_wal_size_; } + bool open_rocksdb_statistics_tickers() { + return open_rocksdb_statistics_tickers_; + } int64_t max_client_response_size() { std::shared_lock l(rwlock_); return max_client_response_size_; @@ -858,6 +861,7 @@ class PikaConf : public pstd::BaseConf { int64_t thread_migrate_keys_num_ = 0; int64_t max_write_buffer_size_ = 0; int64_t max_total_wal_size_ = 0; + bool open_rocksdb_statistics_tickers_ = false; int max_write_buffer_num_ = 0; int min_write_buffer_number_to_merge_ = 1; int level0_stop_writes_trigger_ = 36; diff --git a/src/pika_admin.cc b/src/pika_admin.cc index 9e974bd7c1..be545585ba 100644 --- a/src/pika_admin.cc +++ b/src/pika_admin.cc @@ -1306,7 +1306,12 @@ void InfoCmd::InfoRocksDB(std::string& info) { } std::string rocksdb_info; db_item.second->DBLockShared(); - db_item.second->storage()->GetRocksDBInfo(rocksdb_info); + if (g_pika_conf->open_rocksdb_statistics_tickers()) { + db_item.second->storage()->GetRocksDBInfo(rocksdb_info, true); + } else { + db_item.second->storage()->GetRocksDBInfo(rocksdb_info, false); + } + db_item.second->DBUnlockShared(); tmp_stream << rocksdb_info; } diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h index 0b520f5800..9ea85798f4 100644 --- a/src/storage/include/storage/storage.h +++ b/src/storage/include/storage/storage.h @@ -1091,7 +1091,7 @@ class Storage { const std::string& db_type, const std::unordered_map& options); Status EnableAutoCompaction(const OptionType& option_type, const std::string& db_type, const std::unordered_map& options); - void GetRocksDBInfo(std::string& info); + void GetRocksDBInfo(std::string& info, bool open_ticker); private: std::vector> insts_; diff --git a/src/storage/src/redis.cc b/src/storage/src/redis.cc index ccd14ba759..238b72ca16 100644 --- a/src/storage/src/redis.cc +++ b/src/storage/src/redis.cc @@ -266,7 +266,7 @@ Status Redis::SetOptions(const OptionType& option_type, const std::unordered_map return s; } -void Redis::GetRocksDBInfo(std::string& info, const char* prefix) { +void Redis::GetRocksDBInfo(std::string& info, const char* prefix, bool open_ticker) { std::ostringstream string_stream; string_stream << "#" << prefix << "RocksDB" << "\r\n"; @@ -290,7 +290,7 @@ void Redis::GetRocksDBInfo(std::string& info, const char* prefix) { } }; - auto write_ticker_count = [&](uint32_t, const char *metric) { + auto write_ticker_count = [&](uint32_t tick_type, const char *metric) { if (db_statistics_ == nullptr) { return; } @@ -311,32 +311,10 @@ void Redis::GetRocksDBInfo(std::string& info, const char* prefix) { write_aggregated_int_property(rocksdb::DB::Properties::kNumImmutableMemTableFlushed, "num_immutable_mem_table_flushed"); write_aggregated_int_property(rocksdb::DB::Properties::kMemTableFlushPending, "mem_table_flush_pending"); write_aggregated_int_property(rocksdb::DB::Properties::kNumRunningFlushes, "num_running_flushes"); - write_ticker_count(rocksdb::Tickers::MEMTABLE_HIT, "memtable_hit"); - write_ticker_count(rocksdb::Tickers::MEMTABLE_MISS, "memtable_miss"); - - write_ticker_count(rocksdb::Tickers::BYTES_WRITTEN, "bytes_written"); - write_ticker_count(rocksdb::Tickers::BYTES_READ, "bytes_read"); - write_ticker_count(rocksdb::Tickers::ITER_BYTES_READ, "iter_bytes_read"); - write_ticker_count(rocksdb::Tickers::GET_HIT_L0, "get_hit_l0"); - write_ticker_count(rocksdb::Tickers::GET_HIT_L1, "get_hit_l1"); - write_ticker_count(rocksdb::Tickers::GET_HIT_L2_AND_UP, "get_hit_l2_and_up"); - - write_ticker_count(rocksdb::Tickers::BLOOM_FILTER_USEFUL, "bloom_filter_useful"); - write_ticker_count(rocksdb::Tickers::BLOOM_FILTER_FULL_POSITIVE, "bloom_filter_full_positive"); - write_ticker_count(rocksdb::Tickers::BLOOM_FILTER_FULL_TRUE_POSITIVE, "bloom_filter_full_true_positive"); - write_ticker_count(rocksdb::Tickers::BLOOM_FILTER_PREFIX_CHECKED, "bloom_filter_prefix_checked"); - write_ticker_count(rocksdb::Tickers::BLOOM_FILTER_PREFIX_USEFUL, "bloom_filter_prefix_useful"); // compaction write_aggregated_int_property(rocksdb::DB::Properties::kCompactionPending, "compaction_pending"); write_aggregated_int_property(rocksdb::DB::Properties::kNumRunningCompactions, "num_running_compactions"); - write_ticker_count(rocksdb::Tickers::COMPACTION_KEY_DROP_NEWER_ENTRY, "compaction_key_drop_newer_entry"); - write_ticker_count(rocksdb::Tickers::COMPACTION_KEY_DROP_OBSOLETE, "compaction_key_drop_obsolete"); - write_ticker_count(rocksdb::Tickers::COMPACTION_KEY_DROP_USER, "compaction_key_drop_user"); - write_ticker_count(rocksdb::Tickers::COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE, "compaction_optimized_del_drop_obsolete"); - write_ticker_count(rocksdb::Tickers::COMPACT_READ_BYTES, "compact_read_bytes"); - write_ticker_count(rocksdb::Tickers::COMPACT_WRITE_BYTES, "compact_write_bytes"); - write_ticker_count(rocksdb::Tickers::FLUSH_WRITE_BYTES, "flush_write_bytes"); // background errors write_aggregated_int_property(rocksdb::DB::Properties::kBackgroundErrors, "background_errors"); @@ -349,21 +327,6 @@ void Redis::GetRocksDBInfo(std::string& info, const char* prefix) { // keys write_aggregated_int_property(rocksdb::DB::Properties::kEstimateNumKeys, "estimate_num_keys"); - write_ticker_count(rocksdb::Tickers::NUMBER_KEYS_READ, "number_keys_read"); - write_ticker_count(rocksdb::Tickers::NUMBER_KEYS_WRITTEN, "number_keys_written"); - write_ticker_count(rocksdb::Tickers::NUMBER_KEYS_UPDATED, "number_keys_updated"); - write_ticker_count(rocksdb::Tickers::NUMBER_OF_RESEEKS_IN_ITERATION, "number_of_reseeks_in_iteration"); - - write_ticker_count(rocksdb::Tickers::NUMBER_DB_SEEK, "number_db_seek"); - write_ticker_count(rocksdb::Tickers::NUMBER_DB_NEXT, "number_db_next"); - write_ticker_count(rocksdb::Tickers::NUMBER_DB_PREV, "number_db_prev"); - write_ticker_count(rocksdb::Tickers::NUMBER_DB_SEEK_FOUND, "number_db_seek_found"); - write_ticker_count(rocksdb::Tickers::NUMBER_DB_NEXT_FOUND, "number_db_next_found"); - write_ticker_count(rocksdb::Tickers::NUMBER_DB_PREV_FOUND, "number_db_prev_found"); - write_ticker_count(rocksdb::Tickers::LAST_LEVEL_READ_BYTES, "last_level_read_bytes"); - write_ticker_count(rocksdb::Tickers::LAST_LEVEL_READ_COUNT, "last_level_read_count"); - write_ticker_count(rocksdb::Tickers::NON_LAST_LEVEL_READ_BYTES, "non_last_level_read_bytes"); - write_ticker_count(rocksdb::Tickers::NON_LAST_LEVEL_READ_COUNT, "non_last_level_read_count"); // table readers mem write_aggregated_int_property(rocksdb::DB::Properties::kEstimateTableReadersMem, "estimate_table_readers_mem"); @@ -395,8 +358,6 @@ void Redis::GetRocksDBInfo(std::string& info, const char* prefix) { write_property(rocksdb::DB::Properties::kCompressionRatioAtLevelPrefix+"6", "compression_ratio_at_level6"); write_aggregated_int_property(rocksdb::DB::Properties::kTotalSstFilesSize, "total_sst_files_size"); write_aggregated_int_property(rocksdb::DB::Properties::kLiveSstFilesSize, "live_sst_files_size"); - write_ticker_count(rocksdb::Tickers::NO_FILE_OPENS, "no_file_opens"); - write_ticker_count(rocksdb::Tickers::NO_FILE_ERRORS, "no_file_errors"); // pending compaction bytes write_aggregated_int_property(rocksdb::DB::Properties::kEstimatePendingCompactionBytes, "estimate_pending_compaction_bytes"); @@ -405,43 +366,96 @@ void Redis::GetRocksDBInfo(std::string& info, const char* prefix) { write_aggregated_int_property(rocksdb::DB::Properties::kBlockCacheCapacity, "block_cache_capacity"); write_aggregated_int_property(rocksdb::DB::Properties::kBlockCacheUsage, "block_cache_usage"); write_aggregated_int_property(rocksdb::DB::Properties::kBlockCachePinnedUsage, "block_cache_pinned_usage"); - write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_INDEX_HIT, "block_cache_index_hit"); - write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_INDEX_MISS, "block_cache_index_miss"); - write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_FILTER_HIT, "block_cache_filter_hit"); - write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_FILTER_MISS, "block_cache_filter_miss"); - write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_DATA_HIT, "block_cache_data_hit"); - write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_DATA_MISS, "block_cache_data_miss"); - write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_BYTES_READ, "block_cache_bytes_read"); - write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_BYTES_WRITE, "block_cache_bytes_write"); // blob files write_aggregated_int_property(rocksdb::DB::Properties::kNumBlobFiles, "num_blob_files"); write_aggregated_int_property(rocksdb::DB::Properties::kBlobStats, "blob_stats"); write_aggregated_int_property(rocksdb::DB::Properties::kTotalBlobFileSize, "total_blob_file_size"); write_aggregated_int_property(rocksdb::DB::Properties::kLiveBlobFileSize, "live_blob_file_size"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_NUM_KEYS_WRITTEN, "blob_db_num_keys_written"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_NUM_KEYS_READ, "blob_db_num_keys_read"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_BYTES_WRITTEN, "blob_db_bytes_written"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_BYTES_READ, "blob_db_bytes_read"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_NUM_SEEK, "blob_db_num_seek"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_NUM_NEXT, "blob_db_num_next"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_NUM_PREV, "blob_db_num_prev"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_BLOB_FILE_BYTES_WRITTEN, "blob_db_blob_file_bytes_written"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_BLOB_FILE_BYTES_READ, "blob_db_blob_file_bytes_read"); - - write_ticker_count(rocksdb::Tickers::BLOB_DB_GC_NUM_FILES, "blob_db_gc_num_files"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_GC_NUM_NEW_FILES, "blob_db_gc_num_new_files"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_GC_NUM_KEYS_RELOCATED, "blob_db_gc_num_keys_relocated"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_GC_BYTES_RELOCATED, "blob_db_gc_bytes_relocated"); write_aggregated_int_property(rocksdb::DB::Properties::kBlobCacheCapacity, "blob_cache_capacity"); write_aggregated_int_property(rocksdb::DB::Properties::kBlobCacheUsage, "blob_cache_usage"); write_aggregated_int_property(rocksdb::DB::Properties::kBlobCachePinnedUsage, "blob_cache_pinned_usage"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_CACHE_MISS, "blob_db_cache_miss"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_CACHE_HIT, "blob_db_cache_hit"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_CACHE_BYTES_READ, "blob_db_cache_bytes_read"); - write_ticker_count(rocksdb::Tickers::BLOB_DB_CACHE_BYTES_WRITE, "blob_db_cache_bytes_write"); + if (open_ticker) { + // memtables num + write_ticker_count(rocksdb::Tickers::MEMTABLE_HIT, "memtable_hit"); + write_ticker_count(rocksdb::Tickers::MEMTABLE_MISS, "memtable_miss"); + + write_ticker_count(rocksdb::Tickers::BYTES_WRITTEN, "bytes_written"); + write_ticker_count(rocksdb::Tickers::BYTES_READ, "bytes_read"); + write_ticker_count(rocksdb::Tickers::ITER_BYTES_READ, "iter_bytes_read"); + write_ticker_count(rocksdb::Tickers::GET_HIT_L0, "get_hit_l0"); + write_ticker_count(rocksdb::Tickers::GET_HIT_L1, "get_hit_l1"); + write_ticker_count(rocksdb::Tickers::GET_HIT_L2_AND_UP, "get_hit_l2_and_up"); + + write_ticker_count(rocksdb::Tickers::BLOOM_FILTER_USEFUL, "bloom_filter_useful"); + write_ticker_count(rocksdb::Tickers::BLOOM_FILTER_FULL_POSITIVE, "bloom_filter_full_positive"); + write_ticker_count(rocksdb::Tickers::BLOOM_FILTER_FULL_TRUE_POSITIVE, "bloom_filter_full_true_positive"); + write_ticker_count(rocksdb::Tickers::BLOOM_FILTER_PREFIX_CHECKED, "bloom_filter_prefix_checked"); + write_ticker_count(rocksdb::Tickers::BLOOM_FILTER_PREFIX_USEFUL, "bloom_filter_prefix_useful"); + + // compaction + write_ticker_count(rocksdb::Tickers::COMPACTION_KEY_DROP_NEWER_ENTRY, "compaction_key_drop_newer_entry"); + write_ticker_count(rocksdb::Tickers::COMPACTION_KEY_DROP_OBSOLETE, "compaction_key_drop_obsolete"); + write_ticker_count(rocksdb::Tickers::COMPACTION_KEY_DROP_USER, "compaction_key_drop_user"); + write_ticker_count(rocksdb::Tickers::COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE, "compaction_optimized_del_drop_obsolete"); + write_ticker_count(rocksdb::Tickers::COMPACT_READ_BYTES, "compact_read_bytes"); + write_ticker_count(rocksdb::Tickers::COMPACT_WRITE_BYTES, "compact_write_bytes"); + write_ticker_count(rocksdb::Tickers::FLUSH_WRITE_BYTES, "flush_write_bytes"); + + // keys + write_ticker_count(rocksdb::Tickers::NUMBER_KEYS_READ, "number_keys_read"); + write_ticker_count(rocksdb::Tickers::NUMBER_KEYS_WRITTEN, "number_keys_written"); + write_ticker_count(rocksdb::Tickers::NUMBER_KEYS_UPDATED, "number_keys_updated"); + write_ticker_count(rocksdb::Tickers::NUMBER_OF_RESEEKS_IN_ITERATION, "number_of_reseeks_in_iteration"); + + write_ticker_count(rocksdb::Tickers::NUMBER_DB_SEEK, "number_db_seek"); + write_ticker_count(rocksdb::Tickers::NUMBER_DB_NEXT, "number_db_next"); + write_ticker_count(rocksdb::Tickers::NUMBER_DB_PREV, "number_db_prev"); + write_ticker_count(rocksdb::Tickers::NUMBER_DB_SEEK_FOUND, "number_db_seek_found"); + write_ticker_count(rocksdb::Tickers::NUMBER_DB_NEXT_FOUND, "number_db_next_found"); + write_ticker_count(rocksdb::Tickers::NUMBER_DB_PREV_FOUND, "number_db_prev_found"); + write_ticker_count(rocksdb::Tickers::LAST_LEVEL_READ_BYTES, "last_level_read_bytes"); + write_ticker_count(rocksdb::Tickers::LAST_LEVEL_READ_COUNT, "last_level_read_count"); + write_ticker_count(rocksdb::Tickers::NON_LAST_LEVEL_READ_BYTES, "non_last_level_read_bytes"); + write_ticker_count(rocksdb::Tickers::NON_LAST_LEVEL_READ_COUNT, "non_last_level_read_count"); + + // sst files + write_ticker_count(rocksdb::Tickers::NO_FILE_OPENS, "no_file_opens"); + write_ticker_count(rocksdb::Tickers::NO_FILE_ERRORS, "no_file_errors"); + + // block cache + write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_INDEX_HIT, "block_cache_index_hit"); + write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_INDEX_MISS, "block_cache_index_miss"); + write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_FILTER_HIT, "block_cache_filter_hit"); + write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_FILTER_MISS, "block_cache_filter_miss"); + write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_DATA_HIT, "block_cache_data_hit"); + write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_DATA_MISS, "block_cache_data_miss"); + write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_BYTES_READ, "block_cache_bytes_read"); + write_ticker_count(rocksdb::Tickers::BLOCK_CACHE_BYTES_WRITE, "block_cache_bytes_write"); + + // blob files + write_ticker_count(rocksdb::Tickers::BLOB_DB_NUM_KEYS_WRITTEN, "blob_db_num_keys_written"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_NUM_KEYS_READ, "blob_db_num_keys_read"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_BYTES_WRITTEN, "blob_db_bytes_written"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_BYTES_READ, "blob_db_bytes_read"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_NUM_SEEK, "blob_db_num_seek"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_NUM_NEXT, "blob_db_num_next"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_NUM_PREV, "blob_db_num_prev"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_BLOB_FILE_BYTES_WRITTEN, "blob_db_blob_file_bytes_written"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_BLOB_FILE_BYTES_READ, "blob_db_blob_file_bytes_read"); + + write_ticker_count(rocksdb::Tickers::BLOB_DB_GC_NUM_FILES, "blob_db_gc_num_files"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_GC_NUM_NEW_FILES, "blob_db_gc_num_new_files"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_GC_NUM_KEYS_RELOCATED, "blob_db_gc_num_keys_relocated"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_GC_BYTES_RELOCATED, "blob_db_gc_bytes_relocated"); + + write_ticker_count(rocksdb::Tickers::BLOB_DB_CACHE_MISS, "blob_db_cache_miss"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_CACHE_HIT, "blob_db_cache_hit"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_CACHE_BYTES_READ, "blob_db_cache_bytes_read"); + write_ticker_count(rocksdb::Tickers::BLOB_DB_CACHE_BYTES_WRITE, "blob_db_cache_bytes_write"); + } // column family stats std::map mapvalues; db_->rocksdb::DB::GetMapProperty(rocksdb::DB::Properties::kCFStats,&mapvalues); diff --git a/src/storage/src/redis.h b/src/storage/src/redis.h index 21c2d4f59e..59f520d3c5 100644 --- a/src/storage/src/redis.h +++ b/src/storage/src/redis.h @@ -242,7 +242,7 @@ class Redis { std::vector GetStreamCFHandles() { return {handles_.begin() + kMetaCF, handles_.end()}; } - void GetRocksDBInfo(std::string &info, const char *prefix); + void GetRocksDBInfo(std::string &info, const char *prefix, bool open_ticker); // Sets Commands Status SAdd(const Slice& key, const std::vector& members, int32_t* ret); diff --git a/src/storage/src/storage.cc b/src/storage/src/storage.cc index ddeac6dd37..375155a3ad 100644 --- a/src/storage/src/storage.cc +++ b/src/storage/src/storage.cc @@ -1920,11 +1920,11 @@ Status Storage::EnableAutoCompaction(const OptionType& option_type, return s; } -void Storage::GetRocksDBInfo(std::string& info) { +void Storage::GetRocksDBInfo(std::string& info, bool open_ticker) { char temp[12] = {0}; for (const auto& inst : insts_) { snprintf(temp, sizeof(temp), "instance%d_", inst->GetIndex()); - inst->GetRocksDBInfo(info, temp); + inst->GetRocksDBInfo(info, temp, open_ticker); } }