[4/4][ResourceMngmt] Account Bloom/Ribbon filter construction memory in global memory limit

HISTORY.md

@@ -6,6 +6,7 @@
 * Provided support for tracking per-sst user-defined timestamp information in MANIFEST.
 * Added new option "adaptive_readahead" in ReadOptions. For iterators, RocksDB does auto-readahead on noticing sequential reads and by enabling this option, readahead_size of current file (if reads are sequential) will be carried forward to next file instead of starting from the scratch at each level (except L0 level files). If reads are not sequential it will fall back to 8KB. This option is applicable only for RocksDB internal prefetch buffer and isn't supported with underlying file system prefetching.
 * Added the read count and read bytes related stats to Statistics for tiered storage hot, warm, and cold file reads.
+* Added an option to dynamically charge an updating estimated memory usage of block-based table building to block cache if block cache available. It currently only includes charging memory usage of constructing (new) Bloom Filter and Ribbon Filter to block cache. To enable this feature, set `BlockBasedTableOptions::reserve_table_builder_memory = true`.
 
 ### Bug Fixes
 * Prevent a `CompactRange()` with `CompactRangeOptions::change_level == true` from possibly causing corruption to the LSM state (overlapping files within a level) when run in parallel with another manual compaction. Note that setting `force_consistency_checks == true` (the default) would cause the DB to enter read-only mode in this scenario and return `Status::Corruption`, rather than committing any corruption.

cache/cache_entry_roles.cc

@@ -20,6 +20,7 @@ std::array<const char*, kNumCacheEntryRoles> kCacheEntryRoleToCamelString{{
     "OtherBlock",
     "WriteBuffer",
     "CompressionDictionaryBuildingBuffer",
+    "FilterConstruction",
     "Misc",
 }};
 
@@ -32,6 +33,7 @@ std::array<const char*, kNumCacheEntryRoles> kCacheEntryRoleToHyphenString{{
     "other-block",
     "write-buffer",
     "compression-dictionary-building-buffer",
+    "filter-construction",
     "misc",
 }};
 

cache/cache_entry_roles.h

@@ -36,6 +36,9 @@ enum class CacheEntryRole {
   // BlockBasedTableBuilder reservations to account for
   // compression dictionary building buffer's memory usage
   kCompressionDictionaryBuildingBuffer,
+  // Filter reservations to account for
+  // (new) bloom and ribbon filter construction's memory usage
+  kFilterConstruction,
   // Default bucket, for miscellaneous cache entries. Do not use for
   // entries that could potentially add up to large usage.
   kMisc,

cache/cache_reservation_manager.cc

@@ -95,6 +95,11 @@ template Status
 CacheReservationManager::MakeCacheReservation<CacheEntryRole::kMisc>(
     std::size_t incremental_memory_used,
     std::unique_ptr<CacheReservationHandle<CacheEntryRole::kMisc>>* handle);
+template Status CacheReservationManager::MakeCacheReservation<
+    CacheEntryRole::kFilterConstruction>(
+    std::size_t incremental_memory_used,
+    std::unique_ptr<
+        CacheReservationHandle<CacheEntryRole::kFilterConstruction>>* handle);
 
 template <CacheEntryRole R>
 Status CacheReservationManager::IncreaseCacheReservation(
@@ -153,6 +158,14 @@ Slice CacheReservationManager::GetNextCacheKey() {
   return Slice(cache_key_, static_cast<std::size_t>(end - cache_key_));
 }
 
+template <CacheEntryRole R>
+Cache::DeleterFn CacheReservationManager::TEST_GetNoopDeleterForRole() {
+  return GetNoopDeleterForRole<R>();
+}
+
+template Cache::DeleterFn CacheReservationManager::TEST_GetNoopDeleterForRole<
+    CacheEntryRole::kFilterConstruction>();
+
 template <CacheEntryRole R>
 CacheReservationHandle<R>::CacheReservationHandle(
     std::size_t incremental_memory_used,
@@ -175,4 +188,5 @@ CacheReservationHandle<R>::~CacheReservationHandle() {
 // Explicitly instantiate templates for "CacheEntryRole" values we use.
 // This makes it possible to keep the template definitions in the .cc file.
 template class CacheReservationHandle<CacheEntryRole::kMisc>;
+template class CacheReservationHandle<CacheEntryRole::kFilterConstruction>;
 }  // namespace ROCKSDB_NAMESPACE

cache/cache_reservation_manager.h

@@ -150,6 +150,12 @@ class CacheReservationManager
 
   static constexpr std::size_t GetDummyEntrySize() { return kSizeDummyEntry; }
 
+  // For testing only - it is to help ensure the NoopDeleterForRole<R>
+  // accessed from CacheReservationManager and the one accessed from the test
+  // are from the same translation units
+  template <CacheEntryRole R>
+  static Cache::DeleterFn TEST_GetNoopDeleterForRole();
+
  private:
   static constexpr std::size_t kSizeDummyEntry = 256 * 1024;
   // The key will be longer than keys for blocks in SST files so they won't