Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[1/4][PerfImprove][ResourceMngmt] Deallocate payload of BlockBasedTableBuilder::Rep::FilterBlockBuilder earlier for Full/PartitionedFilter #9070

Closed
Closed
Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
33df98c
Early deallocation of BlockBasedTableBuilder::Rep::FilterBlockBuilder
hx235 Oct 24, 2021
2d9e438
Update HISTORY.md
hx235 Oct 24, 2021
5c0b5b2
Early dellocate filter data payload instead of the FilterBlockBuilder…
hx235 Oct 26, 2021
7379a51
Update HISTORY.md
hx235 Oct 26, 2021
ba04424
Default valued filter_data
hx235 Oct 26, 2021
1f47508
Reword HISTORY.md
hx235 Oct 26, 2021
de05361
Some polishing
hx235 Oct 26, 2021
517364b
Move filter_data.reset(); in the loop/remove r->filter_builder.reset(…
hx235 Oct 26, 2021
540df79
Move declaration filter_data inside the loop
hx235 Oct 26, 2021
cb767df
try to fail some test
hx235 Oct 26, 2021
cdc0cd4
Fix a bug causing new logic not to be executed
hx235 Oct 26, 2021
b526afa
Clarify FilterBlock::Finish interface
hx235 Oct 26, 2021
91f09e6
Handle partitioned filter case & add tests
hx235 Oct 28, 2021
a7207b2
Updated HISTORY.md
hx235 Oct 28, 2021
7798baa
Fix some format
hx235 Oct 28, 2021
4901410
remove redundant test
hx235 Oct 29, 2021
8ab01c5
Unify filters and filter_gc
hx235 Oct 29, 2021
7d69f58
Make format
hx235 Oct 29, 2021
b31db63
Remove changes to table_test
hx235 Oct 29, 2021
37488c6
Remove comment for weird api
hx235 Oct 30, 2021
581ef76
Fix edge case when partitioned filter passsed in nullptr output param
hx235 Nov 2, 2021
1d3003b
Simplify the writing in FullFilterBlockBuilder::Finish's return
hx235 Nov 2, 2021
732cfcb
reset last_filter_data after filters empty
hx235 Nov 3, 2021
d80cf48
Moved HISTORY update to correct section
hx235 Nov 4, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
remove redundant test
  • Loading branch information
hx235 committed Nov 3, 2021
commit 4901410cc1e3b6ec0de6224f7e51e16701ced40a
119 changes: 0 additions & 119 deletions table/table_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -5332,125 +5332,6 @@ TEST_P(
ASSERT_OK(builder->Finish());
EXPECT_EQ(cache->GetPinnedUsage(), 0 * kSizeDummyEntry);
}

class BlockBasedTableEarlyDeallocateFilterDataTest
: public TableTest,
virtual public ::testing::WithParamInterface<
std::tuple<uint32_t /* format */, BlockBasedTable::Rep::FilterType,
BloomFilterPolicy::Mode>> {
public:
BlockBasedTableEarlyDeallocateFilterDataTest()
: format_(std::get<0>(GetParam())),
filter_type_(std::get<1>(GetParam())),
mode_(std::get<2>(GetParam())) {}

BlockBasedTableOptions GetBlockBasedTableOptions() {
BlockBasedTableOptions table_options;

table_options.format_version = format_;

if (filter_type_ == BlockBasedTable::Rep::FilterType::kPartitionedFilter) {
table_options.partition_filters = true;
table_options.index_type =
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
}

switch (mode_) {
case BloomFilterPolicy::Mode::kFastLocalBloom:
table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
break;
case BloomFilterPolicy::Mode::kStandard128Ribbon:
table_options.filter_policy.reset(NewRibbonFilterPolicy(10, -1));
break;
case BloomFilterPolicy::Mode::kDeprecatedBlock:
table_options.filter_policy.reset(NewBloomFilterPolicy(10, true));
break;
default:
break;
}

return table_options;
}

private:
uint32_t format_;
BlockBasedTable::Rep::FilterType filter_type_;
BloomFilterPolicy::Mode mode_;
};

INSTANTIATE_TEST_CASE_P(
BlockBasedTableOptions, BlockBasedTableEarlyDeallocateFilterDataTest,
::testing::Values(
std::make_tuple(test::kDefaultFormatVersion,
BlockBasedTable::Rep::FilterType::kFullFilter,
BloomFilterPolicy::Mode::kFastLocalBloom),
std::make_tuple(test::kDefaultFormatVersion,
BlockBasedTable::Rep::FilterType::kFullFilter,
BloomFilterPolicy::Mode::kStandard128Ribbon),
std::make_tuple(test::kDefaultFormatVersion,
BlockBasedTable::Rep::FilterType::kPartitionedFilter,
BloomFilterPolicy::Mode::kFastLocalBloom),
std::make_tuple(test::kDefaultFormatVersion,
BlockBasedTable::Rep::FilterType::kPartitionedFilter,
BloomFilterPolicy::Mode::kStandard128Ribbon),
std::make_tuple(test::kDefaultFormatVersion,
BlockBasedTable::Rep::FilterType::kBlockFilter,
BloomFilterPolicy::Mode::kDeprecatedBlock),
std::make_tuple(test::kLatestFormatVersion,
BlockBasedTable::Rep::FilterType::kFullFilter,
BloomFilterPolicy::Mode::kFastLocalBloom),
std::make_tuple(test::kLatestFormatVersion,
BlockBasedTable::Rep::FilterType::kFullFilter,
BloomFilterPolicy::Mode::kStandard128Ribbon),
std::make_tuple(test::kLatestFormatVersion,
BlockBasedTable::Rep::FilterType::kPartitionedFilter,
BloomFilterPolicy::Mode::kFastLocalBloom),
std::make_tuple(test::kLatestFormatVersion,
BlockBasedTable::Rep::FilterType::kPartitionedFilter,
BloomFilterPolicy::Mode::kStandard128Ribbon),
std::make_tuple(test::kLatestFormatVersion,
BlockBasedTable::Rep::FilterType::kBlockFilter,
BloomFilterPolicy::Mode::kDeprecatedBlock)));

// To verify such an early deallocation of filter data in Full/Partition Filter
// does not cause segmentation fault and pass ASAN/UBSAN test
TEST_P(BlockBasedTableEarlyDeallocateFilterDataTest,
EarlyDeallocateFilterDataTest) {
BlockBasedTableOptions table_options = GetBlockBasedTableOptions();

Options options;
options.compression = kSnappyCompression;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));

test::StringSink* sink = new test::StringSink();
std::unique_ptr<FSWritableFile> holder(sink);
std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
std::move(holder), "test_file_name", FileOptions()));

ImmutableOptions ioptions(options);
MutableCFOptions moptions(options);
InternalKeyComparator ikc(options.comparator);
IntTblPropCollectorFactories int_tbl_prop_collector_factories;

std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, ikc,
&int_tbl_prop_collector_factories, kSnappyCompression,
options.compression_opts, kUnknownColumnFamily,
"test_cf", -1 /* level */),
file_writer.get()));

const std::size_t num_entries = 10000;

for (std::size_t i = 0; i < num_entries; ++i) {
std::string key = DBTestBase::Key(static_cast<int>(i));
std::string value = "val" + std::to_string(static_cast<int>(i));
InternalKey ik(key, i /* sequnce number */, kTypeValue);
builder->Add(ik.Encode(), value);
}

ASSERT_OK(builder->Finish());
}

} // namespace ROCKSDB_NAMESPACE

int main(int argc, char** argv) {
Expand Down