diff options
Diffstat (limited to 'vespalib')
47 files changed, 956 insertions, 320 deletions
diff --git a/vespalib/src/tests/btree/btree_store/btree_store_test.cpp b/vespalib/src/tests/btree/btree_store/btree_store_test.cpp index e7d923d0e87..974aafb392a 100644 --- a/vespalib/src/tests/btree/btree_store/btree_store_test.cpp +++ b/vespalib/src/tests/btree/btree_store/btree_store_test.cpp @@ -5,9 +5,12 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/gtest/gtest.h> using vespalib::GenerationHandler; +using vespalib::datastore::CompactionSpec; +using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryRef; namespace vespalib::btree { @@ -73,61 +76,115 @@ BTreeStoreTest::~BTreeStoreTest() inc_generation(); } +namespace { + +class ChangeWriter { + std::vector<EntryRef*> _old_refs; +public: + ChangeWriter(uint32_t capacity); + ~ChangeWriter(); + void write(const std::vector<EntryRef>& refs); + void emplace_back(EntryRef& ref) { _old_refs.emplace_back(&ref); } +}; + +ChangeWriter::ChangeWriter(uint32_t capacity) + : _old_refs() +{ + _old_refs.reserve(capacity); +} + +ChangeWriter::~ChangeWriter() = default; + +void +ChangeWriter::write(const std::vector<EntryRef> &refs) +{ + assert(refs.size() == _old_refs.size()); + auto old_ref_itr = _old_refs.begin(); + for (auto ref : refs) { + **old_ref_itr = ref; + ++old_ref_itr; + } + assert(old_ref_itr == _old_refs.end()); + _old_refs.clear(); +} + +} + void BTreeStoreTest::test_compact_sequence(uint32_t sequence_length) { auto &store = _store; + uint32_t entry_ref_offset_bits = TreeStore::RefType::offset_bits; EntryRef ref1 = add_sequence(4, 4 + sequence_length); EntryRef ref2 = add_sequence(5, 5 + sequence_length); - EntryRef old_ref1 = ref1; - EntryRef old_ref2 = ref2; std::vector<EntryRef> refs; + refs.reserve(2); + refs.emplace_back(ref1); + refs.emplace_back(ref2); + std::vector<EntryRef> temp_refs; for (int i = 0; i < 1000; ++i) { - refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length)); + temp_refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length)); } - for (auto& ref : refs) { + for (auto& ref : temp_refs) { store.clear(ref); } inc_generation(); + ChangeWriter change_writer(refs.size()); + std::vector<EntryRef> move_refs; + move_refs.reserve(refs.size()); auto usage_before = store.getMemoryUsage(); for (uint32_t pass = 0; pass < 15; ++pass) { - auto to_hold = store.start_compact_worst_buffers(); - ref1 = store.move(ref1); - ref2 = store.move(ref2); + CompactionSpec compaction_spec(true, false); + CompactionStrategy compaction_strategy; + auto to_hold = store.start_compact_worst_buffers(compaction_spec, compaction_strategy); + std::vector<bool> filter(TreeStore::RefType::numBuffers()); + for (auto buffer_id : to_hold) { + filter[buffer_id] = true; + } + for (auto& ref : refs) { + if (ref.valid() && filter[ref.buffer_id(entry_ref_offset_bits)]) { + move_refs.emplace_back(ref); + change_writer.emplace_back(ref); + } + } + store.move(move_refs); + change_writer.write(move_refs); + move_refs.clear(); store.finishCompact(to_hold); inc_generation(); } - EXPECT_NE(old_ref1, ref1); - EXPECT_NE(old_ref2, ref2); - EXPECT_EQ(make_exp_sequence(4, 4 + sequence_length), get_sequence(ref1)); - EXPECT_EQ(make_exp_sequence(5, 5 + sequence_length), get_sequence(ref2)); + EXPECT_NE(ref1, refs[0]); + EXPECT_NE(ref2, refs[1]); + EXPECT_EQ(make_exp_sequence(4, 4 + sequence_length), get_sequence(refs[0])); + EXPECT_EQ(make_exp_sequence(5, 5 + sequence_length), get_sequence(refs[1])); auto usage_after = store.getMemoryUsage(); EXPECT_GT(usage_before.deadBytes(), usage_after.deadBytes()); - store.clear(ref1); - store.clear(ref2); + store.clear(refs[0]); + store.clear(refs[1]); } TEST_F(BTreeStoreTest, require_that_nodes_for_multiple_btrees_are_compacted) { auto &store = this->_store; - EntryRef ref1 = add_sequence(4, 40); - EntryRef ref2 = add_sequence(100, 130); + std::vector<EntryRef> refs; + refs.emplace_back(add_sequence(4, 40)); + refs.emplace_back(add_sequence(100, 130)); store.clear(add_sequence(1000, 20000)); inc_generation(); auto usage_before = store.getMemoryUsage(); for (uint32_t pass = 0; pass < 15; ++pass) { - auto to_hold = store.start_compact_worst_btree_nodes(); - store.move_btree_nodes(ref1); - store.move_btree_nodes(ref2); + CompactionStrategy compaction_strategy; + auto to_hold = store.start_compact_worst_btree_nodes(compaction_strategy); + store.move_btree_nodes(refs); store.finish_compact_worst_btree_nodes(to_hold); inc_generation(); } - EXPECT_EQ(make_exp_sequence(4, 40), get_sequence(ref1)); - EXPECT_EQ(make_exp_sequence(100, 130), get_sequence(ref2)); + EXPECT_EQ(make_exp_sequence(4, 40), get_sequence(refs[0])); + EXPECT_EQ(make_exp_sequence(100, 130), get_sequence(refs[1])); auto usage_after = store.getMemoryUsage(); EXPECT_GT(usage_before.deadBytes(), usage_after.deadBytes()); - store.clear(ref1); - store.clear(ref2); + store.clear(refs[0]); + store.clear(refs[1]); } TEST_F(BTreeStoreTest, require_that_short_arrays_are_compacted) diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index 4af0b9672f2..bd4f4f8ee08 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -17,6 +17,7 @@ #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/test/btree/btree_printer.h> #include <vespa/vespalib/gtest/gtest.h> @@ -24,6 +25,7 @@ LOG_SETUP("btree_test"); using vespalib::GenerationHandler; +using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryRef; namespace vespalib::btree { @@ -1599,8 +1601,9 @@ TEST_F(BTreeTest, require_that_compaction_works) auto memory_usage_before = t.getAllocator().getMemoryUsage(); t.foreach_key([&before_list](int key) { before_list.emplace_back(key); }); make_iterators(t, before_list, before_iterators); + CompactionStrategy compaction_strategy; for (int i = 0; i < 15; ++i) { - t.compact_worst(); + t.compact_worst(compaction_strategy); } inc_generation(g, t); auto memory_usage_after = t.getAllocator().getMemoryUsage(); diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index dbd6d41f5e6..c58e357a9a1 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -3,6 +3,8 @@ #include <vespa/vespalib/test/datastore/buffer_stats.h> #include <vespa/vespalib/test/datastore/memstats.h> #include <vespa/vespalib/datastore/array_store.hpp> +#include <vespa/vespalib/datastore/compaction_spec.h> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -124,7 +126,9 @@ struct Fixture } template <typename TestedRefType> void compactWorst(bool compactMemory, bool compactAddressSpace) { - ICompactionContext::UP ctx = store.compactWorst(compactMemory, compactAddressSpace); + CompactionSpec compaction_spec(compactMemory, compactAddressSpace); + CompactionStrategy compaction_strategy; + ICompactionContext::UP ctx = store.compactWorst(compaction_spec, compaction_strategy); std::vector<TestedRefType> refs; for (auto itr = refStore.begin(); itr != refStore.end(); ++itr) { refs.emplace_back(itr->first); diff --git a/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp b/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp index 6e984f286c1..796e19a97d1 100644 --- a/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp +++ b/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/datastore/sharded_hash_map.h> +#include <vespa/vespalib/datastore/entry_ref_filter.h> #include <vespa/vespalib/datastore/i_compactable.h> #include <vespa/vespalib/datastore/unique_store_allocator.h> #include <vespa/vespalib/datastore/unique_store_comparator.h> @@ -12,12 +13,14 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/datastore/unique_store_allocator.hpp> +#include <iostream> #include <thread> #include <vespa/log/log.h> LOG_SETUP("vespalib_datastore_shared_hash_test"); using vespalib::datastore::EntryRef; +using vespalib::datastore::EntryRefFilter; using vespalib::datastore::ICompactable; using RefT = vespalib::datastore::EntryRefT<22>; using MyAllocator = vespalib::datastore::UniqueStoreAllocator<uint32_t, RefT>; @@ -27,6 +30,26 @@ using MyHashMap = vespalib::datastore::ShardedHashMap; using GenerationHandler = vespalib::GenerationHandler; using vespalib::makeLambdaTask; +constexpr uint32_t small_population = 50; +/* + * large_population should trigger multiple callbacks from normalize_values + * and foreach_value + */ +constexpr uint32_t large_population = 1200; + +namespace vespalib::datastore { + +/* + * Print EntryRef as RefT which is used by test_normalize_values and + * test_foreach_value to differentiate between buffers + */ +void PrintTo(const EntryRef &ref, std::ostream* os) { + RefT iref(ref); + *os << "RefT(" << iref.offset() << "," << iref.bufferId() << ")"; +} + +} + namespace { void consider_yield(uint32_t i) @@ -58,6 +81,19 @@ public: } }; +uint32_t select_buffer(uint32_t i) { + if ((i % 2) == 0) { + return 0; + } + if ((i % 3) == 0) { + return 1; + } + if ((i % 5) == 0) { + return 2; + } + return 3; +} + } struct DataStoreShardedHashTest : public ::testing::Test @@ -86,7 +122,11 @@ struct DataStoreShardedHashTest : public ::testing::Test void read_work(uint32_t cnt); void read_work(); void write_work(uint32_t cnt); - void populate_sample_data(); + void populate_sample_data(uint32_t cnt); + void populate_sample_values(uint32_t cnt); + void clear_sample_values(uint32_t cnt); + void test_normalize_values(bool use_filter, bool one_filter); + void test_foreach_value(bool one_filter); }; @@ -213,13 +253,94 @@ DataStoreShardedHashTest::write_work(uint32_t cnt) } void -DataStoreShardedHashTest::populate_sample_data() +DataStoreShardedHashTest::populate_sample_data(uint32_t cnt) { - for (uint32_t i = 0; i < 50; ++i) { + for (uint32_t i = 0; i < cnt; ++i) { insert(i); } } +void +DataStoreShardedHashTest::populate_sample_values(uint32_t cnt) +{ + for (uint32_t i = 0; i < cnt; ++i) { + MyCompare comp(_store, i); + auto result = _hash_map.find(comp, EntryRef()); + ASSERT_NE(result, nullptr); + EXPECT_EQ(i, _allocator.get_wrapped(result->first.load_relaxed()).value()); + result->second.store_relaxed(RefT(i + 200, select_buffer(i))); + } +} + +void +DataStoreShardedHashTest::clear_sample_values(uint32_t cnt) +{ + for (uint32_t i = 0; i < cnt; ++i) { + MyCompare comp(_store, i); + auto result = _hash_map.find(comp, EntryRef()); + ASSERT_NE(result, nullptr); + EXPECT_EQ(i, _allocator.get_wrapped(result->first.load_relaxed()).value()); + result->second.store_relaxed(EntryRef()); + } +} + +namespace { + +template <typename RefT> +EntryRefFilter +make_entry_ref_filter(bool one_filter) +{ + if (one_filter) { + EntryRefFilter filter(RefT::numBuffers(), RefT::offset_bits); + filter.add_buffer(3); + return filter; + } + return EntryRefFilter::create_all_filter(RefT::numBuffers(), RefT::offset_bits); +} + +} + +void +DataStoreShardedHashTest::test_normalize_values(bool use_filter, bool one_filter) +{ + populate_sample_data(large_population); + populate_sample_values(large_population); + if (use_filter) { + auto filter = make_entry_ref_filter<RefT>(one_filter); + EXPECT_TRUE(_hash_map.normalize_values([](std::vector<EntryRef> &refs) noexcept { for (auto &ref : refs) { RefT iref(ref); ref = RefT(iref.offset() + 300, iref.bufferId()); } }, filter)); + } else { + EXPECT_TRUE(_hash_map.normalize_values([](EntryRef ref) noexcept { RefT iref(ref); return RefT(iref.offset() + 300, iref.bufferId()); })); + } + for (uint32_t i = 0; i < large_population; ++i) { + MyCompare comp(_store, i); + auto result = _hash_map.find(comp, EntryRef()); + ASSERT_NE(result, nullptr); + EXPECT_EQ(i, _allocator.get_wrapped(result->first.load_relaxed()).value()); + ASSERT_EQ(select_buffer(i), RefT(result->second.load_relaxed()).bufferId()); + if (use_filter && one_filter && select_buffer(i) != 3) { + ASSERT_EQ(i + 200, RefT(result->second.load_relaxed()).offset()); + } else { + ASSERT_EQ(i + 500, RefT(result->second.load_relaxed()).offset()); + } + result->second.store_relaxed(EntryRef()); + } +} + +void +DataStoreShardedHashTest::test_foreach_value(bool one_filter) +{ + populate_sample_data(large_population); + populate_sample_values(large_population); + + auto filter = make_entry_ref_filter<RefT>(one_filter); + std::vector<EntryRef> exp_refs; + EXPECT_FALSE(_hash_map.normalize_values([&exp_refs](std::vector<EntryRef>& refs) { exp_refs.insert(exp_refs.end(), refs.begin(), refs.end()); }, filter)); + std::vector<EntryRef> act_refs; + _hash_map.foreach_value([&act_refs](const std::vector<EntryRef> &refs) { act_refs.insert(act_refs.end(), refs.begin(), refs.end()); }, filter); + EXPECT_EQ(exp_refs, act_refs); + clear_sample_values(large_population); +} + TEST_F(DataStoreShardedHashTest, single_threaded_reader_without_updates) { _report_work = true; @@ -254,7 +375,7 @@ TEST_F(DataStoreShardedHashTest, memory_usage_is_reported) EXPECT_EQ(0, initial_usage.deadBytes()); EXPECT_EQ(0, initial_usage.allocatedBytesOnHold()); auto guard = _generationHandler.takeGuard(); - for (uint32_t i = 0; i < 50; ++i) { + for (uint32_t i = 0; i < small_population; ++i) { insert(i); } auto usage = _hash_map.get_memory_usage(); @@ -264,30 +385,31 @@ TEST_F(DataStoreShardedHashTest, memory_usage_is_reported) TEST_F(DataStoreShardedHashTest, foreach_key_works) { - populate_sample_data(); + populate_sample_data(small_population); std::vector<uint32_t> keys; _hash_map.foreach_key([this, &keys](EntryRef ref) { keys.emplace_back(_allocator.get_wrapped(ref).value()); }); std::sort(keys.begin(), keys.end()); - EXPECT_EQ(50, keys.size()); - for (uint32_t i = 0; i < 50; ++i) { + EXPECT_EQ(small_population, keys.size()); + for (uint32_t i = 0; i < small_population; ++i) { EXPECT_EQ(i, keys[i]); } } TEST_F(DataStoreShardedHashTest, move_keys_works) { - populate_sample_data(); + populate_sample_data(small_population); std::vector<EntryRef> refs; _hash_map.foreach_key([&refs](EntryRef ref) { refs.emplace_back(ref); }); std::vector<EntryRef> new_refs; MyCompactable my_compactable(_allocator, new_refs); - _hash_map.move_keys(my_compactable, std::vector<bool>(RefT::numBuffers(), true), RefT::offset_bits); + auto filter = make_entry_ref_filter<RefT>(false); + _hash_map.move_keys(my_compactable, filter); std::vector<EntryRef> verify_new_refs; _hash_map.foreach_key([&verify_new_refs](EntryRef ref) { verify_new_refs.emplace_back(ref); }); - EXPECT_EQ(50u, refs.size()); + EXPECT_EQ(small_population, refs.size()); EXPECT_NE(refs, new_refs); EXPECT_EQ(new_refs, verify_new_refs); - for (uint32_t i = 0; i < 50; ++i) { + for (uint32_t i = 0; i < small_population; ++i) { EXPECT_NE(refs[i], new_refs[i]); auto value = _allocator.get_wrapped(refs[i]).value(); auto new_value = _allocator.get_wrapped(refs[i]).value(); @@ -297,29 +419,33 @@ TEST_F(DataStoreShardedHashTest, move_keys_works) TEST_F(DataStoreShardedHashTest, normalize_values_works) { - populate_sample_data(); - for (uint32_t i = 0; i < 50; ++i) { - MyCompare comp(_store, i); - auto result = _hash_map.find(comp, EntryRef()); - ASSERT_NE(result, nullptr); - EXPECT_EQ(i, _allocator.get_wrapped(result->first.load_relaxed()).value()); - result->second.store_relaxed(EntryRef(i + 200)); - } - _hash_map.normalize_values([](EntryRef ref) noexcept { return EntryRef(ref.ref() + 300); }); - for (uint32_t i = 0; i < 50; ++i) { - MyCompare comp(_store, i); - auto result = _hash_map.find(comp, EntryRef()); - ASSERT_NE(result, nullptr); - EXPECT_EQ(i, _allocator.get_wrapped(result->first.load_relaxed()).value()); - ASSERT_EQ(i + 500, result->second.load_relaxed().ref()); - result->second.store_relaxed(EntryRef()); - } + test_normalize_values(false, false); +} + +TEST_F(DataStoreShardedHashTest, normalize_values_all_filter_works) +{ + test_normalize_values(true, false); +} + +TEST_F(DataStoreShardedHashTest, normalize_values_one_filter_works) +{ + test_normalize_values(true, true); +} + +TEST_F(DataStoreShardedHashTest, foreach_value_all_filter_works) +{ + test_foreach_value(false); +} + +TEST_F(DataStoreShardedHashTest, foreach_value_one_filter_works) +{ + test_foreach_value(true); } TEST_F(DataStoreShardedHashTest, compact_worst_shard_works) { - populate_sample_data(); - for (uint32_t i = 10; i < 50; ++i) { + populate_sample_data(small_population); + for (uint32_t i = 10; i < small_population; ++i) { remove(i); } commit(); diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index ccb18f13871..917c91f2dff 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -1,4 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/datastore/compaction_spec.h> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/datastore/unique_store.hpp> #include <vespa/vespalib/datastore/unique_store_remapper.h> #include <vespa/vespalib/datastore/unique_store_string_allocator.hpp> @@ -111,7 +113,9 @@ struct TestBase : public ::testing::Test { store.trimHoldLists(generation); } void compactWorst() { - auto remapper = store.compact_worst(true, true); + CompactionSpec compaction_spec(true, true); + CompactionStrategy compaction_strategy; + auto remapper = store.compact_worst(compaction_spec, compaction_strategy); std::vector<EntryRef> refs; for (const auto &elem : refStore) { refs.push_back(elem.first); diff --git a/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp b/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp index 8d82c10d340..4a8b7eafe6a 100644 --- a/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp +++ b/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/datastore/unique_store.hpp> #include <vespa/vespalib/datastore/unique_store_dictionary.hpp> #include <vespa/vespalib/datastore/sharded_hash_map.h> @@ -137,8 +138,9 @@ TYPED_TEST(UniqueStoreDictionaryTest, compaction_works) this->inc_generation(); auto btree_memory_usage_before = this->dict.get_btree_memory_usage(); auto hash_memory_usage_before = this->dict.get_hash_memory_usage(); + CompactionStrategy compaction_strategy; for (uint32_t i = 0; i < 15; ++i) { - this->dict.compact_worst(true, true); + this->dict.compact_worst(true, true, compaction_strategy); } this->inc_generation(); auto btree_memory_usage_after = this->dict.get_btree_memory_usage(); diff --git a/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp b/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp index e129ef2a389..812d06868fd 100644 --- a/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp +++ b/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp @@ -7,57 +7,93 @@ using namespace vespalib; using namespace vespalib::net::tls; -bool glob_matches(vespalib::stringref pattern, vespalib::stringref string_to_check) { - auto glob = CredentialMatchPattern::create_from_glob(pattern); +bool dns_glob_matches(vespalib::stringref pattern, vespalib::stringref string_to_check) { + auto glob = CredentialMatchPattern::create_from_dns_glob(pattern); return glob->matches(string_to_check); } +bool uri_glob_matches(vespalib::stringref pattern, vespalib::stringref string_to_check) { + auto glob = CredentialMatchPattern::create_from_uri_glob(pattern); + return glob->matches(string_to_check); +} + +void verify_all_glob_types_match(vespalib::stringref pattern, vespalib::stringref string_to_check) { + EXPECT_TRUE(dns_glob_matches(pattern, string_to_check)); + EXPECT_TRUE(uri_glob_matches(pattern, string_to_check)); +} + +void verify_all_glob_types_mismatch(vespalib::stringref pattern, vespalib::stringref string_to_check) { + EXPECT_FALSE(dns_glob_matches(pattern, string_to_check)); + EXPECT_FALSE(uri_glob_matches(pattern, string_to_check)); +} + TEST("glob without wildcards matches entire string") { - EXPECT_TRUE(glob_matches("foo", "foo")); - EXPECT_FALSE(glob_matches("foo", "fooo")); - EXPECT_FALSE(glob_matches("foo", "ffoo")); + verify_all_glob_types_match("foo", "foo"); + verify_all_glob_types_mismatch("foo", "fooo"); + verify_all_glob_types_mismatch("foo", "ffoo"); } TEST("wildcard glob can match prefix") { - EXPECT_TRUE(glob_matches("foo*", "foo")); - EXPECT_TRUE(glob_matches("foo*", "foobar")); - EXPECT_FALSE(glob_matches("foo*", "ffoo")); + verify_all_glob_types_match("foo*", "foo"); + verify_all_glob_types_match("foo*", "foobar"); + verify_all_glob_types_mismatch("foo*", "ffoo"); } TEST("wildcard glob can match suffix") { - EXPECT_TRUE(glob_matches("*foo", "foo")); - EXPECT_TRUE(glob_matches("*foo", "ffoo")); - EXPECT_FALSE(glob_matches("*foo", "fooo")); + verify_all_glob_types_match("*foo", "foo"); + verify_all_glob_types_match("*foo", "ffoo"); + verify_all_glob_types_mismatch("*foo", "fooo"); } TEST("wildcard glob can match substring") { - EXPECT_TRUE(glob_matches("f*o", "fo")); - EXPECT_TRUE(glob_matches("f*o", "foo")); - EXPECT_TRUE(glob_matches("f*o", "ffoo")); - EXPECT_FALSE(glob_matches("f*o", "boo")); + verify_all_glob_types_match("f*o", "fo"); + verify_all_glob_types_match("f*o", "foo"); + verify_all_glob_types_match("f*o", "ffoo"); + verify_all_glob_types_mismatch("f*o", "boo"); } -TEST("wildcard glob does not cross multiple dot delimiter boundaries") { - EXPECT_TRUE(glob_matches("*.bar.baz", "foo.bar.baz")); - EXPECT_TRUE(glob_matches("*.bar.baz", ".bar.baz")); - EXPECT_FALSE(glob_matches("*.bar.baz", "zoid.foo.bar.baz")); - EXPECT_TRUE(glob_matches("foo.*.baz", "foo.bar.baz")); - EXPECT_FALSE(glob_matches("foo.*.baz", "foo.bar.zoid.baz")); +TEST("single char DNS glob matches single character") { + EXPECT_TRUE(dns_glob_matches("f?o", "foo")); + EXPECT_FALSE(dns_glob_matches("f?o", "fooo")); + EXPECT_FALSE(dns_glob_matches("f?o", "ffoo")); } -TEST("single char glob matches non dot characters") { - EXPECT_TRUE(glob_matches("f?o", "foo")); - EXPECT_FALSE(glob_matches("f?o", "fooo")); - EXPECT_FALSE(glob_matches("f?o", "ffoo")); - EXPECT_FALSE(glob_matches("f?o", "f.o")); +// Due to URIs being able to contain '?' characters as a query separator, don't use it for wildcarding. +TEST("URI glob matching treats question mark character as literal match") { + EXPECT_TRUE(uri_glob_matches("f?o", "f?o")); + EXPECT_FALSE(uri_glob_matches("f?o", "foo")); + EXPECT_FALSE(uri_glob_matches("f?o", "f?oo")); +} + +TEST("wildcard DNS glob does not cross multiple dot delimiter boundaries") { + EXPECT_TRUE(dns_glob_matches("*.bar.baz", "foo.bar.baz")); + EXPECT_TRUE(dns_glob_matches("*.bar.baz", ".bar.baz")); + EXPECT_FALSE(dns_glob_matches("*.bar.baz", "zoid.foo.bar.baz")); + EXPECT_TRUE(dns_glob_matches("foo.*.baz", "foo.bar.baz")); + EXPECT_FALSE(dns_glob_matches("foo.*.baz", "foo.bar.zoid.baz")); +} + +TEST("wildcard URI glob does not cross multiple fwd slash delimiter boundaries") { + EXPECT_TRUE(uri_glob_matches("*/bar/baz", "foo/bar/baz")); + EXPECT_TRUE(uri_glob_matches("*/bar/baz", "/bar/baz")); + EXPECT_FALSE(uri_glob_matches("*/bar/baz", "bar/baz")); + EXPECT_FALSE(uri_glob_matches("*/bar/baz", "/bar/baz/")); + EXPECT_FALSE(uri_glob_matches("*/bar/baz", "zoid/foo/bar/baz")); + EXPECT_TRUE(uri_glob_matches("foo/*/baz", "foo/bar/baz")); + EXPECT_FALSE(uri_glob_matches("foo/*/baz", "foo/bar/zoid/baz")); + EXPECT_TRUE(uri_glob_matches("foo/*/baz", "foo/bar.zoid/baz")); // No special handling of dots +} + +TEST("single char DNS glob matches non dot characters only") { + EXPECT_FALSE(dns_glob_matches("f?o", "f.o")); } TEST("special basic regex characters are escaped") { - EXPECT_TRUE(glob_matches("$[.\\^", "$[.\\^")); + verify_all_glob_types_match("$[.\\^", "$[.\\^"); } TEST("special extended regex characters are ignored") { - EXPECT_TRUE(glob_matches("{)(+|]}", "{)(+|]}")); + verify_all_glob_types_match("{)(+|]}", "{)(+|]}"); } // TODO CN + SANs @@ -116,7 +152,7 @@ TEST("DNS SAN requirement without glob pattern is matched as exact string") { EXPECT_FALSE(verify(authorized, creds_with_dns_sans({{"hello.world.bar"}}))); } -TEST("DNS SAN requirement can include glob wildcards") { +TEST("DNS SAN requirement can include glob wildcards, delimited by dot character") { auto authorized = authorized_peers({policy_with({required_san_dns("*.w?rld")})}); EXPECT_TRUE(verify(authorized, creds_with_dns_sans({{"hello.world"}}))); EXPECT_TRUE(verify(authorized, creds_with_dns_sans({{"greetings.w0rld"}}))); @@ -124,8 +160,8 @@ TEST("DNS SAN requirement can include glob wildcards") { EXPECT_FALSE(verify(authorized, creds_with_dns_sans({{"world"}}))); } -// FIXME make this RFC 2459-compliant with subdomain matching, case insensitity for host etc -TEST("URI SAN requirement is matched as exact string in cheeky, pragmatic violation of RFC 2459") { +// TODO consider making this RFC 2459-compliant with case insensitivity for scheme and host +TEST("URI SAN requirement without glob pattern is matched as exact string") { auto authorized = authorized_peers({policy_with({required_san_uri("foo://bar.baz/zoid")})}); EXPECT_TRUE(verify(authorized, creds_with_uri_sans({{"foo://bar.baz/zoid"}}))); EXPECT_FALSE(verify(authorized, creds_with_uri_sans({{"foo://bar.baz/zoi"}}))); @@ -136,6 +172,25 @@ TEST("URI SAN requirement is matched as exact string in cheeky, pragmatic violat EXPECT_FALSE(verify(authorized, creds_with_uri_sans({{"foo://BAR.baz/zoid"}}))); } +// TODO consider making this RFC 2459-compliant with case insensitivity for scheme and host +TEST("URI SAN requirement can include glob wildcards, delimited by fwd slash character") { + auto authorized = authorized_peers({policy_with({required_san_uri("myscheme://my/*/uri")})}); + EXPECT_TRUE(verify(authorized, creds_with_uri_sans({{"myscheme://my/cool/uri"}}))); + EXPECT_TRUE(verify(authorized, creds_with_uri_sans({{"myscheme://my/really.cool/uri"}}))); // Not delimited by dots + EXPECT_FALSE(verify(authorized, creds_with_uri_sans({{"theirscheme://my/cool/uri"}}))); + EXPECT_FALSE(verify(authorized, creds_with_uri_sans({{"myscheme://their/cool/uri"}}))); + EXPECT_FALSE(verify(authorized, creds_with_uri_sans({{"myscheme://my/cool/uris"}}))); + EXPECT_FALSE(verify(authorized, creds_with_uri_sans({{"myscheme://my/swag/uri/"}}))); + EXPECT_FALSE(verify(authorized, creds_with_uri_sans({{"myscheme://my/uri"}}))); +} + +TEST("URI SAN requirement can include query part even though it's rather silly to do so") { + auto authorized = authorized_peers({policy_with({required_san_uri("myscheme://my/fancy/*?magic")})}); + EXPECT_TRUE(verify(authorized, creds_with_uri_sans({{"myscheme://my/fancy/uri?magic"}}))); + EXPECT_TRUE(verify(authorized, creds_with_uri_sans({{"myscheme://my/fancy/?magic"}}))); + EXPECT_FALSE(verify(authorized, creds_with_uri_sans({{"myscheme://my/fancy/urimagic"}}))); +} + TEST("multi-SAN policy requires all SANs to be present in certificate") { auto authorized = authorized_peers({policy_with({required_san_dns("hello.world"), required_san_dns("foo.bar"), @@ -157,6 +212,13 @@ TEST("wildcard DNS SAN in certificate is not treated as a wildcard match by poli EXPECT_FALSE(verify(authorized, creds_with_dns_sans({{"*.world"}}))); } +TEST("wildcard URI SAN in certificate is not treated as a wildcard match by policy") { + auto authorized = authorized_peers({policy_with({required_san_uri("hello://world")})}); + EXPECT_FALSE(verify(authorized, creds_with_uri_sans({{"hello://*"}}))); +} + +// TODO this is just by coincidence since we match '*' as any other character, not because we interpret +// the wildcard in the SAN as anything special during matching. Consider if we need/want to handle explicitly. TEST("wildcard DNS SAN in certificate is still matched by wildcard policy SAN") { auto authorized = authorized_peers({policy_with({required_san_dns("*.world")})}); EXPECT_TRUE(verify(authorized, creds_with_dns_sans({{"*.world"}}))); diff --git a/vespalib/src/tests/thread/thread_test.cpp b/vespalib/src/tests/thread/thread_test.cpp index 43951b4b734..ee4f97c34cc 100644 --- a/vespalib/src/tests/thread/thread_test.cpp +++ b/vespalib/src/tests/thread/thread_test.cpp @@ -6,6 +6,8 @@ using namespace vespalib; +VESPA_THREAD_STACK_TAG(test_agent_thread); + struct Agent : public Runnable { bool started; int loopCnt; @@ -22,7 +24,7 @@ struct Agent : public Runnable { TEST("thread never started") { Agent agent; { - Thread thread(agent); + Thread thread(agent, test_agent_thread); } EXPECT_TRUE(!agent.started); EXPECT_EQUAL(0, agent.loopCnt); @@ -31,7 +33,7 @@ TEST("thread never started") { TEST("normal operation") { Agent agent; { - Thread thread(agent); + Thread thread(agent, test_agent_thread); thread.start(); std::this_thread::sleep_for(20ms); thread.stop().join(); @@ -43,7 +45,7 @@ TEST("normal operation") { TEST("stop before start") { Agent agent; { - Thread thread(agent); + Thread thread(agent, test_agent_thread); thread.stop(); thread.start(); thread.join(); diff --git a/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp b/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp index 9ad0e95667b..cf84ab03a25 100644 --- a/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp +++ b/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp @@ -19,19 +19,14 @@ assertUsage(const MemoryUsage & exp, const MemoryUsage & act) TEST("test generation holder") { - typedef std::unique_ptr<int32_t> IntPtr; GenerationHolder gh; - gh.hold(GenerationHeldBase::UP(new RcuVectorHeld<int32_t>(sizeof(int32_t), - IntPtr(new int32_t(0))))); + gh.hold(std::make_unique<RcuVectorHeld<int32_t>>(sizeof(int32_t), 0)); gh.transferHoldLists(0); - gh.hold(GenerationHeldBase::UP(new RcuVectorHeld<int32_t>(sizeof(int32_t), - IntPtr(new int32_t(1))))); + gh.hold(std::make_unique<RcuVectorHeld<int32_t>>(sizeof(int32_t), 1)); gh.transferHoldLists(1); - gh.hold(GenerationHeldBase::UP(new RcuVectorHeld<int32_t>(sizeof(int32_t), - IntPtr(new int32_t(2))))); + gh.hold(std::make_unique<RcuVectorHeld<int32_t>>(sizeof(int32_t), 2)); gh.transferHoldLists(2); - gh.hold(GenerationHeldBase::UP(new RcuVectorHeld<int32_t>(sizeof(int32_t), - IntPtr(new int32_t(4))))); + gh.hold(std::make_unique<RcuVectorHeld<int32_t>>(sizeof(int32_t), 4)); gh.transferHoldLists(4); EXPECT_EQUAL(4u * sizeof(int32_t), gh.getHeldBytes()); gh.trimHoldLists(0); @@ -40,8 +35,7 @@ TEST("test generation holder") EXPECT_EQUAL(3u * sizeof(int32_t), gh.getHeldBytes()); gh.trimHoldLists(2); EXPECT_EQUAL(2u * sizeof(int32_t), gh.getHeldBytes()); - gh.hold(GenerationHeldBase::UP(new RcuVectorHeld<int32_t>(sizeof(int32_t), - IntPtr(new int32_t(6))))); + gh.hold(std::make_unique<RcuVectorHeld<int32_t>>(sizeof(int32_t), 6)); gh.transferHoldLists(6); EXPECT_EQUAL(3u * sizeof(int32_t), gh.getHeldBytes()); gh.trimHoldLists(6); diff --git a/vespalib/src/vespa/vespalib/btree/btree.h b/vespalib/src/vespa/vespalib/btree/btree.h index 2b03e70fbdf..f87d5751743 100644 --- a/vespalib/src/vespa/vespalib/btree/btree.h +++ b/vespalib/src/vespa/vespalib/btree/btree.h @@ -6,6 +6,8 @@ #include "noaggrcalc.h" #include <vespa/vespalib/util/generationhandler.h> +namespace vespalib::datastore { class CompactionStrategy; } + namespace vespalib::btree { /** @@ -149,7 +151,7 @@ public: _tree.thaw(itr); } - void compact_worst(); + void compact_worst(const datastore::CompactionStrategy& compaction_strategy); template <typename FunctionType> void diff --git a/vespalib/src/vespa/vespalib/btree/btree.hpp b/vespalib/src/vespa/vespalib/btree/btree.hpp index c4a588bc63e..473d1f4735e 100644 --- a/vespalib/src/vespa/vespalib/btree/btree.hpp +++ b/vespalib/src/vespa/vespalib/btree/btree.hpp @@ -26,9 +26,9 @@ BTree<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>::~BTree() template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT, class AggrCalcT> void -BTree<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>::compact_worst() +BTree<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>::compact_worst(const datastore::CompactionStrategy& compaction_strategy) { - auto to_hold = _alloc.start_compact_worst(); + auto to_hold = _alloc.start_compact_worst(compaction_strategy); _tree.move_nodes(_alloc); _alloc.finishCompact(to_hold); } diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.h b/vespalib/src/vespa/vespalib/btree/btreeiterator.h index 325ce0e0e47..30123b1946e 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.h +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.h @@ -113,6 +113,9 @@ public: return _node->getData(_idx); } + // Only use during compaction when changing reference to moved value + DataType &getWData() { return getWNode()->getWData(_idx); } + bool valid() const { @@ -881,6 +884,9 @@ public: _leaf.getWNode()->writeData(_leaf.getIdx(), data); } + // Only use during compaction when changing reference to moved value + DataType &getWData() { return _leaf.getWData(); } + /** * Set a new key for the current iterator position. * The new key must have the same semantic meaning as the old key. diff --git a/vespalib/src/vespa/vespalib/btree/btreenode.h b/vespalib/src/vespa/vespalib/btree/btreenode.h index d8752d77f0b..468f17fcd1a 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenode.h +++ b/vespalib/src/vespa/vespalib/btree/btreenode.h @@ -99,6 +99,8 @@ public: } const DataT &getData(uint32_t idx) const { return _data[idx]; } + // Only use during compaction when changing reference to moved value + DataT &getWData(uint32_t idx) { return _data[idx]; } void setData(uint32_t idx, const DataT &data) { _data[idx] = data; } static bool hasData() { return true; } }; @@ -120,6 +122,9 @@ public: return BTreeNoLeafData::_instance; } + // Only use during compaction when changing reference to moved value + BTreeNoLeafData &getWData(uint32_t) const { return BTreeNoLeafData::_instance; } + void setData(uint32_t idx, const BTreeNoLeafData &data) { (void) idx; (void) data; diff --git a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h index 93615ddef82..27e73b3a2b6 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h @@ -29,6 +29,7 @@ public: using BTreeRootBaseType = BTreeRootBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>; using generation_t = vespalib::GenerationHandler::generation_t; using NodeStore = BTreeNodeStore<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>; + using CompactionStrategy = datastore::CompactionStrategy; using EntryRef = datastore::EntryRef; using DataStoreBase = datastore::DataStoreBase; @@ -165,7 +166,7 @@ public: bool getCompacting(EntryRef ref) const { return _nodeStore.getCompacting(ref); } std::vector<uint32_t> startCompact() { return _nodeStore.startCompact(); } - std::vector<uint32_t> start_compact_worst() { return _nodeStore.start_compact_worst(); } + std::vector<uint32_t> start_compact_worst(const CompactionStrategy& compaction_strategy) { return _nodeStore.start_compact_worst(compaction_strategy); } void finishCompact(const std::vector<uint32_t> &toHold) { return _nodeStore.finishCompact(toHold); diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.h b/vespalib/src/vespa/vespalib/btree/btreenodestore.h index 70a9ba6c73c..444bf641899 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.h @@ -56,6 +56,7 @@ public: typedef typename LeafNodeType::RefPair LeafNodeTypeRefPair; typedef vespalib::GenerationHandler::generation_t generation_t; using EntryRef = datastore::EntryRef; + using CompactionStrategy = datastore::CompactionStrategy; enum NodeTypes { @@ -159,7 +160,7 @@ public: std::vector<uint32_t> startCompact(); - std::vector<uint32_t> start_compact_worst(); + std::vector<uint32_t> start_compact_worst(const CompactionStrategy& compaction_strategy); void finishCompact(const std::vector<uint32_t> &toHold); diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp b/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp index ff4fa59cd74..91953507eb0 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp @@ -3,6 +3,7 @@ #pragma once #include "btreenodestore.h" +#include <vespa/vespalib/datastore/compaction_spec.h> #include <vespa/vespalib/datastore/datastore.hpp> namespace vespalib::btree { @@ -71,9 +72,9 @@ template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS> std::vector<uint32_t> BTreeNodeStore<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>:: -start_compact_worst() +start_compact_worst(const CompactionStrategy &compaction_strategy) { - return _store.startCompactWorstBuffers(true, false); + return _store.startCompactWorstBuffers(datastore::CompactionSpec(true, false), compaction_strategy); } template <typename KeyT, typename DataT, typename AggrT, diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.h b/vespalib/src/vespa/vespalib/btree/btreestore.h index 82913987e44..a79259c6e57 100644 --- a/vespalib/src/vespa/vespalib/btree/btreestore.h +++ b/vespalib/src/vespa/vespalib/btree/btreestore.h @@ -49,6 +49,8 @@ public: TraitsT::INTERNAL_SLOTS, TraitsT::LEAF_SLOTS, AggrCalcT> Builder; + using CompactionSpec = datastore::CompactionSpec; + using CompactionStrategy = datastore::CompactionStrategy; using EntryRef = datastore::EntryRef; template <typename EntryType> using BufferType = datastore::BufferType<EntryType>; @@ -298,6 +300,9 @@ public: bool isSmallArray(const EntryRef ref) const; + static bool isBTree(uint32_t typeId) { return typeId == BUFFERTYPE_BTREE; } + bool isBTree(RefType ref) const { return isBTree(getTypeId(ref)); } + /** * Returns the cluster size for the type id. * Cluster size == 0 means we have a tree for the given reference. @@ -389,12 +394,12 @@ public: void foreach_frozen(EntryRef ref, FunctionType func) const; - std::vector<uint32_t> start_compact_worst_btree_nodes(); + std::vector<uint32_t> start_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy); void finish_compact_worst_btree_nodes(const std::vector<uint32_t>& to_hold); - void move_btree_nodes(EntryRef ref); + void move_btree_nodes(const std::vector<EntryRef>& refs); - std::vector<uint32_t> start_compact_worst_buffers(); - EntryRef move(EntryRef ref); + std::vector<uint32_t> start_compact_worst_buffers(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy); + void move(std::vector<EntryRef>& refs); private: static constexpr size_t MIN_BUFFER_ARRAYS = 128u; diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.hpp b/vespalib/src/vespa/vespalib/btree/btreestore.hpp index 15c546a0368..c0985ff8f94 100644 --- a/vespalib/src/vespa/vespalib/btree/btreestore.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreestore.hpp @@ -5,6 +5,7 @@ #include "btreestore.h" #include "btreebuilder.h" #include "btreebuilder.hpp" +#include <vespa/vespalib/datastore/compaction_spec.h> #include <vespa/vespalib/datastore/datastore.hpp> #include <vespa/vespalib/util/optimized.h> @@ -972,10 +973,10 @@ template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT, typename AggrCalcT> std::vector<uint32_t> BTreeStore<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>:: -start_compact_worst_btree_nodes() +start_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy) { _builder.clear(); - return _allocator.start_compact_worst(); + return _allocator.start_compact_worst(compaction_strategy); } template <typename KeyT, typename DataT, typename AggrT, typename CompareT, @@ -991,15 +992,15 @@ template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT, typename AggrCalcT> void BTreeStore<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>:: -move_btree_nodes(EntryRef ref) +move_btree_nodes(const std::vector<EntryRef>& refs) { - if (ref.valid()) { + for (auto& ref : refs) { RefType iRef(ref); - uint32_t clusterSize = getClusterSize(iRef); - if (clusterSize == 0) { - BTreeType *tree = getWTreeEntry(iRef); - tree->move_nodes(_allocator); - } + assert(iRef.valid()); + uint32_t typeId = getTypeId(iRef); + assert(isBTree(typeId)); + BTreeType *tree = getWTreeEntry(iRef); + tree->move_nodes(_allocator); } } @@ -1007,31 +1008,33 @@ template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT, typename AggrCalcT> std::vector<uint32_t> BTreeStore<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>:: -start_compact_worst_buffers() +start_compact_worst_buffers(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { freeze(); - return _store.startCompactWorstBuffers(true, false); + return _store.startCompactWorstBuffers(compaction_spec, compaction_strategy); } template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT, typename AggrCalcT> -typename BTreeStore<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>::EntryRef +void BTreeStore<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>:: -move(EntryRef ref) +move(std::vector<EntryRef> &refs) { - if (!ref.valid() || !_store.getCompacting(ref)) { - return ref; - } - RefType iRef(ref); - uint32_t clusterSize = getClusterSize(iRef); - if (clusterSize == 0) { - BTreeType *tree = getWTreeEntry(iRef); - auto ref_and_ptr = allocBTreeCopy(*tree); - tree->prepare_hold(); - return ref_and_ptr.ref; + for (auto& ref : refs) { + RefType iRef(ref); + assert(iRef.valid()); + assert(_store.getCompacting(iRef)); + uint32_t clusterSize = getClusterSize(iRef); + if (clusterSize == 0) { + BTreeType *tree = getWTreeEntry(iRef); + auto ref_and_ptr = allocBTreeCopy(*tree); + tree->prepare_hold(); + ref = ref_and_ptr.ref; + } else { + const KeyDataType *shortArray = getKeyDataEntry(iRef, clusterSize); + ref = allocKeyDataCopy(shortArray, clusterSize).ref; + } } - const KeyDataType *shortArray = getKeyDataEntry(iRef, clusterSize); - return allocKeyDataCopy(shortArray, clusterSize).ref; } } diff --git a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt index 6c6f5258555..c36077e4dd0 100644 --- a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt @@ -5,9 +5,11 @@ vespa_add_library(vespalib_vespalib_datastore OBJECT array_store_config.cpp buffer_type.cpp bufferstate.cpp + compaction_strategy.cpp datastore.cpp datastorebase.cpp entryref.cpp + entry_ref_filter.cpp fixed_size_hash_map.cpp sharded_hash_map.cpp unique_store.cpp diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index 3ba0caae5b9..d9b62c310b5 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -96,7 +96,7 @@ public: } void remove(EntryRef ref); - ICompactionContext::UP compactWorst(bool compactMemory, bool compactAddressSpace); + ICompactionContext::UP compactWorst(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy); vespalib::MemoryUsage getMemoryUsage() const { return _store.getMemoryUsage(); } /** diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp index 5600c64eb3d..bbbd52c354d 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp @@ -3,6 +3,8 @@ #pragma once #include "array_store.h" +#include "compaction_spec.h" +#include "entry_ref_filter.h" #include "datastore.hpp" #include <atomic> #include <algorithm> @@ -127,47 +129,38 @@ private: DataStoreBase &_dataStore; ArrayStoreType &_store; std::vector<uint32_t> _bufferIdsToCompact; + EntryRefFilter _filter; - bool compactingBuffer(uint32_t bufferId) { - return std::find(_bufferIdsToCompact.begin(), _bufferIdsToCompact.end(), - bufferId) != _bufferIdsToCompact.end(); - } public: CompactionContext(DataStoreBase &dataStore, ArrayStoreType &store, std::vector<uint32_t> bufferIdsToCompact) : _dataStore(dataStore), _store(store), - _bufferIdsToCompact(std::move(bufferIdsToCompact)) - {} + _bufferIdsToCompact(std::move(bufferIdsToCompact)), + _filter(RefT::numBuffers(), RefT::offset_bits) + { + _filter.add_buffers(_bufferIdsToCompact); + } ~CompactionContext() override { _dataStore.finishCompact(_bufferIdsToCompact); } void compact(vespalib::ArrayRef<EntryRef> refs) override { - if (!_bufferIdsToCompact.empty()) { - for (auto &ref : refs) { - if (ref.valid()) { - RefT internalRef(ref); - if (compactingBuffer(internalRef.bufferId())) { - EntryRef newRef = _store.add(_store.get(ref)); - std::atomic_thread_fence(std::memory_order_release); - ref = newRef; - } - } + for (auto &ref : refs) { + if (ref.valid() && _filter.has(ref)) { + EntryRef newRef = _store.add(_store.get(ref)); + std::atomic_thread_fence(std::memory_order_release); + ref = newRef; } } } void compact(vespalib::ArrayRef<AtomicEntryRef> refs) override { - if (!_bufferIdsToCompact.empty()) { - for (auto &ref : refs) { - if (ref.load_relaxed().valid()) { - RefT internalRef(ref.load_relaxed()); - if (compactingBuffer(internalRef.bufferId())) { - EntryRef newRef = _store.add(_store.get(ref.load_relaxed())); - std::atomic_thread_fence(std::memory_order_release); - ref.store_release(newRef); - } - } + for (auto &atomic_entry_ref : refs) { + auto ref = atomic_entry_ref.load_relaxed(); + if (ref.valid() && _filter.has(ref)) { + EntryRef newRef = _store.add(_store.get(ref)); + std::atomic_thread_fence(std::memory_order_release); + atomic_entry_ref.store_release(newRef); } } } @@ -177,9 +170,9 @@ public: template <typename EntryT, typename RefT> ICompactionContext::UP -ArrayStore<EntryT, RefT>::compactWorst(bool compactMemory, bool compactAddressSpace) +ArrayStore<EntryT, RefT>::compactWorst(CompactionSpec compaction_spec, const CompactionStrategy &compaction_strategy) { - std::vector<uint32_t> bufferIdsToCompact = _store.startCompactWorstBuffers(compactMemory, compactAddressSpace); + std::vector<uint32_t> bufferIdsToCompact = _store.startCompactWorstBuffers(compaction_spec, compaction_strategy); return std::make_unique<arraystore::CompactionContext<EntryT, RefT>> (_store, *this, std::move(bufferIdsToCompact)); } diff --git a/vespalib/src/vespa/vespalib/datastore/compaction_spec.h b/vespalib/src/vespa/vespalib/datastore/compaction_spec.h new file mode 100644 index 00000000000..c554f3229dd --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/compaction_spec.h @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib::datastore { + +/* + * Class describing how to compact a compactable data structure. + * + * memory - to reduce amount of "dead" memory + * address_space - to avoid running out of free buffers in data store + * (i.e. move data from small buffers to larger buffers) + */ +class CompactionSpec +{ + bool _compact_memory; + bool _compact_address_space; +public: + CompactionSpec() + : _compact_memory(false), + _compact_address_space(false) + { + } + CompactionSpec(bool compact_memory_, bool compact_address_space_) noexcept + : _compact_memory(compact_memory_), + _compact_address_space(compact_address_space_) + { + } + bool compact() const noexcept { return _compact_memory || _compact_address_space; } + bool compact_memory() const noexcept { return _compact_memory; } + bool compact_address_space() const noexcept { return _compact_address_space; } +}; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/compaction_strategy.cpp b/vespalib/src/vespa/vespalib/datastore/compaction_strategy.cpp new file mode 100644 index 00000000000..2dbd501f78e --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/compaction_strategy.cpp @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "compaction_strategy.h" +#include "compaction_spec.h" +#include <vespa/vespalib/util/memoryusage.h> +#include <vespa/vespalib/util/address_space.h> +#include <iostream> + +namespace vespalib::datastore { + +bool +CompactionStrategy::should_compact_memory(const MemoryUsage& memory_usage) const +{ + return should_compact_memory(memory_usage.usedBytes(), memory_usage.deadBytes()); +} + +bool +CompactionStrategy::should_compact_address_space(const AddressSpace& address_space) const +{ + return should_compact_address_space(address_space.used(), address_space.dead()); +} + +CompactionSpec +CompactionStrategy::should_compact(const MemoryUsage& memory_usage, const AddressSpace& address_space) const +{ + return CompactionSpec(should_compact_memory(memory_usage), should_compact_address_space(address_space)); +} + +std::ostream& operator<<(std::ostream& os, const CompactionStrategy& compaction_strategy) +{ + os << "{maxDeadBytesRatio=" << compaction_strategy.getMaxDeadBytesRatio() << + ", maxDeadAddressSpaceRatio=" << compaction_strategy.getMaxDeadAddressSpaceRatio() << + "}"; + return os; +} + +} diff --git a/vespalib/src/vespa/vespalib/datastore/compaction_strategy.h b/vespalib/src/vespa/vespalib/datastore/compaction_strategy.h new file mode 100644 index 00000000000..9ca4a64a55b --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/compaction_strategy.h @@ -0,0 +1,75 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <iosfwd> +#include <cstdint> + +namespace vespalib { + +class AddressSpace; +class MemoryUsage; + +} + +namespace vespalib::datastore { + +class CompactionSpec; + +/* + * Class describing compaction strategy for a compactable data structure. + */ +class CompactionStrategy +{ +public: + static constexpr size_t DEAD_BYTES_SLACK = 0x10000u; + static constexpr size_t DEAD_ADDRESS_SPACE_SLACK = 0x10000u; +private: + double _maxDeadBytesRatio; // Max ratio of dead bytes before compaction + double _maxDeadAddressSpaceRatio; // Max ratio of dead address space before compaction + uint32_t _max_buffers; // Max number of buffers to compact for each reason (memory usage, address space usage) + bool should_compact_memory(size_t used_bytes, size_t dead_bytes) const { + return ((dead_bytes >= DEAD_BYTES_SLACK) && + (dead_bytes > used_bytes * getMaxDeadBytesRatio())); + } + bool should_compact_address_space(size_t used_address_space, size_t dead_address_space) const { + return ((dead_address_space >= DEAD_ADDRESS_SPACE_SLACK) && + (dead_address_space > used_address_space * getMaxDeadAddressSpaceRatio())); + } +public: + CompactionStrategy() noexcept + : _maxDeadBytesRatio(0.05), + _maxDeadAddressSpaceRatio(0.2), + _max_buffers(1) + { + } + CompactionStrategy(double maxDeadBytesRatio, double maxDeadAddressSpaceRatio) noexcept + : _maxDeadBytesRatio(maxDeadBytesRatio), + _maxDeadAddressSpaceRatio(maxDeadAddressSpaceRatio), + _max_buffers(1) + { + } + CompactionStrategy(double maxDeadBytesRatio, double maxDeadAddressSpaceRatio, uint32_t max_buffers) noexcept + : _maxDeadBytesRatio(maxDeadBytesRatio), + _maxDeadAddressSpaceRatio(maxDeadAddressSpaceRatio), + _max_buffers(max_buffers) + { + } + double getMaxDeadBytesRatio() const { return _maxDeadBytesRatio; } + double getMaxDeadAddressSpaceRatio() const { return _maxDeadAddressSpaceRatio; } + uint32_t get_max_buffers() const noexcept { return _max_buffers; } + bool operator==(const CompactionStrategy & rhs) const { + return (_maxDeadBytesRatio == rhs._maxDeadBytesRatio) && + (_maxDeadAddressSpaceRatio == rhs._maxDeadAddressSpaceRatio) && + (_max_buffers == rhs._max_buffers); + } + bool operator!=(const CompactionStrategy & rhs) const { return !(operator==(rhs)); } + + bool should_compact_memory(const MemoryUsage& memory_usage) const; + bool should_compact_address_space(const AddressSpace& address_space) const; + CompactionSpec should_compact(const MemoryUsage& memory_usage, const AddressSpace& address_space) const; +}; + +std::ostream& operator<<(std::ostream& os, const CompactionStrategy& compaction_strategy); + +} diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp index b5cab50bc33..059171e1f02 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "datastore.h" +#include "compaction_spec.h" #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <limits> @@ -526,8 +527,9 @@ DataStoreBase::markCompacting(uint32_t bufferId) } std::vector<uint32_t> -DataStoreBase::startCompactWorstBuffers(bool compactMemory, bool compactAddressSpace) +DataStoreBase::startCompactWorstBuffers(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { + (void) compaction_strategy; constexpr uint32_t noBufferId = std::numeric_limits<uint32_t>::max(); uint32_t worstMemoryBufferId = noBufferId; uint32_t worstAddressSpaceBufferId = noBufferId; @@ -540,11 +542,11 @@ DataStoreBase::startCompactWorstBuffers(bool compactMemory, bool compactAddressS uint32_t arraySize = typeHandler->getArraySize(); uint32_t reservedElements = typeHandler->getReservedElements(bufferId); size_t deadElems = state.getDeadElems() - reservedElements; - if (compactMemory && deadElems > worstDeadElems) { + if (compaction_spec.compact_memory() && deadElems > worstDeadElems) { worstMemoryBufferId = bufferId; worstDeadElems = deadElems; } - if (compactAddressSpace) { + if (compaction_spec.compact_address_space()) { size_t deadArrays = deadElems / arraySize; if (deadArrays > worstDeadArrays) { worstAddressSpaceBufferId = bufferId; diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.h b/vespalib/src/vespa/vespalib/datastore/datastorebase.h index 6903ae12c9c..e98d9531806 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.h +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.h @@ -12,6 +12,9 @@ namespace vespalib::datastore { +class CompactionSpec; +class CompactionStrategy; + /** * Abstract class used to store data of potential different types in underlying memory buffers. * @@ -368,7 +371,7 @@ public: } uint32_t startCompactWorstBuffer(uint32_t typeId); - std::vector<uint32_t> startCompactWorstBuffers(bool compactMemory, bool compactAddressSpace); + std::vector<uint32_t> startCompactWorstBuffers(CompactionSpec compaction_spec, const CompactionStrategy &compaction_strategy); uint64_t get_compaction_count() const { return _compaction_count.load(std::memory_order_relaxed); } void inc_compaction_count() const { ++_compaction_count; } bool has_held_buffers() const noexcept { return _hold_buffer_count != 0u; } diff --git a/vespalib/src/vespa/vespalib/datastore/entry_ref_filter.cpp b/vespalib/src/vespa/vespalib/datastore/entry_ref_filter.cpp new file mode 100644 index 00000000000..87c3c87636c --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/entry_ref_filter.cpp @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "entry_ref_filter.h" + +namespace vespalib::datastore { + +EntryRefFilter::EntryRefFilter(std::vector<bool> filter, uint32_t offset_bits) + : _filter(std::move(filter)), + _offset_bits(offset_bits) +{ +} + +EntryRefFilter::EntryRefFilter(uint32_t num_buffers, uint32_t offset_bits) + : _filter(num_buffers), + _offset_bits(offset_bits) +{ +} + +EntryRefFilter::~EntryRefFilter() = default; + +EntryRefFilter +EntryRefFilter::create_all_filter(uint32_t num_buffers, uint32_t offset_bits) +{ + std::vector<bool> filter(num_buffers, true); + return EntryRefFilter(std::move(filter), offset_bits); +} + +} diff --git a/vespalib/src/vespa/vespalib/datastore/entry_ref_filter.h b/vespalib/src/vespa/vespalib/datastore/entry_ref_filter.h new file mode 100644 index 00000000000..c06d843fbd0 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/entry_ref_filter.h @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "entryref.h" +#include <vector> + +namespace vespalib::datastore { + +/* + * Class to filter entry refs based on which buffer the entry is referencing. + * + * Buffers being allowed have corresponding bit in _filter set. + */ +class EntryRefFilter { + std::vector<bool> _filter; + uint32_t _offset_bits; + EntryRefFilter(std::vector<bool> filter, uint32_t offset_bits); +public: + EntryRefFilter(uint32_t num_buffers, uint32_t offset_bits); + ~EntryRefFilter(); + bool has(EntryRef ref) const { + uint32_t buffer_id = ref.buffer_id(_offset_bits); + return _filter[buffer_id]; + } + void add_buffer(uint32_t buffer_id) { _filter[buffer_id] = true; } + void add_buffers(const std::vector<uint32_t>& ids) { + for (auto buffer_id : ids) { + _filter[buffer_id] = true; + } + } + static EntryRefFilter create_all_filter(uint32_t num_buffers, uint32_t offset_bits); +}; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp index db9fee8ea70..6f001ce3c94 100644 --- a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp +++ b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp @@ -2,6 +2,7 @@ #include "fixed_size_hash_map.h" #include "entry_comparator.h" +#include "entry_ref_filter.h" #include "i_compactable.h" #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/util/memoryusage.h> @@ -182,7 +183,7 @@ FixedSizeHashMap::foreach_key(const std::function<void(EntryRef)>& callback) con } void -FixedSizeHashMap::move_keys(ICompactable& compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits) +FixedSizeHashMap::move_keys(ICompactable& compactable, const EntryRefFilter &compacting_buffers) { for (auto& chain_head : _chain_heads) { uint32_t node_idx = chain_head.load_relaxed(); @@ -190,8 +191,7 @@ FixedSizeHashMap::move_keys(ICompactable& compactable, const std::vector<bool>& auto& node = _nodes[node_idx]; EntryRef old_ref = node.get_kv().first.load_relaxed(); assert(old_ref.valid()); - uint32_t buffer_id = old_ref.buffer_id(entry_ref_offset_bits); - if (compacting_buffers[buffer_id]) { + if (compacting_buffers.has(old_ref)) { EntryRef new_ref = compactable.move(old_ref); node.get_kv().first.store_release(new_ref); } @@ -220,4 +220,104 @@ FixedSizeHashMap::normalize_values(const std::function<EntryRef(EntryRef)>& norm return changed; } +namespace { + +class ChangeWriter { + std::vector<AtomicEntryRef*> _atomic_refs; +public: + ChangeWriter(uint32_t capacity); + ~ChangeWriter(); + bool write(const std::vector<EntryRef> &refs); + void emplace_back(AtomicEntryRef &atomic_ref) { _atomic_refs.emplace_back(&atomic_ref); } +}; + +ChangeWriter::ChangeWriter(uint32_t capacity) + : _atomic_refs() +{ + _atomic_refs.reserve(capacity); +} + +ChangeWriter::~ChangeWriter() = default; + +bool +ChangeWriter::write(const std::vector<EntryRef> &refs) +{ + bool changed = false; + assert(refs.size() == _atomic_refs.size()); + auto atomic_ref = _atomic_refs.begin(); + for (auto ref : refs) { + EntryRef old_ref = (*atomic_ref)->load_relaxed(); + if (ref != old_ref) { + (*atomic_ref)->store_release(ref); + changed = true; + } + ++atomic_ref; + } + assert(atomic_ref == _atomic_refs.end()); + _atomic_refs.clear(); + return changed; +} + +} + +bool +FixedSizeHashMap::normalize_values(const std::function<void(std::vector<EntryRef>&)>& normalize, const EntryRefFilter& filter) +{ + std::vector<EntryRef> refs; + refs.reserve(1024); + bool changed = false; + ChangeWriter change_writer(refs.capacity()); + for (auto& chain_head : _chain_heads) { + uint32_t node_idx = chain_head.load_relaxed(); + while (node_idx != no_node_idx) { + auto& node = _nodes[node_idx]; + EntryRef ref = node.get_kv().second.load_relaxed(); + if (ref.valid()) { + if (filter.has(ref)) { + refs.emplace_back(ref); + change_writer.emplace_back(node.get_kv().second); + if (refs.size() >= refs.capacity()) { + normalize(refs); + changed |= change_writer.write(refs); + refs.clear(); + } + } + } + node_idx = node.get_next_node_idx().load(std::memory_order_relaxed); + } + } + if (!refs.empty()) { + normalize(refs); + changed |= change_writer.write(refs); + } + return changed; +} + +void +FixedSizeHashMap::foreach_value(const std::function<void(const std::vector<EntryRef>&)>& callback, const EntryRefFilter& filter) +{ + std::vector<EntryRef> refs; + refs.reserve(1024); + for (auto& chain_head : _chain_heads) { + uint32_t node_idx = chain_head.load_relaxed(); + while (node_idx != no_node_idx) { + auto& node = _nodes[node_idx]; + EntryRef ref = node.get_kv().second.load_relaxed(); + if (ref.valid()) { + if (filter.has(ref)) { + refs.emplace_back(ref); + if (refs.size() >= refs.capacity()) { + callback(refs); + refs.clear(); + } + } + } + node_idx = node.get_next_node_idx().load(std::memory_order_relaxed); + } + } + if (!refs.empty()) { + callback(refs); + } +} + } diff --git a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h index 035cd84dbee..c522bcc3c33 100644 --- a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h +++ b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h @@ -18,6 +18,7 @@ class MemoryUsage; } namespace vespalib::datastore { +class EntryRefFilter; struct ICompactable; class ShardedHashComparator { @@ -158,8 +159,26 @@ public: size_t size() const noexcept { return _count; } MemoryUsage get_memory_usage() const; void foreach_key(const std::function<void(EntryRef)>& callback) const; - void move_keys(ICompactable& compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits); + void move_keys(ICompactable& compactable, const EntryRefFilter &compacting_buffers); + /* + * Scan dictionary and call normalize function for each value. If + * returned value is different then write back the modified value to + * the dictionary. Used when clearing all posting lists. + */ bool normalize_values(const std::function<EntryRef(EntryRef)>& normalize); + /* + * Scan dictionary and call normalize function for batches of values + * that pass the filter. Write back modified values to the dictionary. + * Used by compaction of posting lists when moving short arrays, + * bitvectors or btree roots. + */ + bool normalize_values(const std::function<void(std::vector<EntryRef>&)>& normalize, const EntryRefFilter& filter); + /* + * Scan dictionary and call callback function for batches of values + * that pass the filter. Used by compaction of posting lists when + * moving btree nodes. + */ + void foreach_value(const std::function<void(const std::vector<EntryRef>&)>& callback, const EntryRefFilter& filter); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h index 886ec095dcd..4fd3bcad5e5 100644 --- a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h @@ -10,7 +10,9 @@ namespace vespalib::datastore { +class CompactionStrategy; class EntryComparator; +class EntryRefFilter; struct ICompactable; class IUniqueStoreDictionaryReadSnapshot; class UniqueStoreAddResult; @@ -28,7 +30,7 @@ public: virtual UniqueStoreAddResult add(const EntryComparator& comp, std::function<EntryRef(void)> insertEntry) = 0; virtual EntryRef find(const EntryComparator& comp) = 0; virtual void remove(const EntryComparator& comp, EntryRef ref) = 0; - virtual void move_keys(ICompactable& compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits) = 0; + virtual void move_keys(ICompactable& compactable, const EntryRefFilter& compacting_buffers) = 0; virtual uint32_t get_num_uniques() const = 0; virtual vespalib::MemoryUsage get_memory_usage() const = 0; virtual void build(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) = 0; @@ -40,7 +42,7 @@ public: virtual vespalib::MemoryUsage get_btree_memory_usage() const = 0; virtual vespalib::MemoryUsage get_hash_memory_usage() const = 0; virtual bool has_held_buffers() const = 0; - virtual void compact_worst(bool compact_btree_dictionary, bool compact_hash_dictionary) = 0; + virtual void compact_worst(bool compact_btree_dictionary, bool compact_hash_dictionary, const CompactionStrategy& compaction_strategy) = 0; }; } diff --git a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp index da4db92a309..019b98a53dd 100644 --- a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp +++ b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp @@ -171,12 +171,12 @@ ShardedHashMap::foreach_key(std::function<void(EntryRef)> callback) const } void -ShardedHashMap::move_keys(ICompactable& compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits) +ShardedHashMap::move_keys(ICompactable& compactable, const EntryRefFilter& compacting_buffers) { for (size_t i = 0; i < num_shards; ++i) { auto map = _maps[i].load(std::memory_order_relaxed); if (map != nullptr) { - map->move_keys(compactable, compacting_buffers, entry_ref_offset_bits); + map->move_keys(compactable, compacting_buffers); } } } @@ -195,6 +195,31 @@ ShardedHashMap::normalize_values(std::function<EntryRef(EntryRef)> normalize) } bool +ShardedHashMap::normalize_values(std::function<void(std::vector<EntryRef>&)> normalize, const EntryRefFilter& filter) +{ + bool changed = false; + for (size_t i = 0; i < num_shards; ++i) { + auto map = _maps[i].load(std::memory_order_relaxed); + if (map != nullptr) { + changed |= map->normalize_values(normalize, filter); + } + } + return changed; +} + +void +ShardedHashMap::foreach_value(std::function<void(const std::vector<EntryRef>&)> callback, const EntryRefFilter& filter) +{ + for (size_t i = 0; i < num_shards; ++i) { + auto map = _maps[i].load(std::memory_order_relaxed); + if (map != nullptr) { + map->foreach_value(callback, filter); + } + } +} + + +bool ShardedHashMap::has_held_buffers() const { return _gen_holder.getHeldBytes() != 0; diff --git a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.h b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.h index df07f7a1990..e0ba9488351 100644 --- a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.h +++ b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.h @@ -11,6 +11,7 @@ namespace vespalib { class MemoryUsage; } namespace vespalib::datastore { class EntryComparator; +class EntryRefFilter; class FixedSizeHashMap; struct ICompactable; @@ -57,8 +58,10 @@ public: const EntryComparator &get_default_comparator() const noexcept { return *_comp; } MemoryUsage get_memory_usage() const; void foreach_key(std::function<void(EntryRef)> callback) const; - void move_keys(ICompactable& compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits); + void move_keys(ICompactable& compactable, const EntryRefFilter& compacting_buffers); bool normalize_values(std::function<EntryRef(EntryRef)> normalize); + bool normalize_values(std::function<void(std::vector<EntryRef>&)> normalize, const EntryRefFilter& filter); + void foreach_value(std::function<void(const std::vector<EntryRef>&)> callback, const EntryRefFilter& filter); bool has_held_buffers() const; void compact_worst_shard(); }; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.h b/vespalib/src/vespa/vespalib/datastore/unique_store.h index 38643d84be0..aea98f406e8 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.h @@ -55,11 +55,11 @@ public: EntryRef find(EntryConstRefType value); EntryConstRefType get(EntryRef ref) const { return _allocator.get(ref); } void remove(EntryRef ref); - std::unique_ptr<Remapper> compact_worst(bool compact_memory, bool compact_address_space); + std::unique_ptr<Remapper> compact_worst(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy); vespalib::MemoryUsage getMemoryUsage() const; vespalib::MemoryUsage get_values_memory_usage() const { return _store.getMemoryUsage(); } vespalib::MemoryUsage get_dictionary_memory_usage() const { return _dict->get_memory_usage(); } - vespalib::AddressSpace get_address_space_usage() const; + vespalib::AddressSpace get_values_address_space_usage() const; // TODO: Consider exposing only the needed functions from allocator Allocator& get_allocator() { return _allocator; } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index d375dbae149..b73b714a6bc 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -102,11 +102,9 @@ private: std::vector<uint32_t> _bufferIdsToCompact; void allocMapping() { - _compacting_buffer.resize(RefT::numBuffers()); _mapping.resize(RefT::numBuffers()); for (const auto bufferId : _bufferIdsToCompact) { BufferState &state = _dataStore.getBufferState(bufferId); - _compacting_buffer[bufferId] = true; _mapping[bufferId].resize(state.get_used_arrays()); } } @@ -124,7 +122,7 @@ private: } void fillMapping() { - _dict.move_keys(*this, _compacting_buffer, RefT::offset_bits); + _dict.move_keys(*this, _compacting_buffer); } public: @@ -140,6 +138,7 @@ public: _bufferIdsToCompact(std::move(bufferIdsToCompact)) { if (!_bufferIdsToCompact.empty()) { + _compacting_buffer.add_buffers(_bufferIdsToCompact); allocMapping(); fillMapping(); } @@ -158,9 +157,9 @@ public: template <typename EntryT, typename RefT, typename Compare, typename Allocator> std::unique_ptr<typename UniqueStore<EntryT, RefT, Compare, Allocator>::Remapper> -UniqueStore<EntryT, RefT, Compare, Allocator>::compact_worst(bool compact_memory, bool compact_address_space) +UniqueStore<EntryT, RefT, Compare, Allocator>::compact_worst(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - std::vector<uint32_t> bufferIdsToCompact = _store.startCompactWorstBuffers(compact_memory, compact_address_space); + std::vector<uint32_t> bufferIdsToCompact = _store.startCompactWorstBuffers(compaction_spec, compaction_strategy); if (bufferIdsToCompact.empty()) { return std::unique_ptr<Remapper>(); } else { @@ -179,7 +178,7 @@ UniqueStore<EntryT, RefT, Compare, Allocator>::getMemoryUsage() const template <typename EntryT, typename RefT, typename Compare, typename Allocator> vespalib::AddressSpace -UniqueStore<EntryT, RefT, Compare, Allocator>::get_address_space_usage() const +UniqueStore<EntryT, RefT, Compare, Allocator>::get_values_address_space_usage() const { return _allocator.get_data_store().getAddressSpaceUsage(); } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h index 3b0169b5a34..d64588e3242 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h @@ -79,7 +79,7 @@ public: UniqueStoreAddResult add(const EntryComparator& comp, std::function<EntryRef(void)> insertEntry) override; EntryRef find(const EntryComparator& comp) override; void remove(const EntryComparator& comp, EntryRef ref) override; - void move_keys(ICompactable& compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits) override; + void move_keys(ICompactable& compactable, const EntryRefFilter& compacting_buffers) override; uint32_t get_num_uniques() const override; vespalib::MemoryUsage get_memory_usage() const override; void build(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) override; @@ -91,7 +91,7 @@ public: vespalib::MemoryUsage get_btree_memory_usage() const override; vespalib::MemoryUsage get_hash_memory_usage() const override; bool has_held_buffers() const override; - void compact_worst(bool compact_btree_dictionary, bool compact_hash_dictionary) override; + void compact_worst(bool compact_btree_dictionary, bool compact_hash_dictionary, const CompactionStrategy& compaction_strategy) override; }; } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp index e88376be9fb..4375b38cf7c 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp @@ -4,6 +4,7 @@ #include "datastore.hpp" #include "entry_comparator_wrapper.h" +#include "entry_ref_filter.h" #include "i_compactable.h" #include "unique_store_add_result.h" #include "unique_store_dictionary.h" @@ -139,15 +140,14 @@ UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::remove(const template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void -UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::move_keys(ICompactable &compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits) +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::move_keys(ICompactable &compactable, const EntryRefFilter& compacting_buffers) { if constexpr (has_btree_dictionary) { auto itr = this->_btree_dict.begin(); while (itr.valid()) { EntryRef oldRef(itr.getKey()); assert(oldRef.valid()); - uint32_t buffer_id = oldRef.buffer_id(entry_ref_offset_bits); - if (compacting_buffers[buffer_id]) { + if (compacting_buffers.has(oldRef)) { EntryRef newRef(compactable.move(oldRef)); this->_btree_dict.thaw(itr); itr.writeKey(newRef); @@ -160,7 +160,7 @@ UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::move_keys(ICo ++itr; } } else { - this->_hash_dict.move_keys(compactable, compacting_buffers, entry_ref_offset_bits); + this->_hash_dict.move_keys(compactable, compacting_buffers); } } @@ -339,11 +339,11 @@ UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::has_held_buff template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void -UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::compact_worst(bool compact_btree_dictionary, bool compact_hash_dictionary) +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::compact_worst(bool compact_btree_dictionary, bool compact_hash_dictionary, const CompactionStrategy& compaction_strategy) { if constexpr (has_btree_dictionary) { if (compact_btree_dictionary) { - this->_btree_dict.compact_worst(); + this->_btree_dict.compact_worst(compaction_strategy); } } else { (void) compact_btree_dictionary; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.h b/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.h index 4a8d72c8685..2501c4fafd9 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.h @@ -3,6 +3,7 @@ #pragma once #include "entryref.h" +#include "entry_ref_filter.h" #include <vector> #include <vespa/vespalib/stllike/allocator.h> @@ -18,43 +19,35 @@ public: using RefType = RefT; protected: - std::vector<bool> _compacting_buffer; + EntryRefFilter _compacting_buffer; std::vector<std::vector<EntryRef, allocator_large<EntryRef>>> _mapping; public: UniqueStoreRemapper() - : _compacting_buffer(), + : _compacting_buffer(RefT::numBuffers(), RefT::offset_bits), _mapping() { } virtual ~UniqueStoreRemapper() = default; EntryRef remap(EntryRef ref) const { - if (ref.valid()) { - RefType internal_ref(ref); - if (!_compacting_buffer[internal_ref.bufferId()]) { - // No remapping for references to buffers not being compacted - return ref; - } else { - auto &inner_mapping = _mapping[internal_ref.bufferId()]; - assert(internal_ref.unscaled_offset() < inner_mapping.size()); - EntryRef mapped_ref = inner_mapping[internal_ref.unscaled_offset()]; - assert(mapped_ref.valid()); - return mapped_ref; - } - } else { - return EntryRef(); - } + RefType internal_ref(ref); + auto &inner_mapping = _mapping[internal_ref.bufferId()]; + assert(internal_ref.unscaled_offset() < inner_mapping.size()); + EntryRef mapped_ref = inner_mapping[internal_ref.unscaled_offset()]; + assert(mapped_ref.valid()); + return mapped_ref; } void remap(vespalib::ArrayRef<EntryRef> refs) const { for (auto &ref : refs) { - auto mapped_ref = remap(ref); - if (mapped_ref != ref) { - ref = mapped_ref; + if (ref.valid() && _compacting_buffer.has(ref)) { + ref = remap(ref); } } } + const EntryRefFilter& get_entry_ref_filter() const noexcept { return _compacting_buffer; } + virtual void done() = 0; }; diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp index b4f7eb5cd96..7407ffd6a4e 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp @@ -17,28 +17,18 @@ namespace vespalib::hwaccelrated { namespace { -class Factory { -public: - virtual ~Factory() = default; - virtual IAccelrated::UP create() const = 0; -}; - -class GenericFactory :public Factory{ -public: - IAccelrated::UP create() const override { return std::make_unique<GenericAccelrator>(); } -}; - +IAccelrated::UP create_accelerator() { #ifdef __x86_64__ -class Avx2Factory :public Factory{ -public: - IAccelrated::UP create() const override { return std::make_unique<Avx2Accelrator>(); } -}; - -class Avx512Factory :public Factory{ -public: - IAccelrated::UP create() const override { return std::make_unique<Avx512Accelrator>(); } -}; + __builtin_cpu_init(); + if (__builtin_cpu_supports("avx512f")) { + return std::make_unique<Avx512Accelrator>(); + } + if (__builtin_cpu_supports("avx2")) { + return std::make_unique<Avx2Accelrator>(); + } #endif + return std::make_unique<GenericAccelrator>(); +} template<typename T> std::vector<T> createAndFill(size_t sz) { @@ -247,42 +237,14 @@ RuntimeVerificator::RuntimeVerificator() verify(thisCpu); } -class Selector -{ -public: - Selector() __attribute__((noinline)); - IAccelrated::UP create() { return _factory->create(); } -private: - std::unique_ptr<Factory> _factory; -}; - -Selector::Selector() : - _factory() -{ -#ifdef __x86_64__ - __builtin_cpu_init (); - if (__builtin_cpu_supports("avx512f")) { - _factory = std::make_unique<Avx512Factory>(); - } else if (__builtin_cpu_supports("avx2")) { - _factory = std::make_unique<Avx2Factory>(); - } else { - _factory = std::make_unique<GenericFactory>(); - } -#else - _factory = std::make_unique<GenericFactory>(); -#endif -} - } -static Selector _G_selector; - RuntimeVerificator _G_verifyAccelrator; const IAccelrated & IAccelrated::getAccelerator() { - static IAccelrated::UP accelrator = _G_selector.create(); + static IAccelrated::UP accelrator = create_accelerator(); return *accelrator; } diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp b/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp index 149ad01b947..a476e23e6cb 100644 --- a/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp @@ -22,23 +22,29 @@ bool is_regex_special_char(char c) noexcept { case '\\': case '+': case '.': + case '?': + case '*': return true; default: return false; } } -std::string dot_separated_glob_to_regex(vespalib::stringref glob) { +// Important: `delimiter` MUST NOT be a character that needs escaping within a regex [charset] +template <bool SupportSingleCharMatch> +std::string char_delimited_glob_to_regex(vespalib::stringref glob, char delimiter) { std::string ret = "^"; ret.reserve(glob.size() + 2); + // Note: we explicitly stop matching at a delimiter boundary. + // This is to make path fragment matching less vulnerable to dirty tricks. + const std::string wildcard_pattern = std::string("[^") + delimiter + "]*"; + // Same applies for single chars; they should only match _within_ a delimited boundary. + const std::string single_char_pattern = std::string("[^") + delimiter + "]"; for (auto c : glob) { if (c == '*') { - // Note: we explicitly stop matching at a dot separator boundary. - // This is to make host name matching less vulnerable to dirty tricks. - ret += "[^.]*"; - } else if (c == '?') { - // Same applies for single chars; they should only match _within_ a dot boundary. - ret += "[^.]"; + ret += wildcard_pattern; + } else if (c == '?' && SupportSingleCharMatch) { + ret += single_char_pattern; } else { if (is_regex_special_char(c)) { ret += '\\'; @@ -52,14 +58,25 @@ std::string dot_separated_glob_to_regex(vespalib::stringref glob) { class RegexHostMatchPattern : public CredentialMatchPattern { Regex _pattern_as_regex; -public: - explicit RegexHostMatchPattern(vespalib::stringref glob_pattern) - : _pattern_as_regex(Regex::from_pattern(dot_separated_glob_to_regex(glob_pattern))) + explicit RegexHostMatchPattern(std::string_view glob_pattern) + : _pattern_as_regex(Regex::from_pattern(glob_pattern)) { } +public: + RegexHostMatchPattern(RegexHostMatchPattern&&) noexcept = default; ~RegexHostMatchPattern() override = default; - [[nodiscard]] bool matches(vespalib::stringref str) const override { + RegexHostMatchPattern& operator=(RegexHostMatchPattern&&) noexcept = default; + + [[nodiscard]] static RegexHostMatchPattern from_dns_glob_pattern(vespalib::stringref glob_pattern) { + return RegexHostMatchPattern(char_delimited_glob_to_regex<true>(glob_pattern, '.')); + } + + [[nodiscard]] static RegexHostMatchPattern from_uri_glob_pattern(vespalib::stringref glob_pattern) { + return RegexHostMatchPattern(char_delimited_glob_to_regex<false>(glob_pattern, '/')); + } + + [[nodiscard]] bool matches(vespalib::stringref str) const noexcept override { return _pattern_as_regex.full_match(std::string_view(str.data(), str.size())); } }; @@ -73,15 +90,19 @@ public: } ~ExactMatchPattern() override = default; - [[nodiscard]] bool matches(vespalib::stringref str) const override { + [[nodiscard]] bool matches(vespalib::stringref str) const noexcept override { return (str == _must_match_exactly); } }; } // anon ns -std::shared_ptr<const CredentialMatchPattern> CredentialMatchPattern::create_from_glob(vespalib::stringref glob_pattern) { - return std::make_shared<const RegexHostMatchPattern>(glob_pattern); +std::shared_ptr<const CredentialMatchPattern> CredentialMatchPattern::create_from_dns_glob(vespalib::stringref glob_pattern) { + return std::make_shared<const RegexHostMatchPattern>(RegexHostMatchPattern::from_dns_glob_pattern(glob_pattern)); +} + +std::shared_ptr<const CredentialMatchPattern> CredentialMatchPattern::create_from_uri_glob(vespalib::stringref glob_pattern) { + return std::make_shared<const RegexHostMatchPattern>(RegexHostMatchPattern::from_uri_glob_pattern(glob_pattern)); } std::shared_ptr<const CredentialMatchPattern> CredentialMatchPattern::create_exact_match(vespalib::stringref str) { @@ -91,9 +112,8 @@ std::shared_ptr<const CredentialMatchPattern> CredentialMatchPattern::create_exa RequiredPeerCredential::RequiredPeerCredential(Field field, vespalib::string must_match_pattern) : _field(field), _original_pattern(std::move(must_match_pattern)), - // FIXME it's not RFC 2459-compliant to use exact-matching for URIs, but that's all we currently need. - _match_pattern(field == Field::SAN_URI ? CredentialMatchPattern::create_exact_match(_original_pattern) - : CredentialMatchPattern::create_from_glob(_original_pattern)) + _match_pattern(field == Field::SAN_URI ? CredentialMatchPattern::create_from_uri_glob(_original_pattern) + : CredentialMatchPattern::create_from_dns_glob(_original_pattern)) { } @@ -111,11 +131,21 @@ void print_joined(std::ostream& os, const Collection& coll, const char* sep) { os << e; } } + +constexpr const char* to_string(RequiredPeerCredential::Field field) noexcept { + switch (field) { + case RequiredPeerCredential::Field::CN: return "CN"; + case RequiredPeerCredential::Field::SAN_DNS: return "SAN_DNS"; + case RequiredPeerCredential::Field::SAN_URI: return "SAN_URI"; + default: abort(); + } +} + } std::ostream& operator<<(std::ostream& os, const RequiredPeerCredential& cred) { os << "RequiredPeerCredential(" - << (cred.field() == RequiredPeerCredential::Field::CN ? "CN" : "SAN_DNS") + << to_string(cred.field()) << " matches '" << cred.original_pattern() << "')"; diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h index c5721858518..4166efc4312 100644 --- a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h +++ b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h @@ -10,9 +10,10 @@ namespace vespalib::net::tls { struct CredentialMatchPattern { virtual ~CredentialMatchPattern() = default; - [[nodiscard]] virtual bool matches(vespalib::stringref str) const = 0; + [[nodiscard]] virtual bool matches(vespalib::stringref str) const noexcept = 0; - static std::shared_ptr<const CredentialMatchPattern> create_from_glob(vespalib::stringref pattern); + static std::shared_ptr<const CredentialMatchPattern> create_from_dns_glob(vespalib::stringref glob_pattern); + static std::shared_ptr<const CredentialMatchPattern> create_from_uri_glob(vespalib::stringref glob_pattern); static std::shared_ptr<const CredentialMatchPattern> create_exact_match(vespalib::stringref pattern); }; @@ -37,7 +38,7 @@ public: && (_original_pattern == rhs._original_pattern)); } - [[nodiscard]] bool matches(vespalib::stringref str) const { + [[nodiscard]] bool matches(vespalib::stringref str) const noexcept { return (_match_pattern && _match_pattern->matches(str)); } diff --git a/vespalib/src/vespa/vespalib/util/rcuvector.h b/vespalib/src/vespa/vespalib/util/rcuvector.h index 0396ee0d459..dd4fa660279 100644 --- a/vespalib/src/vespa/vespalib/util/rcuvector.h +++ b/vespalib/src/vespa/vespalib/util/rcuvector.h @@ -13,10 +13,10 @@ namespace vespalib { template <typename T> class RcuVectorHeld : public GenerationHeldBase { - std::unique_ptr<T> _data; + T _data; public: - RcuVectorHeld(size_t size, std::unique_ptr<T> data); + RcuVectorHeld(size_t size, T&& data); ~RcuVectorHeld(); }; @@ -121,7 +121,7 @@ public: void reset(); void shrink(size_t newSize) __attribute__((noinline)); - void replaceVector(std::unique_ptr<ArrayType> replacement); + void replaceVector(ArrayType replacement); }; template <typename T> diff --git a/vespalib/src/vespa/vespalib/util/rcuvector.hpp b/vespalib/src/vespa/vespalib/util/rcuvector.hpp index 9d7c8ea57d6..3c455149dfd 100644 --- a/vespalib/src/vespa/vespalib/util/rcuvector.hpp +++ b/vespalib/src/vespa/vespalib/util/rcuvector.hpp @@ -9,7 +9,7 @@ namespace vespalib { template <typename T> -RcuVectorHeld<T>::RcuVectorHeld(size_t size, std::unique_ptr<T> data) +RcuVectorHeld<T>::RcuVectorHeld(size_t size, T&& data) : GenerationHeldBase(size), _data(std::move(data)) { } @@ -52,20 +52,21 @@ RcuVectorBase<T>::~RcuVectorBase() = default; template <typename T> void RcuVectorBase<T>::expand(size_t newCapacity) { - std::unique_ptr<ArrayType> tmpData(new ArrayType()); - tmpData->reserve(newCapacity); + ArrayType tmpData; + tmpData.reserve(newCapacity); for (const T & v : _data) { - tmpData->push_back_fast(v); + tmpData.push_back_fast(v); } replaceVector(std::move(tmpData)); } template <typename T> void -RcuVectorBase<T>::replaceVector(std::unique_ptr<ArrayType> replacement) { - replacement->swap(_data); // atomic switch of underlying data - size_t holdSize = replacement->capacity() * sizeof(T); - GenerationHeldBase::UP hold(new RcuVectorHeld<ArrayType>(holdSize, std::move(replacement))); +RcuVectorBase<T>::replaceVector(ArrayType replacement) { + std::atomic_thread_fence(std::memory_order_release); + replacement.swap(_data); // atomic switch of underlying data + size_t holdSize = replacement.capacity() * sizeof(T); + auto hold = std::make_unique<RcuVectorHeld<ArrayType>>(holdSize, std::move(replacement)); _genHolder.hold(std::move(hold)); onReallocation(); } @@ -90,17 +91,18 @@ RcuVectorBase<T>::shrink(size_t newSize) return; } if (!_data.try_unreserve(wantedCapacity)) { - std::unique_ptr<ArrayType> tmpData(new ArrayType()); - tmpData->reserve(wantedCapacity); - tmpData->resize(newSize); + ArrayType tmpData; + tmpData.reserve(wantedCapacity); + tmpData.resize(newSize); for (uint32_t i = 0; i < newSize; ++i) { - (*tmpData)[i] = _data[i]; + tmpData[i] = _data[i]; } + std::atomic_thread_fence(std::memory_order_release); // Users of RCU vector must ensure that no readers use old size // after swap. Attribute vectors uses _committedDocIdLimit for this. - tmpData->swap(_data); // atomic switch of underlying data - size_t holdSize = tmpData->capacity() * sizeof(T); - GenerationHeldBase::UP hold(new RcuVectorHeld<ArrayType>(holdSize, std::move(tmpData))); + tmpData.swap(_data); // atomic switch of underlying data + size_t holdSize = tmpData.capacity() * sizeof(T); + auto hold = std::make_unique<RcuVectorHeld<ArrayType>>(holdSize, std::move(tmpData)); _genHolder.hold(std::move(hold)); onReallocation(); } diff --git a/vespalib/src/vespa/vespalib/util/simple_thread_bundle.cpp b/vespalib/src/vespa/vespalib/util/simple_thread_bundle.cpp index 80bbb3a7ad2..ab83d4e05fd 100644 --- a/vespalib/src/vespa/vespalib/util/simple_thread_bundle.cpp +++ b/vespalib/src/vespa/vespalib/util/simple_thread_bundle.cpp @@ -8,6 +8,8 @@ using namespace vespalib::fixed_thread_bundle; namespace vespalib { +VESPA_THREAD_STACK_TAG(simple_thread_bundle_executor); + namespace { struct SignalHook : Runnable { @@ -43,7 +45,7 @@ Runnable::UP wrap(Runnable *runnable) { } Runnable::UP chain(Runnable::UP first, Runnable::UP second) { - return Runnable::UP(new HookPair(std::move(first), std::move(second))); + return std::make_unique<HookPair>(std::move(first), std::move(second)); } } // namespace vespalib::<unnamed> @@ -173,4 +175,19 @@ SimpleThreadBundle::run(const std::vector<Runnable*> &targets) latch.await(); } +SimpleThreadBundle::Worker::Worker(Signal &s, Runnable::UP h) + : thread(*this, simple_thread_bundle_executor), + signal(s), + hook(std::move(h)) +{ + thread.start(); +} +void +SimpleThreadBundle::Worker::run() { + for (size_t gen = 0; signal.wait(gen) > 0; ) { + hook->run(); +} + +} + } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/simple_thread_bundle.h b/vespalib/src/vespa/vespalib/util/simple_thread_bundle.h index f0aaccc2525..d9a29ee7bef 100644 --- a/vespalib/src/vespa/vespalib/util/simple_thread_bundle.h +++ b/vespalib/src/vespa/vespalib/util/simple_thread_bundle.h @@ -112,14 +112,8 @@ private: Thread thread; Signal &signal; Runnable::UP hook; - Worker(Signal &s, Runnable::UP h) : thread(*this), signal(s), hook(std::move(h)) { - thread.start(); - } - void run() override { - for (size_t gen = 0; signal.wait(gen) > 0; ) { - hook->run(); - } - } + Worker(Signal &s, Runnable::UP h); + void run() override; }; Work _work; diff --git a/vespalib/src/vespa/vespalib/util/thread.cpp b/vespalib/src/vespa/vespalib/util/thread.cpp index c02a7a3b063..c3230bf313d 100644 --- a/vespalib/src/vespa/vespalib/util/thread.cpp +++ b/vespalib/src/vespa/vespalib/util/thread.cpp @@ -9,9 +9,9 @@ namespace vespalib { __thread Thread *Thread::_currentThread = nullptr; -Thread::Proxy::Proxy(Thread &parent, Runnable &target) - : thread(parent), runnable(target), - start(), started(), cancel(false) +Thread::Proxy::Proxy(Thread &parent, Runnable &target, init_fun_t init_fun_in) + : thread(parent), runnable(target), init_fun(std::move(init_fun_in)), + start(), started(), cancel(false) { } void @@ -22,7 +22,7 @@ Thread::Proxy::Run(FastOS_ThreadInterface *, void *) start.await(); if (!cancel) { started.countDown(); - runnable.run(); + init_fun(runnable); } assert(_currentThread == &thread); _currentThread = nullptr; @@ -30,8 +30,8 @@ Thread::Proxy::Run(FastOS_ThreadInterface *, void *) Thread::Proxy::~Proxy() = default; -Thread::Thread(Runnable &runnable) - : _proxy(*this, runnable), +Thread::Thread(Runnable &runnable, init_fun_t init_fun_in) + : _proxy(*this, runnable, std::move(init_fun_in)), _pool(STACK_SIZE, 1), _lock(), _cond(), diff --git a/vespalib/src/vespa/vespalib/util/thread.h b/vespalib/src/vespa/vespalib/util/thread.h index 8873f23ee98..e08f3ca1100 100644 --- a/vespalib/src/vespa/vespalib/util/thread.h +++ b/vespalib/src/vespa/vespalib/util/thread.h @@ -15,17 +15,19 @@ namespace vespalib { class Thread : public Active { private: + using init_fun_t = Runnable::init_fun_t; enum { STACK_SIZE = 256*1024 }; static __thread Thread *_currentThread; struct Proxy : FastOS_Runnable { Thread &thread; Runnable &runnable; + init_fun_t init_fun; vespalib::Gate start; vespalib::Gate started; bool cancel; - Proxy(Thread &parent, Runnable &target); + Proxy(Thread &parent, Runnable &target, init_fun_t init_fun_in); ~Proxy() override; void Run(FastOS_ThreadInterface *thisThread, void *arguments) override; @@ -39,7 +41,7 @@ private: bool _woken; public: - Thread(Runnable &runnable); + Thread(Runnable &runnable, init_fun_t init_fun_in); ~Thread() override; void start() override; Thread &stop() override; |