Add member functions for scanning dictionary and compacting posting store.

3 files changed, 61 insertions, 26 deletions

diff --git a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp
index f534d1ce73f..7e49f8498da 100644
--- a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp
+++ b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp
@@ -64,9 +64,12 @@ protected:
 
     void inc_generation()
     {
+        _value_store.freeze_dictionary();
         _store.freeze();
+        _value_store.transfer_hold_lists(_gen_handler.getCurrentGeneration());
         _store.transferHoldLists(_gen_handler.getCurrentGeneration());
         _gen_handler.incGeneration();
+        _value_store.trim_hold_lists(_gen_handler.getFirstUsedGeneration());
         _store.trimHoldLists(_gen_handler.getFirstUsedGeneration());
     }
 
@@ -97,7 +100,8 @@ protected:
         return sequence;
     }
 
-    std::vector<EntryRef> populate(uint32_t sequence_length);
+    void populate(uint32_t sequence_length);
+    EntryRef get_posting_ref(int key);
     void test_compact_btree_nodes(uint32_t sequence_length);
     void test_compact_sequence(uint32_t sequence_length);
 };
@@ -114,16 +118,18 @@ PostingStoreTest::PostingStoreTest()
 
 PostingStoreTest::~PostingStoreTest()
 {
+    _value_store.get_dictionary().clear_all_posting_lists([this](EntryRef posting_idx) { _store.clear(posting_idx); });
     _store.clearBuilder();
     inc_generation();
 }
 
-std::vector<EntryRef>
+void
 PostingStoreTest::populate(uint32_t sequence_length)
 {
-    auto &store = _store;
-    EntryRef ref1 = add_sequence(4, 4 + sequence_length);
-    EntryRef ref2 = add_sequence(5, 5 + sequence_length);
+    auto& store = _store;
+    auto& dictionary = _value_store.get_dictionary();
+    dictionary.update_posting_list(_value_store.insert(1), _value_store.make_comparator(), [this, sequence_length](EntryRef) { return add_sequence(4, 4 + sequence_length); });
+    dictionary.update_posting_list(_value_store.insert(2), _value_store.make_comparator(), [this, sequence_length](EntryRef) { return add_sequence(5, 5 + sequence_length); });
     std::vector<EntryRef> refs;
     for (int i = 0; i < 1000; ++i) {
         refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length));
@@ -132,49 +138,54 @@ PostingStoreTest::populate(uint32_t sequence_length)
         store.clear(ref);
     }
     inc_generation();
-    return { ref1, ref2 };
+}
+
+EntryRef
+PostingStoreTest::get_posting_ref(int key)
+{
+    auto &dictionary = _value_store.get_dictionary();
+    auto root = dictionary.get_frozen_root();
+    return dictionary.find_posting_list(_value_store.make_comparator(key), root).second;
 }
 
 void
 PostingStoreTest::test_compact_sequence(uint32_t sequence_length)
 {
-    auto populated_refs = populate(sequence_length);
+    populate(sequence_length);
     auto &store = _store;
-    EntryRef ref1 = populated_refs[0];
-    EntryRef ref2 = populated_refs[1];
+    EntryRef old_ref1 = get_posting_ref(1);
+    EntryRef old_ref2 = get_posting_ref(2);
     auto usage_before = store.getMemoryUsage();
     for (uint32_t pass = 0; pass < 15; ++pass) {
-        auto to_hold = store.start_compact_worst_buffers();
-        ref1 = store.move(ref1);
-        ref2 = store.move(ref2);
-        store.finishCompact(to_hold);
+        store.compact_worst_buffers();
         inc_generation();
     }
-    EXPECT_NE(populated_refs[0], ref1);
-    EXPECT_NE(populated_refs[1], ref2);
+    EntryRef ref1 = get_posting_ref(1);
+    EntryRef ref2 = get_posting_ref(2);
+    EXPECT_NE(old_ref1, ref1);
+    EXPECT_NE(old_ref2, ref2);
     EXPECT_EQ(make_exp_sequence(4, 4 + sequence_length), get_sequence(ref1));
     EXPECT_EQ(make_exp_sequence(5, 5 + sequence_length), get_sequence(ref2));
     auto usage_after = store.getMemoryUsage();
     EXPECT_GT(usage_before.deadBytes(), usage_after.deadBytes());
-    store.clear(ref1);
-    store.clear(ref2);
 }
 
 void
 PostingStoreTest::test_compact_btree_nodes(uint32_t sequence_length)
 {
-    auto populated_refs = populate(sequence_length);
+    populate(sequence_length);
     auto &store = _store;
-    EntryRef ref1 = populated_refs[0];
-    EntryRef ref2 = populated_refs[1];
+    EntryRef old_ref1 = get_posting_ref(1);
+    EntryRef old_ref2 = get_posting_ref(2);
     auto usage_before = store.getMemoryUsage();
     for (uint32_t pass = 0; pass < 15; ++pass) {
-        auto to_hold = store.start_compact_worst_btree_nodes();
-        store.move_btree_nodes(ref1);
-        store.move_btree_nodes(ref2);
-        store.finish_compact_worst_btree_nodes(to_hold);
+        store.compact_worst_btree_nodes();
         inc_generation();
     }
+    EntryRef ref1 = get_posting_ref(1);
+    EntryRef ref2 = get_posting_ref(2);
+    EXPECT_EQ(old_ref1, ref1);
+    EXPECT_EQ(old_ref2, ref2);
     EXPECT_EQ(make_exp_sequence(4, 4 + sequence_length), get_sequence(ref1));
     EXPECT_EQ(make_exp_sequence(5, 5 + sequence_length), get_sequence(ref2));
     auto usage_after = store.getMemoryUsage();
@@ -185,8 +196,6 @@ PostingStoreTest::test_compact_btree_nodes(uint32_t sequence_length)
     } else {
         EXPECT_EQ(usage_before.deadBytes(), usage_after.deadBytes());
     }
-    store.clear(ref1);
-    store.clear(ref2);
 }
 
 VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(PostingStoreMultiTest,
diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp
index d0cf9ef4e43..336f53fb66a 100644
--- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp
@@ -697,6 +697,29 @@ PostingStore<DataT>::move(EntryRef ref)
     return allocKeyDataCopy(shortArray, clusterSize).ref;
 }
 
+template <typename DataT>
+void
+PostingStore<DataT>::compact_worst_btree_nodes()
+{
+    auto to_hold = this->start_compact_worst_btree_nodes();
+    _dictionary.normalize_posting_lists([this](EntryRef posting_idx) -> EntryRef
+                                        {
+                                            move_btree_nodes(posting_idx);
+                                            return posting_idx;
+                                        });
+    this->finish_compact_worst_btree_nodes(to_hold);
+}
+
+template <typename DataT>
+void
+PostingStore<DataT>::compact_worst_buffers()
+{
+    auto to_hold = this->start_compact_worst_buffers();
+    _dictionary.normalize_posting_lists([this](EntryRef posting_idx) -> EntryRef
+                                        { return move(posting_idx); });
+    this->finishCompact(to_hold);
+}
+
 template class PostingStore<BTreeNoLeafData>;
 template class PostingStore<int32_t>;
 
diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h
index 62254c6f012..1abe9c839c0 100644
--- a/searchlib/src/vespa/searchlib/attribute/postingstore.h
+++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h
@@ -187,6 +187,9 @@ public:
 
     void move_btree_nodes(EntryRef ref);
     EntryRef move(EntryRef ref);
+
+    void compact_worst_btree_nodes();
+    void compact_worst_buffers();
 private:
     size_t internalSize(uint32_t typeId, const RefType & iRef) const;
     size_t internalFrozenSize(uint32_t typeId, const RefType & iRef) const;