Add EntryRefFilter class.

13 files changed, 144 insertions, 56 deletions

diff --git a/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp b/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp
index 138dedbdd38..796e19a97d1 100644
--- a/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp
+++ b/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/vespalib/datastore/sharded_hash_map.h>
+#include <vespa/vespalib/datastore/entry_ref_filter.h>
 #include <vespa/vespalib/datastore/i_compactable.h>
 #include <vespa/vespalib/datastore/unique_store_allocator.h>
 #include <vespa/vespalib/datastore/unique_store_comparator.h>
@@ -19,6 +20,7 @@
 LOG_SETUP("vespalib_datastore_shared_hash_test");
 
 using vespalib::datastore::EntryRef;
+using vespalib::datastore::EntryRefFilter;
 using vespalib::datastore::ICompactable;
 using RefT = vespalib::datastore::EntryRefT<22>;
 using MyAllocator = vespalib::datastore::UniqueStoreAllocator<uint32_t, RefT>;
@@ -123,7 +125,7 @@ struct DataStoreShardedHashTest : public ::testing::Test
     void populate_sample_data(uint32_t cnt);
     void populate_sample_values(uint32_t cnt);
     void clear_sample_values(uint32_t cnt);
-    void test_normalize_values(bool filter, bool one_filter);
+    void test_normalize_values(bool use_filter, bool one_filter);
     void test_foreach_value(bool one_filter);
 };
 
@@ -282,20 +284,30 @@ DataStoreShardedHashTest::clear_sample_values(uint32_t cnt)
     }
 }
 
+namespace {
+
+template <typename RefT>
+EntryRefFilter
+make_entry_ref_filter(bool one_filter)
+{
+    if (one_filter) {
+        EntryRefFilter filter(RefT::numBuffers(), RefT::offset_bits);
+        filter.add_buffer(3);
+        return filter;
+    }
+    return EntryRefFilter::create_all_filter(RefT::numBuffers(), RefT::offset_bits);
+}
+
+}
+
 void
-DataStoreShardedHashTest::test_normalize_values(bool filter, bool one_filter)
+DataStoreShardedHashTest::test_normalize_values(bool use_filter, bool one_filter)
 {
     populate_sample_data(large_population);
     populate_sample_values(large_population);
-    if (filter) {
-        std::vector<bool> bfilter;
-        if (one_filter) {
-            bfilter = std::vector<bool>(RefT::numBuffers());
-            bfilter[3] = true;
-        } else {
-            bfilter = std::vector<bool>(RefT::numBuffers(), true);
-        }
-        EXPECT_TRUE(_hash_map.normalize_values([](std::vector<EntryRef> &refs) noexcept { for (auto &ref : refs) { RefT iref(ref); ref = RefT(iref.offset() + 300, iref.bufferId()); } }, bfilter, RefT::offset_bits));
+    if (use_filter) {
+        auto filter = make_entry_ref_filter<RefT>(one_filter);
+        EXPECT_TRUE(_hash_map.normalize_values([](std::vector<EntryRef> &refs) noexcept { for (auto &ref : refs) { RefT iref(ref); ref = RefT(iref.offset() + 300, iref.bufferId()); } }, filter));
     } else {
         EXPECT_TRUE(_hash_map.normalize_values([](EntryRef ref) noexcept { RefT iref(ref); return RefT(iref.offset() + 300, iref.bufferId()); }));
     }
@@ -305,7 +317,7 @@ DataStoreShardedHashTest::test_normalize_values(bool filter, bool one_filter)
         ASSERT_NE(result, nullptr);
         EXPECT_EQ(i, _allocator.get_wrapped(result->first.load_relaxed()).value());
         ASSERT_EQ(select_buffer(i), RefT(result->second.load_relaxed()).bufferId());
-        if (filter && one_filter && select_buffer(i) != 3) {
+        if (use_filter && one_filter && select_buffer(i) != 3) {
             ASSERT_EQ(i + 200, RefT(result->second.load_relaxed()).offset());
         } else {
             ASSERT_EQ(i + 500, RefT(result->second.load_relaxed()).offset());
@@ -320,17 +332,11 @@ DataStoreShardedHashTest::test_foreach_value(bool one_filter)
     populate_sample_data(large_population);
     populate_sample_values(large_population);
 
-    std::vector<bool> bfilter;
-    if (one_filter) {
-        bfilter = std::vector<bool>(RefT::numBuffers());
-        bfilter[3] = true;
-    } else {
-        bfilter = std::vector<bool>(RefT::numBuffers(), true);
-    }
+    auto filter = make_entry_ref_filter<RefT>(one_filter);
     std::vector<EntryRef> exp_refs;
-    EXPECT_FALSE(_hash_map.normalize_values([&exp_refs](std::vector<EntryRef>& refs) { exp_refs.insert(exp_refs.end(), refs.begin(), refs.end()); }, bfilter, RefT::offset_bits));
+    EXPECT_FALSE(_hash_map.normalize_values([&exp_refs](std::vector<EntryRef>& refs) { exp_refs.insert(exp_refs.end(), refs.begin(), refs.end()); }, filter));
     std::vector<EntryRef> act_refs;
-    _hash_map.foreach_value([&act_refs](const std::vector<EntryRef> &refs) { act_refs.insert(act_refs.end(), refs.begin(), refs.end()); }, bfilter, RefT::offset_bits);
+    _hash_map.foreach_value([&act_refs](const std::vector<EntryRef> &refs) { act_refs.insert(act_refs.end(), refs.begin(), refs.end()); }, filter);
     EXPECT_EQ(exp_refs, act_refs);
     clear_sample_values(large_population);
 }
@@ -396,7 +402,8 @@ TEST_F(DataStoreShardedHashTest, move_keys_works)
     _hash_map.foreach_key([&refs](EntryRef ref) { refs.emplace_back(ref); });
     std::vector<EntryRef> new_refs;
     MyCompactable my_compactable(_allocator, new_refs);
-    _hash_map.move_keys(my_compactable, std::vector<bool>(RefT::numBuffers(), true), RefT::offset_bits);
+    auto filter = make_entry_ref_filter<RefT>(false);
+    _hash_map.move_keys(my_compactable, filter);
     std::vector<EntryRef> verify_new_refs;
     _hash_map.foreach_key([&verify_new_refs](EntryRef ref) { verify_new_refs.emplace_back(ref); });
     EXPECT_EQ(small_population, refs.size());
diff --git a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt
index 6c6f5258555..9b796c62232 100644
--- a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt
@@ -8,6 +8,7 @@ vespa_add_library(vespalib_vespalib_datastore OBJECT
     datastore.cpp
     datastorebase.cpp
     entryref.cpp
+    entry_ref_filter.cpp
     fixed_size_hash_map.cpp
     sharded_hash_map.cpp
     unique_store.cpp
diff --git a/vespalib/src/vespa/vespalib/datastore/entry_ref_filter.cpp b/vespalib/src/vespa/vespalib/datastore/entry_ref_filter.cpp
new file mode 100644
index 00000000000..87c3c87636c
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/datastore/entry_ref_filter.cpp
@@ -0,0 +1,28 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "entry_ref_filter.h"
+
+namespace vespalib::datastore {
+
+EntryRefFilter::EntryRefFilter(std::vector<bool> filter, uint32_t offset_bits)
+    : _filter(std::move(filter)),
+      _offset_bits(offset_bits)
+{
+}
+
+EntryRefFilter::EntryRefFilter(uint32_t num_buffers, uint32_t offset_bits)
+    : _filter(num_buffers),
+      _offset_bits(offset_bits)
+{
+}
+
+EntryRefFilter::~EntryRefFilter() = default;
+
+EntryRefFilter
+EntryRefFilter::create_all_filter(uint32_t num_buffers, uint32_t offset_bits)
+{
+    std::vector<bool> filter(num_buffers, true);
+    return EntryRefFilter(std::move(filter), offset_bits);
+}
+
+}
diff --git a/vespalib/src/vespa/vespalib/datastore/entry_ref_filter.h b/vespalib/src/vespa/vespalib/datastore/entry_ref_filter.h
new file mode 100644
index 00000000000..c06d843fbd0
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/datastore/entry_ref_filter.h
@@ -0,0 +1,35 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "entryref.h"
+#include <vector>
+
+namespace vespalib::datastore {
+
+/*
+ * Class to filter entry refs based on which buffer the entry is referencing.
+ *
+ * Buffers being allowed have corresponding bit in _filter set.
+ */
+class EntryRefFilter {
+    std::vector<bool> _filter;
+    uint32_t          _offset_bits;
+    EntryRefFilter(std::vector<bool> filter, uint32_t offset_bits);
+public:
+    EntryRefFilter(uint32_t num_buffers, uint32_t offset_bits);
+    ~EntryRefFilter();
+    bool has(EntryRef ref) const {
+        uint32_t buffer_id = ref.buffer_id(_offset_bits);
+        return _filter[buffer_id];
+    }
+    void add_buffer(uint32_t buffer_id) { _filter[buffer_id] = true; }
+    void add_buffers(const std::vector<uint32_t>& ids) {
+        for (auto buffer_id : ids) {
+            _filter[buffer_id] = true;
+        }
+    }
+    static EntryRefFilter create_all_filter(uint32_t num_buffers, uint32_t offset_bits);
+};
+
+}
diff --git a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp
index 79e4cb2ff74..6f001ce3c94 100644
--- a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp
+++ b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp
@@ -2,6 +2,7 @@
 
 #include "fixed_size_hash_map.h"
 #include "entry_comparator.h"
+#include "entry_ref_filter.h"
 #include "i_compactable.h"
 #include <vespa/vespalib/util/array.hpp>
 #include <vespa/vespalib/util/memoryusage.h>
@@ -182,7 +183,7 @@ FixedSizeHashMap::foreach_key(const std::function<void(EntryRef)>& callback) con
 }
 
 void
-FixedSizeHashMap::move_keys(ICompactable& compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits)
+FixedSizeHashMap::move_keys(ICompactable& compactable, const EntryRefFilter &compacting_buffers)
 {
     for (auto& chain_head : _chain_heads) {
         uint32_t node_idx = chain_head.load_relaxed();
@@ -190,8 +191,7 @@ FixedSizeHashMap::move_keys(ICompactable& compactable, const std::vector<bool>&
             auto& node = _nodes[node_idx];
             EntryRef old_ref = node.get_kv().first.load_relaxed();
             assert(old_ref.valid());
-            uint32_t buffer_id = old_ref.buffer_id(entry_ref_offset_bits);
-            if (compacting_buffers[buffer_id]) {
+            if (compacting_buffers.has(old_ref)) {
                 EntryRef new_ref = compactable.move(old_ref);
                 node.get_kv().first.store_release(new_ref);
             }
@@ -261,7 +261,7 @@ ChangeWriter::write(const std::vector<EntryRef> &refs)
 }
 
 bool
-FixedSizeHashMap::normalize_values(const std::function<void(std::vector<EntryRef>&)>& normalize, const std::vector<bool>& filter, uint32_t entry_ref_offset_bits)
+FixedSizeHashMap::normalize_values(const std::function<void(std::vector<EntryRef>&)>& normalize, const EntryRefFilter& filter)
 {
     std::vector<EntryRef> refs;
     refs.reserve(1024);
@@ -273,8 +273,7 @@ FixedSizeHashMap::normalize_values(const std::function<void(std::vector<EntryRef
             auto& node = _nodes[node_idx];
             EntryRef ref = node.get_kv().second.load_relaxed();
             if (ref.valid()) {
-                uint32_t buffer_id = ref.buffer_id(entry_ref_offset_bits);
-                if (filter[buffer_id]) {
+                if (filter.has(ref)) {
                     refs.emplace_back(ref);
                     change_writer.emplace_back(node.get_kv().second);
                     if (refs.size() >= refs.capacity()) {
@@ -295,7 +294,7 @@ FixedSizeHashMap::normalize_values(const std::function<void(std::vector<EntryRef
 }
 
 void
-FixedSizeHashMap::foreach_value(const std::function<void(const std::vector<EntryRef>&)>& callback, const std::vector<bool>& filter, uint32_t entry_ref_offset_bits)
+FixedSizeHashMap::foreach_value(const std::function<void(const std::vector<EntryRef>&)>& callback, const EntryRefFilter& filter)
 {
     std::vector<EntryRef> refs;
     refs.reserve(1024);
@@ -305,8 +304,7 @@ FixedSizeHashMap::foreach_value(const std::function<void(const std::vector<Entry
             auto& node = _nodes[node_idx];
             EntryRef ref = node.get_kv().second.load_relaxed();
             if (ref.valid()) {
-                uint32_t buffer_id = ref.buffer_id(entry_ref_offset_bits);
-                if (filter[buffer_id]) {
+                if (filter.has(ref)) {
                     refs.emplace_back(ref);
                     if (refs.size() >= refs.capacity()) {
                         callback(refs);
diff --git a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h
index 0356725883d..c522bcc3c33 100644
--- a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h
+++ b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h
@@ -18,6 +18,7 @@ class MemoryUsage;
 }
 namespace vespalib::datastore {
 
+class EntryRefFilter;
 struct ICompactable;
 
 class ShardedHashComparator {
@@ -158,10 +159,26 @@ public:
     size_t size() const noexcept { return _count; }
     MemoryUsage get_memory_usage() const;
     void foreach_key(const std::function<void(EntryRef)>& callback) const;
-    void move_keys(ICompactable& compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits);
+    void move_keys(ICompactable& compactable, const EntryRefFilter &compacting_buffers);
+    /*
+     * Scan dictionary and call normalize function for each value. If
+     * returned value is different then write back the modified value to
+     * the dictionary. Used when clearing all posting lists.
+     */
     bool normalize_values(const std::function<EntryRef(EntryRef)>& normalize);
-    bool normalize_values(const std::function<void(std::vector<EntryRef>&)>& normalize, const std::vector<bool>& filter, uint32_t entry_ref_offset_bits);
-    void foreach_value(const std::function<void(const std::vector<EntryRef>&)>& callback, const std::vector<bool>& filter, uint32_t entry_ref_offset_bits);
+    /*
+     * Scan dictionary and call normalize function for batches of values
+     * that pass the filter. Write back modified values to the dictionary.
+     * Used by compaction of posting lists when moving short arrays,
+     * bitvectors or btree roots.
+     */
+    bool normalize_values(const std::function<void(std::vector<EntryRef>&)>& normalize, const EntryRefFilter& filter);
+    /*
+     * Scan dictionary and call callback function for batches of values
+     * that pass the filter. Used by compaction of posting lists when
+     * moving btree nodes.
+     */
+    void foreach_value(const std::function<void(const std::vector<EntryRef>&)>& callback, const EntryRefFilter& filter);
 };
 
 }
diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h
index 886ec095dcd..cf848167070 100644
--- a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h
+++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h
@@ -11,6 +11,7 @@
 namespace vespalib::datastore {
 
 class EntryComparator;
+class EntryRefFilter;
 struct ICompactable;
 class IUniqueStoreDictionaryReadSnapshot;
 class UniqueStoreAddResult;
@@ -28,7 +29,7 @@ public:
     virtual UniqueStoreAddResult add(const EntryComparator& comp, std::function<EntryRef(void)> insertEntry) = 0;
     virtual EntryRef find(const EntryComparator& comp) = 0;
     virtual void remove(const EntryComparator& comp, EntryRef ref) = 0;
-    virtual void move_keys(ICompactable& compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits) = 0;
+    virtual void move_keys(ICompactable& compactable, const EntryRefFilter& compacting_buffers) = 0;
     virtual uint32_t get_num_uniques() const = 0;
     virtual vespalib::MemoryUsage get_memory_usage() const = 0;
     virtual void build(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) = 0;
diff --git a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp
index 89f1b5f7b6f..019b98a53dd 100644
--- a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp
+++ b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp
@@ -171,12 +171,12 @@ ShardedHashMap::foreach_key(std::function<void(EntryRef)> callback) const
 }
 
 void
-ShardedHashMap::move_keys(ICompactable& compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits)
+ShardedHashMap::move_keys(ICompactable& compactable, const EntryRefFilter& compacting_buffers)
 {
     for (size_t i = 0; i < num_shards; ++i) {
         auto map = _maps[i].load(std::memory_order_relaxed);
         if (map != nullptr) {
-            map->move_keys(compactable, compacting_buffers, entry_ref_offset_bits);
+            map->move_keys(compactable, compacting_buffers);
         }
     }
 }
@@ -195,25 +195,25 @@ ShardedHashMap::normalize_values(std::function<EntryRef(EntryRef)> normalize)
 }
 
 bool
-ShardedHashMap::normalize_values(std::function<void(std::vector<EntryRef>&)> normalize, const std::vector<bool>& filter, uint32_t entry_ref_offset_bits)
+ShardedHashMap::normalize_values(std::function<void(std::vector<EntryRef>&)> normalize, const EntryRefFilter& filter)
 {
     bool changed = false;
     for (size_t i = 0; i < num_shards; ++i) {
         auto map = _maps[i].load(std::memory_order_relaxed);
         if (map != nullptr) {
-            changed |= map->normalize_values(normalize, filter, entry_ref_offset_bits);
+            changed |= map->normalize_values(normalize, filter);
         }
     }
     return changed;
 }
 
 void
-ShardedHashMap::foreach_value(std::function<void(const std::vector<EntryRef>&)> callback, const std::vector<bool>& filter, uint32_t entry_ref_offset_bits)
+ShardedHashMap::foreach_value(std::function<void(const std::vector<EntryRef>&)> callback, const EntryRefFilter& filter)
 {
     for (size_t i = 0; i < num_shards; ++i) {
         auto map = _maps[i].load(std::memory_order_relaxed);
         if (map != nullptr) {
-            map->foreach_value(callback, filter, entry_ref_offset_bits);
+            map->foreach_value(callback, filter);
         }
     }
 }
diff --git a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.h b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.h
index c12c6a4bbb2..e0ba9488351 100644
--- a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.h
+++ b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.h
@@ -11,6 +11,7 @@ namespace vespalib { class MemoryUsage; }
 namespace vespalib::datastore {
 
 class EntryComparator;
+class EntryRefFilter;
 class FixedSizeHashMap;
 struct ICompactable;
 
@@ -57,10 +58,10 @@ public:
     const EntryComparator &get_default_comparator() const noexcept { return *_comp; }
     MemoryUsage get_memory_usage() const;
     void foreach_key(std::function<void(EntryRef)> callback) const;
-    void move_keys(ICompactable& compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits);
+    void move_keys(ICompactable& compactable, const EntryRefFilter& compacting_buffers);
     bool normalize_values(std::function<EntryRef(EntryRef)> normalize);
-    bool normalize_values(std::function<void(std::vector<EntryRef>&)> normalize, const std::vector<bool>& filter, uint32_t entry_ref_offset_bits);
-    void foreach_value(std::function<void(const std::vector<EntryRef>&)> callback, const std::vector<bool>& filter, uint32_t entry_ref_offset_bits);
+    bool normalize_values(std::function<void(std::vector<EntryRef>&)> normalize, const EntryRefFilter& filter);
+    void foreach_value(std::function<void(const std::vector<EntryRef>&)> callback, const EntryRefFilter& filter);
     bool has_held_buffers() const;
     void compact_worst_shard();
 };
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp
index d375dbae149..b02a2e52185 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp
@@ -102,11 +102,9 @@ private:
     std::vector<uint32_t> _bufferIdsToCompact;
 
     void allocMapping() {
-        _compacting_buffer.resize(RefT::numBuffers());
         _mapping.resize(RefT::numBuffers());
         for (const auto bufferId : _bufferIdsToCompact) {
             BufferState &state = _dataStore.getBufferState(bufferId);
-            _compacting_buffer[bufferId] = true;
             _mapping[bufferId].resize(state.get_used_arrays());
         }
     }
@@ -124,7 +122,7 @@ private:
     }
     
     void fillMapping() {
-        _dict.move_keys(*this, _compacting_buffer, RefT::offset_bits);
+        _dict.move_keys(*this, _compacting_buffer);
     }
 
 public:
@@ -140,6 +138,7 @@ public:
           _bufferIdsToCompact(std::move(bufferIdsToCompact))
     {
         if (!_bufferIdsToCompact.empty()) {
+            _compacting_buffer.add_buffers(_bufferIdsToCompact);
             allocMapping();
             fillMapping();
         }
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h
index 3b0169b5a34..54d541853c7 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h
@@ -79,7 +79,7 @@ public:
     UniqueStoreAddResult add(const EntryComparator& comp, std::function<EntryRef(void)> insertEntry) override;
     EntryRef find(const EntryComparator& comp) override;
     void remove(const EntryComparator& comp, EntryRef ref) override;
-    void move_keys(ICompactable& compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits) override;
+    void move_keys(ICompactable& compactable, const EntryRefFilter& compacting_buffers) override;
     uint32_t get_num_uniques() const override;
     vespalib::MemoryUsage get_memory_usage() const override;
     void build(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) override;
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp
index e88376be9fb..13ae0a317e0 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp
@@ -4,6 +4,7 @@
 
 #include "datastore.hpp"
 #include "entry_comparator_wrapper.h"
+#include "entry_ref_filter.h"
 #include "i_compactable.h"
 #include "unique_store_add_result.h"
 #include "unique_store_dictionary.h"
@@ -139,15 +140,14 @@ UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::remove(const
 
 template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT>
 void
-UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::move_keys(ICompactable &compactable, const std::vector<bool>& compacting_buffers, uint32_t entry_ref_offset_bits)
+UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::move_keys(ICompactable &compactable, const EntryRefFilter& compacting_buffers)
 {
     if constexpr (has_btree_dictionary) {
         auto itr = this->_btree_dict.begin();
         while (itr.valid()) {
             EntryRef oldRef(itr.getKey());
             assert(oldRef.valid());
-            uint32_t buffer_id = oldRef.buffer_id(entry_ref_offset_bits);
-            if (compacting_buffers[buffer_id]) {
+            if (compacting_buffers.has(oldRef)) {
                 EntryRef newRef(compactable.move(oldRef));
                 this->_btree_dict.thaw(itr);
                 itr.writeKey(newRef);
@@ -160,7 +160,7 @@ UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::move_keys(ICo
             ++itr;
         }
     } else {
-        this->_hash_dict.move_keys(compactable, compacting_buffers, entry_ref_offset_bits);
+        this->_hash_dict.move_keys(compactable, compacting_buffers);
     }
 }
 
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.h b/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.h
index 4a8d72c8685..873af07a902 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_remapper.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include "entryref.h"
+#include "entry_ref_filter.h"
 #include <vector>
 #include <vespa/vespalib/stllike/allocator.h>
 
@@ -18,11 +19,11 @@ public:
     using RefType = RefT;
 
 protected:
-    std::vector<bool> _compacting_buffer;
+    EntryRefFilter _compacting_buffer;
     std::vector<std::vector<EntryRef, allocator_large<EntryRef>>> _mapping;
 public:
     UniqueStoreRemapper()
-        : _compacting_buffer(),
+        : _compacting_buffer(RefT::numBuffers(), RefT::offset_bits),
           _mapping()
     {
     }
@@ -30,11 +31,11 @@ public:
 
     EntryRef remap(EntryRef ref) const {
         if (ref.valid()) {
-            RefType internal_ref(ref);
-            if (!_compacting_buffer[internal_ref.bufferId()]) {
+            if (!_compacting_buffer.has(ref)) {
                 // No remapping for references to buffers not being compacted
                 return ref;
             } else {
+                RefType internal_ref(ref);
                 auto &inner_mapping = _mapping[internal_ref.bufferId()];
                 assert(internal_ref.unscaled_offset() < inner_mapping.size());
                 EntryRef mapped_ref = inner_mapping[internal_ref.unscaled_offset()];