diff options
author | Tor Egge <Tor.Egge@online.no> | 2023-01-11 13:17:09 +0100 |
---|---|---|
committer | Tor Egge <Tor.Egge@online.no> | 2023-01-11 13:17:09 +0100 |
commit | 8929e6c459e7dcbc1ed4ab102ce86545d5736a57 (patch) | |
tree | cde7ae590bc4e8b9f4867b4a299e2b57e73977ac /searchlib | |
parent | afb9f81ffd4befa141dfb22d64190a5725d001e7 (diff) |
Rename HnswGraph nested types:
LinkStore => LinkArrayStore
NodeStore => LevelArrayStore
NodeRef => LevelsRef
NodeRefVector => NodeVector
Diffstat (limited to 'searchlib')
14 files changed, 201 insertions, 201 deletions
diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index d8e189717c3..7fbfe19b42b 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -522,7 +522,7 @@ TYPED_TEST(HnswIndexTest, memory_is_reclaimed_when_doing_changes_to_graph) EXPECT_EQ(0, mem_2.allocatedBytesOnHold()); this->remove_document(2); - size_t node_ref_growth = 0; + size_t nodes_growth = 0; if constexpr (TestFixture::is_single) { this->expect_level_0(1, {3}); this->expect_empty_level_0(2); @@ -532,11 +532,11 @@ TYPED_TEST(HnswIndexTest, memory_is_reclaimed_when_doing_changes_to_graph) this->expect_level_0(1, {2}); this->expect_empty_level_0(3); this->expect_level_0(2, {1}); - node_ref_growth = sizeof(HnswNode); // Entry for nodeid 3 added when adding doc 2 + nodes_growth = sizeof(HnswNode); // Entry for nodeid 3 added when adding doc 2 } auto mem_3 = this->memory_usage(); // We end up in the same state as before document 2 was added and effectively use the same amount of memory. - EXPECT_EQ((mem_1.usedBytes() - mem_1.deadBytes() + node_ref_growth), (mem_3.usedBytes() - mem_3.deadBytes())); + EXPECT_EQ((mem_1.usedBytes() - mem_1.deadBytes() + nodes_growth), (mem_3.usedBytes() - mem_3.deadBytes())); EXPECT_EQ(0, mem_3.allocatedBytesOnHold()); } @@ -642,14 +642,14 @@ make_graph_helper(HnswIndex<type>& index) using LinkArrayRef = typename HnswGraph<type>::LinkArrayRef; auto& graph = index.get_graph(); ResultGraph result(graph.size()); - assert(!graph.get_node_ref(0).valid()); + assert(!graph.get_levels_ref(0).valid()); for (uint32_t doc_id = 1; doc_id < graph.size(); ++doc_id) { auto& node = result[doc_id]; - auto node_ref = graph.get_node_ref(doc_id); + auto levels_ref = graph.get_levels_ref(doc_id); if constexpr (std::is_same_v<std::remove_reference_t<decltype(node)>, uint32_t>) { - node = node_ref.ref(); + node = levels_ref.ref(); } else { - LevelArrayRef level_array(graph.get_level_array(node_ref)); + LevelArrayRef level_array(graph.get_level_array(levels_ref)); for (uint32_t level = 0; level < level_array.size(); ++level) { if constexpr (std::is_same_v<std::remove_reference_t<decltype(node)>, std::vector<uint32_t>>) { node.emplace_back(level_array[level].load_relaxed().ref()); diff --git a/searchlib/src/tests/tensor/hnsw_nodeid_mapping/hnsw_nodeid_mapping_test.cpp b/searchlib/src/tests/tensor/hnsw_nodeid_mapping/hnsw_nodeid_mapping_test.cpp index ac8b21d6136..3dba1aa44ac 100644 --- a/searchlib/src/tests/tensor/hnsw_nodeid_mapping/hnsw_nodeid_mapping_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_nodeid_mapping/hnsw_nodeid_mapping_test.cpp @@ -79,13 +79,13 @@ TEST_F(HnswNodeidMappingTest, free_ids_puts_nodeids_on_hold_list_and_then_free_l TEST_F(HnswNodeidMappingTest, on_load_populates_mapping) { std::vector<HnswNode> nodes(10); - nodes[1].ref().store_relaxed(EntryRef(1)); + nodes[1].levels_ref().store_relaxed(EntryRef(1)); nodes[1].store_docid(7); nodes[1].store_subspace(0); - nodes[2].ref().store_relaxed(EntryRef(2)); + nodes[2].levels_ref().store_relaxed(EntryRef(2)); nodes[2].store_docid(4); nodes[2].store_subspace(0); - nodes[7].ref().store_relaxed(EntryRef(3)); + nodes[7].levels_ref().store_relaxed(EntryRef(3)); nodes[7].store_docid(4); nodes[7].store_subspace(1); mapping.on_load(vespalib::ConstArrayRef(nodes.data(), nodes.size())); diff --git a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp index bf4abdd7cf8..21ee88a46fe 100644 --- a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp @@ -113,7 +113,7 @@ void modify(HnswGraph<type> &graph) { graph.set_link_array(4, 1, V{7}); graph.set_link_array(7, 1, V{4}); - graph.set_entry_node({4, graph.get_node_ref(4), 1}); + graph.set_entry_node({4, graph.get_levels_ref(4), 1}); } @@ -124,7 +124,7 @@ public: GraphType copy; void expect_empty_d(uint32_t nodeid) const { - EXPECT_FALSE(copy.acquire_node_ref(nodeid).valid()); + EXPECT_FALSE(copy.acquire_levels_ref(nodeid).valid()); } void expect_level_0(uint32_t nodeid, const V& exp_links) const { @@ -160,7 +160,7 @@ public: } void expect_docid_and_subspace(uint32_t nodeid) const { - auto& node = copy.node_refs.get_elem_ref(nodeid); + auto& node = copy.nodes.get_elem_ref(nodeid); EXPECT_EQ(fake_docid<GraphType::index_type>(nodeid), fake_get_docid(node, nodeid)); EXPECT_EQ(fake_subspace<GraphType::index_type>(nodeid), node.acquire_subspace()); } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp index 4be7a3e135c..e707b464049 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp @@ -9,13 +9,13 @@ namespace search::tensor { template <HnswIndexType type> HnswGraph<type>::HnswGraph() - : node_refs(), - node_refs_size(1u), - nodes(HnswIndex<type>::make_default_node_store_config(), {}), - links(HnswIndex<type>::make_default_link_store_config(), {}), + : nodes(), + nodes_size(1u), + levels_store(HnswIndex<type>::make_default_level_array_store_config(), {}), + links_store(HnswIndex<type>::make_default_link_array_store_config(), {}), entry_nodeid_and_level() { - node_refs.ensure_size(1, NodeType()); + nodes.ensure_size(1, NodeType()); EntryNode entry; set_entry_node(entry); } @@ -24,68 +24,68 @@ template <HnswIndexType type> HnswGraph<type>::~HnswGraph() = default; template <HnswIndexType type> -typename HnswGraph<type>::NodeRef +typename HnswGraph<type>::LevelsRef HnswGraph<type>::make_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, uint32_t num_levels) { - node_refs.ensure_size(nodeid + 1, NodeType()); + nodes.ensure_size(nodeid + 1, NodeType()); // A document cannot be added twice. - assert(!get_node_ref(nodeid).valid()); + assert(!get_levels_ref(nodeid).valid()); // Note: The level array instance lives as long as the document is present in the index. std::vector<AtomicEntryRef> levels(num_levels, AtomicEntryRef()); - auto node_ref = nodes.add(levels); - auto& node = node_refs[nodeid]; - node.ref().store_release(node_ref); + auto levels_ref = levels_store.add(levels); + auto& node = nodes[nodeid]; + node.levels_ref().store_release(levels_ref); node.store_docid(docid); node.store_subspace(subspace); - if (nodeid >= node_refs_size.load(std::memory_order_relaxed)) { - node_refs_size.store(nodeid + 1, std::memory_order_release); + if (nodeid >= nodes_size.load(std::memory_order_relaxed)) { + nodes_size.store(nodeid + 1, std::memory_order_release); } - return node_ref; + return levels_ref; } template <HnswIndexType type> void HnswGraph<type>::remove_node(uint32_t nodeid) { - auto node_ref = get_node_ref(nodeid); - assert(node_ref.valid()); - auto levels = nodes.get(node_ref); + auto levels_ref = get_levels_ref(nodeid); + assert(levels_ref.valid()); + auto levels = levels_store.get(levels_ref); vespalib::datastore::EntryRef invalid; - node_refs[nodeid].ref().store_release(invalid); + nodes[nodeid].levels_ref().store_release(invalid); // Ensure data referenced through the old ref can be recycled: - nodes.remove(node_ref); + levels_store.remove(levels_ref); for (size_t i = 0; i < levels.size(); ++i) { auto old_links_ref = levels[i].load_relaxed(); - links.remove(old_links_ref); + links_store.remove(old_links_ref); } - if (nodeid + 1 == node_refs_size.load(std::memory_order_relaxed)) { - trim_node_refs_size(); + if (nodeid + 1 == nodes_size.load(std::memory_order_relaxed)) { + trim_nodes_size(); } } template <HnswIndexType type> void -HnswGraph<type>::trim_node_refs_size() +HnswGraph<type>::trim_nodes_size() { - uint32_t check_nodeid = node_refs_size.load(std::memory_order_relaxed) - 1; - while (check_nodeid > 0u && !get_node_ref(check_nodeid).valid()) { + uint32_t check_nodeid = nodes_size.load(std::memory_order_relaxed) - 1; + while (check_nodeid > 0u && !get_levels_ref(check_nodeid).valid()) { --check_nodeid; } - node_refs_size.store(check_nodeid + 1, std::memory_order_release); + nodes_size.store(check_nodeid + 1, std::memory_order_release); } template <HnswIndexType type> void HnswGraph<type>::set_link_array(uint32_t nodeid, uint32_t level, const LinkArrayRef& new_links) { - auto new_links_ref = links.add(new_links); - auto node_ref = get_node_ref(nodeid); - assert(node_ref.valid()); - auto levels = nodes.get_writable(node_ref); + auto new_links_ref = links_store.add(new_links); + auto levels_ref = get_levels_ref(nodeid); + assert(levels_ref.valid()); + auto levels = levels_store.get_writable(levels_ref); assert(level < levels.size()); auto old_links_ref = levels[level].load_relaxed(); levels[level].store_release(new_links_ref); - links.remove(old_links_ref); + links_store.remove(old_links_ref); } template <HnswIndexType type> @@ -93,17 +93,17 @@ typename HnswGraph<type>::Histograms HnswGraph<type>::histograms() const { Histograms result; - size_t num_nodes = node_refs_size.load(std::memory_order_acquire); + size_t num_nodes = nodes_size.load(std::memory_order_acquire); for (size_t i = 0; i < num_nodes; ++i) { - auto node_ref = acquire_node_ref(i); - if (node_ref.valid()) { + auto levels_ref = acquire_levels_ref(i); + if (levels_ref.valid()) { uint32_t levels = 0; uint32_t l0links = 0; - auto level_array = nodes.get(node_ref); + auto level_array = levels_store.get(levels_ref); levels = level_array.size(); if (levels > 0) { auto links_ref = level_array[0].load_acquire(); - auto link_array = links.get(links_ref); + auto link_array = links_store.get(links_ref); l0links = link_array.size(); } while (result.level_histogram.size() <= levels) { @@ -125,7 +125,7 @@ HnswGraph<type>::set_entry_node(EntryNode node) { uint64_t value = node.level; value <<= 32; value |= node.nodeid; - if (node.node_ref.valid()) { + if (node.levels_ref.valid()) { assert(node.level >= 0); assert(node.nodeid > 0); } else { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h index 3fcb1f09eeb..f56d6e60145 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h @@ -31,74 +31,74 @@ struct HnswGraph { using NodeType = typename HnswIndexTraits<type>::NodeType; // Provides mapping from document id -> node reference. - // The reference is used to lookup the node data in NodeStore. - using NodeRefVector = vespalib::RcuVector<NodeType>; - using NodeRef = vespalib::datastore::EntryRef; + // The reference is used to lookup the node data in LevelArrayStore. + using NodeVector = vespalib::RcuVector<NodeType>; + using LevelsRef = vespalib::datastore::EntryRef; // This stores the level arrays for all nodes. // Each node consists of an array of levels (from level 0 to n) where each entry is a reference to the link array at that level. - using NodeStore = vespalib::datastore::ArrayStore<AtomicEntryRef, LevelArrayEntryRefType>; - using LevelArrayRef = NodeStore::ConstArrayRef; + using LevelArrayStore = vespalib::datastore::ArrayStore<AtomicEntryRef, LevelArrayEntryRefType>; + using LevelArrayRef = LevelArrayStore::ConstArrayRef; // This stores all link arrays. // A link array consists of the document ids of the nodes a particular node is linked to. - using LinkStore = vespalib::datastore::ArrayStore<uint32_t, LinkArrayEntryRefType>; - using LinkArrayRef = LinkStore::ConstArrayRef; + using LinkArrayStore = vespalib::datastore::ArrayStore<uint32_t, LinkArrayEntryRefType>; + using LinkArrayRef = LinkArrayStore::ConstArrayRef; - NodeRefVector node_refs; - std::atomic<uint32_t> node_refs_size; - NodeStore nodes; - LinkStore links; + NodeVector nodes; + std::atomic<uint32_t> nodes_size; + LevelArrayStore levels_store; + LinkArrayStore links_store; std::atomic<uint64_t> entry_nodeid_and_level; HnswGraph(); ~HnswGraph(); - NodeRef make_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, uint32_t num_levels); + LevelsRef make_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, uint32_t num_levels); void remove_node(uint32_t nodeid); - void trim_node_refs_size(); + void trim_nodes_size(); - NodeRef get_node_ref(uint32_t nodeid) const { - return node_refs.get_elem_ref(nodeid).ref().load_relaxed(); // Called from writer only + LevelsRef get_levels_ref(uint32_t nodeid) const { + return nodes.get_elem_ref(nodeid).levels_ref().load_relaxed(); // Called from writer only } - const NodeType& acquire_node_refs_elem_ref(uint32_t nodeid) const { - return node_refs.acquire_elem_ref(nodeid); + const NodeType& acquire_node(uint32_t nodeid) const { + return nodes.acquire_elem_ref(nodeid); } - NodeRef acquire_node_ref(uint32_t nodeid) const { - return node_refs.acquire_elem_ref(nodeid).ref().load_acquire(); + LevelsRef acquire_levels_ref(uint32_t nodeid) const { + return nodes.acquire_elem_ref(nodeid).levels_ref().load_acquire(); } - bool still_valid(uint32_t nodeid, NodeRef node_ref) const { - return node_ref.valid() && (acquire_node_ref(nodeid) == node_ref); + bool still_valid(uint32_t nodeid, LevelsRef levels_ref) const { + return levels_ref.valid() && (acquire_levels_ref(nodeid) == levels_ref); } - LevelArrayRef get_level_array(NodeRef node_ref) const { - if (node_ref.valid()) { - return nodes.get(node_ref); + LevelArrayRef get_level_array(LevelsRef levels_ref) const { + if (levels_ref.valid()) { + return levels_store.get(levels_ref); } return LevelArrayRef(); } LevelArrayRef get_level_array(uint32_t nodeid) const { - auto node_ref = get_node_ref(nodeid); - return get_level_array(node_ref); + auto levels_ref = get_levels_ref(nodeid); + return get_level_array(levels_ref); } LevelArrayRef acquire_level_array(uint32_t nodeid) const { - auto node_ref = acquire_node_ref(nodeid); - return get_level_array(node_ref); + auto levels_ref = acquire_levels_ref(nodeid); + return get_level_array(levels_ref); } LinkArrayRef get_link_array(LevelArrayRef levels, uint32_t level) const { if (level < levels.size()) { auto links_ref = levels[level].load_acquire(); if (links_ref.valid()) { - return links.get(links_ref); + return links_store.get(links_ref); } } return LinkArrayRef(); @@ -114,8 +114,8 @@ struct HnswGraph { return get_link_array(levels, level); } - LinkArrayRef get_link_array(NodeRef node_ref, uint32_t level) const { - auto levels = get_level_array(node_ref); + LinkArrayRef get_link_array(LevelsRef levels_ref, uint32_t level) const { + auto levels = get_level_array(levels_ref); return get_link_array(levels, level); } @@ -123,16 +123,16 @@ struct HnswGraph { struct EntryNode { uint32_t nodeid; - NodeRef node_ref; + LevelsRef levels_ref; int32_t level; EntryNode() : nodeid(0), // Note that nodeid 0 is reserved and never used - node_ref(), + levels_ref(), level(-1) {} - EntryNode(uint32_t nodeid_in, NodeRef node_ref_in, int32_t level_in) + EntryNode(uint32_t nodeid_in, LevelsRef levels_ref_in, int32_t level_in) : nodeid(nodeid_in), - node_ref(node_ref_in), + levels_ref(levels_ref_in), level(level_in) {} }; @@ -148,18 +148,18 @@ struct HnswGraph { while (true) { uint64_t value = get_entry_atomic(); entry.nodeid = (uint32_t)value; - entry.node_ref = acquire_node_ref(entry.nodeid); + entry.levels_ref = acquire_levels_ref(entry.nodeid); entry.level = (int32_t)(value >> 32); if ((entry.nodeid == 0) && (entry.level == -1) - && ! entry.node_ref.valid()) + && ! entry.levels_ref.valid()) { // invalid in every way return entry; } if ((entry.nodeid > 0) && (entry.level > -1) - && entry.node_ref.valid() + && entry.levels_ref.valid() && (get_entry_atomic() == value)) { // valid in every way @@ -168,7 +168,7 @@ struct HnswGraph { } } - size_t size() const { return node_refs_size.load(std::memory_order_acquire); } + size_t size() const { return nodes_size.load(std::memory_order_acquire); } struct Histograms { std::vector<uint32_t> level_histogram; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 32087e0e731..0aa4c18c494 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -63,9 +63,9 @@ bool operator< (const PairDist &a, const PairDist &b) { template <HnswIndexType type> vespalib::datastore::ArrayStoreConfig -HnswIndex<type>::make_default_node_store_config() +HnswIndex<type>::make_default_level_array_store_config() { - return NodeStore::optimizedConfigForHugePage(max_level_array_size, + return LevelArrayStore::optimizedConfigForHugePage(max_level_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, vespalib::alloc::MemoryAllocator::PAGE_SIZE, min_num_arrays_for_new_buffer, @@ -74,9 +74,9 @@ HnswIndex<type>::make_default_node_store_config() template <HnswIndexType type> vespalib::datastore::ArrayStoreConfig -HnswIndex<type>::make_default_link_store_config() +HnswIndex<type>::make_default_link_array_store_config() { - return LinkStore::optimizedConfigForHugePage(max_link_array_size, + return LinkArrayStore::optimizedConfigForHugePage(max_link_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, vespalib::alloc::MemoryAllocator::PAGE_SIZE, min_num_arrays_for_new_buffer, @@ -272,9 +272,9 @@ HnswIndex<type>::find_nearest_in_layer(const TypedCells& input, const HnswCandid bool keep_searching = true; while (keep_searching) { keep_searching = false; - for (uint32_t neighbor_nodeid : _graph.get_link_array(nearest.node_ref, level)) { - auto& neighbor_node = _graph.acquire_node_refs_elem_ref(neighbor_nodeid); - auto neighbor_ref = neighbor_node.ref().load_acquire(); + for (uint32_t neighbor_nodeid : _graph.get_link_array(nearest.levels_ref, level)) { + auto& neighbor_node = _graph.acquire_node(neighbor_nodeid); + auto neighbor_ref = neighbor_node.levels_ref().load_acquire(); uint32_t neighbor_docid = acquire_docid(neighbor_node, neighbor_nodeid); uint32_t neighbor_subspace = neighbor_node.acquire_subspace(); double dist = calc_distance(input, neighbor_docid, neighbor_subspace); @@ -317,12 +317,12 @@ HnswIndex<type>::search_layer_helper(const TypedCells& input, uint32_t neighbors break; } candidates.pop(); - for (uint32_t neighbor_nodeid : _graph.get_link_array(cand.node_ref, level)) { + for (uint32_t neighbor_nodeid : _graph.get_link_array(cand.levels_ref, level)) { if (neighbor_nodeid >= nodeid_limit) { continue; } - auto& neighbor_node = _graph.acquire_node_refs_elem_ref(neighbor_nodeid); - auto neighbor_ref = neighbor_node.ref().load_acquire(); + auto& neighbor_node = _graph.acquire_node(neighbor_nodeid); + auto neighbor_ref = neighbor_node.levels_ref().load_acquire(); if ((! neighbor_ref.valid()) || ! visited.try_mark(neighbor_nodeid)) { @@ -351,7 +351,7 @@ void HnswIndex<type>::search_layer(const TypedCells& input, uint32_t neighbors_to_find, BestNeighbors& best_neighbors, uint32_t level, const GlobalFilter *filter) const { - uint32_t nodeid_limit = _graph.node_refs_size.load(std::memory_order_acquire); + uint32_t nodeid_limit = _graph.nodes_size.load(std::memory_order_acquire); if (filter) { nodeid_limit = std::min(filter->size(), nodeid_limit); } @@ -427,8 +427,8 @@ HnswIndex<type>::internal_prepare_add_node(typename HnswIndex::PreparedAddDoc& o int search_level = entry.level; double entry_dist = calc_distance(input_vector, entry.nodeid); uint32_t entry_docid = get_docid(entry.nodeid); - // TODO: check if entry nodeid/node_ref is still valid here - HnswCandidate entry_point(entry.nodeid, entry_docid, entry.node_ref, entry_dist); + // TODO: check if entry nodeid/levels_ref is still valid here + HnswCandidate entry_point(entry.nodeid, entry_docid, entry.levels_ref, entry_dist); while (search_level > node_max_level) { entry_point = find_nearest_in_layer(input_vector, entry_point, search_level); --search_level; @@ -444,9 +444,9 @@ HnswIndex<type>::internal_prepare_add_node(typename HnswIndex::PreparedAddDoc& o auto& links = connections[search_level]; links.reserve(neighbors.used.size()); for (const auto & neighbor : neighbors.used) { - auto neighbor_levels = _graph.get_level_array(neighbor.node_ref); + auto neighbor_levels = _graph.get_level_array(neighbor.levels_ref); if (size_t(search_level) < neighbor_levels.size()) { - links.emplace_back(neighbor.nodeid, neighbor.node_ref); + links.emplace_back(neighbor.nodeid, neighbor.levels_ref); } else { LOG(warning, "in prepare_add(%u), selected neighbor %u is missing level %d (has %zu levels)", op.docid, neighbor.nodeid, search_level, neighbor_levels.size()); @@ -465,10 +465,10 @@ HnswIndex<type>::filter_valid_nodeids(uint32_t level, const typename PreparedAdd valid.reserve(neighbors.size()); for (const auto & neighbor : neighbors) { uint32_t nodeid = neighbor.first; - vespalib::datastore::EntryRef node_ref = neighbor.second; - if (_graph.still_valid(nodeid, node_ref)) { + vespalib::datastore::EntryRef levels_ref = neighbor.second; + if (_graph.still_valid(nodeid, levels_ref)) { assert(nodeid != self_nodeid); - auto levels = _graph.get_level_array(node_ref); + auto levels = _graph.get_level_array(levels_ref); if (level < levels.size()) { valid.push_back(nodeid); } @@ -495,13 +495,13 @@ void HnswIndex<type>::internal_complete_add_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, PreparedAddNode &prepared_node) { int32_t num_levels = prepared_node.connections.size(); - auto node_ref = _graph.make_node(nodeid, docid, subspace, num_levels); + auto levels_ref = _graph.make_node(nodeid, docid, subspace, num_levels); for (int level = 0; level < num_levels; ++level) { auto neighbors = filter_valid_nodeids(level, prepared_node.connections[level], nodeid); connect_new_node(nodeid, neighbors, level); } if (num_levels - 1 > get_entry_level()) { - _graph.set_entry_node({nodeid, node_ref, num_levels - 1}); + _graph.set_entry_node({nodeid, levels_ref, num_levels - 1}); } } @@ -511,7 +511,7 @@ HnswIndex<type>::prepare_add_document(uint32_t docid, VectorBundle vectors, vespalib::GenerationHandler::Guard read_guard) const { - uint32_t max_nodes = _graph.node_refs_size.load(std::memory_order_acquire); + uint32_t max_nodes = _graph.nodes_size.load(std::memory_order_acquire); if (max_nodes < _cfg.min_size_before_two_phase()) { // the first documents added will do all work in write thread // to ensure they are linked together: @@ -530,7 +530,7 @@ HnswIndex<type>::complete_add_document(uint32_t docid, std::unique_ptr<PrepareRe internal_complete_add(docid, *prepared); } else { // we expect this for the first documents added, so no warning for them - if (_graph.node_refs.size() > 1.25 * _cfg.min_size_before_two_phase()) { + if (_graph.nodes.size() > 1.25 * _cfg.min_size_before_two_phase()) { LOG(warning, "complete_add_document(%u) called with invalid prepare_result %s/%u", docid, (prepared ? "valid ptr" : "nullptr"), (prepared ? prepared->docid : 0u)); } @@ -576,8 +576,8 @@ HnswIndex<type>::remove_node(uint32_t nodeid) LinkArrayRef my_links = _graph.get_link_array(nodeid, level); for (uint32_t neighbor_id : my_links) { if (need_new_entrypoint) { - auto entry_node_ref = _graph.get_node_ref(neighbor_id); - _graph.set_entry_node({neighbor_id, entry_node_ref, level}); + auto entry_levels_ref = _graph.get_levels_ref(neighbor_id); + _graph.set_entry_node({neighbor_id, entry_levels_ref, level}); need_new_entrypoint = false; } remove_link_to(neighbor_id, nodeid, level); @@ -608,9 +608,9 @@ HnswIndex<type>::assign_generation(generation_t current_gen) { // Note: RcuVector transfers hold lists as part of reallocation based on current generation. // We need to set the next generation here, as it is incremented on a higher level right after this call. - _graph.node_refs.setGeneration(current_gen + 1); - _graph.nodes.assign_generation(current_gen); - _graph.links.assign_generation(current_gen); + _graph.nodes.setGeneration(current_gen + 1); + _graph.levels_store.assign_generation(current_gen); + _graph.links_store.assign_generation(current_gen); _id_mapping.assign_generation(current_gen); } @@ -618,9 +618,9 @@ template <HnswIndexType type> void HnswIndex<type>::reclaim_memory(generation_t oldest_used_gen) { - _graph.node_refs.reclaim_memory(oldest_used_gen); _graph.nodes.reclaim_memory(oldest_used_gen); - _graph.links.reclaim_memory(oldest_used_gen); + _graph.levels_store.reclaim_memory(oldest_used_gen); + _graph.links_store.reclaim_memory(oldest_used_gen); _id_mapping.reclaim_memory(oldest_used_gen); } @@ -628,15 +628,15 @@ template <HnswIndexType type> void HnswIndex<type>::compact_level_arrays(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - auto compacting_buffers = _graph.nodes.start_compact_worst_buffers(compaction_spec, compaction_strategy); - uint32_t nodeid_limit = _graph.node_refs.size(); + auto compacting_buffers = _graph.levels_store.start_compact_worst_buffers(compaction_spec, compaction_strategy); + uint32_t nodeid_limit = _graph.nodes.size(); auto filter = compacting_buffers->make_entry_ref_filter(); - vespalib::ArrayRef<NodeType> refs(&_graph.node_refs[0], nodeid_limit); + vespalib::ArrayRef<NodeType> refs(&_graph.nodes[0], nodeid_limit); for (auto& ref : refs) { - auto node_ref = ref.ref().load_relaxed(); - if (node_ref.valid() && filter.has(node_ref)) { - EntryRef new_node_ref = _graph.nodes.move_on_compact(node_ref); - ref.ref().store_release(new_node_ref); + auto levels_ref = ref.levels_ref().load_relaxed(); + if (levels_ref.valid() && filter.has(levels_ref)) { + EntryRef new_levels_ref = _graph.levels_store.move_on_compact(levels_ref); + ref.levels_ref().store_release(new_levels_ref); } } compacting_buffers->finish(); @@ -646,12 +646,12 @@ template <HnswIndexType type> void HnswIndex<type>::compact_link_arrays(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - auto context = _graph.links.compactWorst(compaction_spec, compaction_strategy); - uint32_t nodeid_limit = _graph.node_refs.size(); + auto context = _graph.links_store.compactWorst(compaction_spec, compaction_strategy); + uint32_t nodeid_limit = _graph.nodes.size(); for (uint32_t nodeid = 1; nodeid < nodeid_limit; ++nodeid) { - EntryRef level_ref = _graph.get_node_ref(nodeid); - if (level_ref.valid()) { - vespalib::ArrayRef<AtomicEntryRef> refs(_graph.nodes.get_writable(level_ref)); + EntryRef levels_ref = _graph.get_levels_ref(nodeid); + if (levels_ref.valid()) { + vespalib::ArrayRef<AtomicEntryRef> refs(_graph.levels_store.get_writable(levels_ref)); context->compact(refs); } } @@ -661,7 +661,7 @@ template <HnswIndexType type> bool HnswIndex<type>::consider_compact_level_arrays(const CompactionStrategy& compaction_strategy) { - if (!_graph.nodes.has_held_buffers() && _compaction_spec.level_arrays().compact()) { + if (!_graph.levels_store.has_held_buffers() && _compaction_spec.level_arrays().compact()) { compact_level_arrays(_compaction_spec.level_arrays(), compaction_strategy); return true; } @@ -672,7 +672,7 @@ template <HnswIndexType type> bool HnswIndex<type>::consider_compact_link_arrays(const CompactionStrategy& compaction_strategy) { - if (!_graph.links.has_held_buffers() && _compaction_spec.link_arrays().compact()) { + if (!_graph.links_store.has_held_buffers() && _compaction_spec.link_arrays().compact()) { compact_link_arrays(_compaction_spec.link_arrays(), compaction_strategy); return true; } @@ -698,12 +698,12 @@ vespalib::MemoryUsage HnswIndex<type>::update_stat(const CompactionStrategy& compaction_strategy) { vespalib::MemoryUsage result; - result.merge(_graph.node_refs.getMemoryUsage()); - auto level_arrays_memory_usage = _graph.nodes.getMemoryUsage(); - auto level_arrays_address_space_usage = _graph.nodes.addressSpaceUsage(); + result.merge(_graph.nodes.getMemoryUsage()); + auto level_arrays_memory_usage = _graph.levels_store.getMemoryUsage(); + auto level_arrays_address_space_usage = _graph.levels_store.addressSpaceUsage(); result.merge(level_arrays_memory_usage); - auto link_arrays_memory_usage = _graph.links.getMemoryUsage(); - auto link_arrays_address_space_usage = _graph.links.addressSpaceUsage(); + auto link_arrays_memory_usage = _graph.links_store.getMemoryUsage(); + auto link_arrays_address_space_usage = _graph.links_store.addressSpaceUsage(); _compaction_spec = HnswIndexCompactionSpec(compaction_strategy.should_compact(level_arrays_memory_usage, level_arrays_address_space_usage), compaction_strategy.should_compact(link_arrays_memory_usage, link_arrays_address_space_usage)); result.merge(link_arrays_memory_usage); @@ -715,9 +715,9 @@ vespalib::MemoryUsage HnswIndex<type>::memory_usage() const { vespalib::MemoryUsage result; - result.merge(_graph.node_refs.getMemoryUsage()); result.merge(_graph.nodes.getMemoryUsage()); - result.merge(_graph.links.getMemoryUsage()); + result.merge(_graph.levels_store.getMemoryUsage()); + result.merge(_graph.links_store.getMemoryUsage()); result.merge(_id_mapping.memory_usage()); return result; } @@ -726,8 +726,8 @@ template <HnswIndexType type> void HnswIndex<type>::populate_address_space_usage(search::AddressSpaceUsage& usage) const { - usage.set(AddressSpaceComponents::hnsw_node_store, _graph.nodes.addressSpaceUsage()); - usage.set(AddressSpaceComponents::hnsw_link_store, _graph.links.addressSpaceUsage()); + usage.set(AddressSpaceComponents::hnsw_node_store, _graph.levels_store.addressSpaceUsage()); + usage.set(AddressSpaceComponents::hnsw_link_store, _graph.links_store.addressSpaceUsage()); } template <HnswIndexType type> @@ -737,9 +737,9 @@ HnswIndex<type>::get_state(const vespalib::slime::Inserter& inserter) const auto& object = inserter.insertObject(); auto& memUsageObj = object.setObject("memory_usage"); StateExplorerUtils::memory_usage_to_slime(memory_usage(), memUsageObj.setObject("all")); - StateExplorerUtils::memory_usage_to_slime(_graph.node_refs.getMemoryUsage(), memUsageObj.setObject("node_refs")); StateExplorerUtils::memory_usage_to_slime(_graph.nodes.getMemoryUsage(), memUsageObj.setObject("nodes")); - StateExplorerUtils::memory_usage_to_slime(_graph.links.getMemoryUsage(), memUsageObj.setObject("links")); + StateExplorerUtils::memory_usage_to_slime(_graph.levels_store.getMemoryUsage(), memUsageObj.setObject("levels")); + StateExplorerUtils::memory_usage_to_slime(_graph.links_store.getMemoryUsage(), memUsageObj.setObject("links")); object.setLong("nodes", _graph.size()); auto& histogram_array = object.setArray("level_histogram"); auto& links_hst_array = object.setArray("level_0_links_histogram"); @@ -775,12 +775,12 @@ void HnswIndex<type>::shrink_lid_space(uint32_t doc_id_limit) { assert(doc_id_limit >= 1u); - assert(doc_id_limit >= _graph.node_refs_size.load(std::memory_order_relaxed)); - uint32_t old_doc_id_limit = _graph.node_refs.size(); + assert(doc_id_limit >= _graph.nodes_size.load(std::memory_order_relaxed)); + uint32_t old_doc_id_limit = _graph.nodes.size(); if (doc_id_limit >= old_doc_id_limit) { return; } - _graph.node_refs.shrink(doc_id_limit); + _graph.nodes.shrink(doc_id_limit); } template <HnswIndexType type> @@ -850,8 +850,8 @@ HnswIndex<type>::top_k_candidates(const TypedCells &vector, uint32_t k, const Gl int search_level = entry.level; double entry_dist = calc_distance(vector, entry.nodeid); uint32_t entry_docid = get_docid(entry.nodeid); - // TODO: check if entry docid/node_ref is still valid here - HnswCandidate entry_point(entry.nodeid, entry_docid, entry.node_ref, entry_dist); + // TODO: check if entry docid/levels_ref is still valid here + HnswCandidate entry_point(entry.nodeid, entry_docid, entry.levels_ref, entry_dist); while (search_level > 0) { entry_point = find_nearest_in_layer(vector, entry_point, search_level); --search_level; @@ -865,14 +865,14 @@ template <HnswIndexType type> HnswTestNode HnswIndex<type>::get_node(uint32_t nodeid) const { - auto node_ref = _graph.acquire_node_ref(nodeid); - if (!node_ref.valid()) { + auto levels_ref = _graph.acquire_levels_ref(nodeid); + if (!levels_ref.valid()) { return HnswTestNode(); } - auto levels = _graph.nodes.get(node_ref); + auto levels = _graph.levels_store.get(levels_ref); HnswTestNode::LevelArray result; for (const auto& links_ref : levels) { - auto links = _graph.links.get(links_ref.load_acquire()); + auto links = _graph.links_store.get(links_ref.load_acquire()); HnswTestNode::LinkArray result_links(links.begin(), links.end()); std::sort(result_links.begin(), result_links.end()); result.push_back(result_links); @@ -886,13 +886,13 @@ HnswIndex<type>::set_node(uint32_t nodeid, const HnswTestNode &node) { size_t num_levels = node.size(); assert(num_levels > 0); - auto node_ref = _graph.make_node(nodeid, nodeid, 0, num_levels); + auto levels_ref = _graph.make_node(nodeid, nodeid, 0, num_levels); for (size_t level = 0; level < num_levels; ++level) { connect_new_node(nodeid, node.level(level), level); } int max_level = num_levels - 1; if (get_entry_level() < max_level) { - _graph.set_entry_node({nodeid, node_ref, max_level}); + _graph.set_entry_node({nodeid, levels_ref, max_level}); } } @@ -903,12 +903,12 @@ HnswIndex<type>::check_link_symmetry() const bool all_sym = true; size_t nodeid_limit = _graph.size(); for (size_t nodeid = 0; nodeid < nodeid_limit; ++nodeid) { - auto node_ref = _graph.acquire_node_ref(nodeid); - if (node_ref.valid()) { - auto levels = _graph.nodes.get(node_ref); + auto levels_ref = _graph.acquire_levels_ref(nodeid); + if (levels_ref.valid()) { + auto levels = _graph.levels_store.get(levels_ref); uint32_t level = 0; for (const auto& links_ref : levels) { - auto links = _graph.links.get(links_ref.load_acquire()); + auto links = _graph.links_store.get(links_ref.load_acquire()); for (auto neighbor_nodeid : links) { auto neighbor_links = _graph.acquire_link_array(neighbor_nodeid, level); if (! has_link_to(neighbor_links, nodeid)) { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 32058e9ac44..60452db25a6 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -65,7 +65,7 @@ public: if constexpr (NodeType::identity_mapping) { return nodeid; } else { - return _graph.node_refs.acquire_elem_ref(nodeid).acquire_docid(); + return _graph.nodes.acquire_elem_ref(nodeid).acquire_docid(); } } @@ -76,9 +76,9 @@ protected: using GraphType = HnswGraph<type>; using NodeType = typename GraphType::NodeType; using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; - using NodeStore = typename GraphType::NodeStore; + using LevelArrayStore = typename GraphType::LevelArrayStore; - using LinkStore = typename GraphType::LinkStore; + using LinkArrayStore = typename GraphType::LinkArrayStore; using LinkArrayRef = typename GraphType::LinkArrayRef; using LinkArray = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; @@ -137,7 +137,7 @@ protected: if constexpr (NodeType::identity_mapping) { return _vectors.get_vector(nodeid, 0); } else { - auto& ref = _graph.node_refs.acquire_elem_ref(nodeid); + auto& ref = _graph.nodes.acquire_elem_ref(nodeid); uint32_t docid = ref.acquire_docid(); uint32_t subspace = ref.acquire_subspace(); return _vectors.get_vector(docid, subspace); @@ -259,8 +259,8 @@ public: GraphType& get_graph() { return _graph; } IdMapping& get_id_mapping() { return _id_mapping; } - static vespalib::datastore::ArrayStoreConfig make_default_node_store_config(); - static vespalib::datastore::ArrayStoreConfig make_default_link_store_config(); + static vespalib::datastore::ArrayStoreConfig make_default_level_array_store_config(); + static vespalib::datastore::ArrayStoreConfig make_default_link_array_store_config(); }; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp index de9cc760fec..279adbac559 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp @@ -61,12 +61,12 @@ HnswIndexLoader<ReaderType, type>::load_next() if (++_nodeid < _num_nodes) { return true; } else { - _graph.node_refs.ensure_size(std::max(_num_nodes, 1u)); - _graph.node_refs_size.store(std::max(_num_nodes, 1u), std::memory_order_release); - _graph.trim_node_refs_size(); - auto entry_node_ref = _graph.get_node_ref(_entry_nodeid); - _graph.set_entry_node({_entry_nodeid, entry_node_ref, _entry_level}); - _id_mapping.on_load(_graph.node_refs.make_read_view(_graph.node_refs.size())); + _graph.nodes.ensure_size(std::max(_num_nodes, 1u)); + _graph.nodes_size.store(std::max(_num_nodes, 1u), std::memory_order_release); + _graph.trim_nodes_size(); + auto entry_levels_ref = _graph.get_levels_ref(_entry_nodeid); + _graph.set_entry_node({_entry_nodeid, entry_levels_ref, _entry_level}); + _id_mapping.on_load(_graph.nodes.make_read_view(_graph.nodes.size())); _complete = true; return false; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp index 370e2ddf92a..70daeb34f5c 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp @@ -14,11 +14,11 @@ template <HnswIndexType type> size_t count_valid_link_arrays(const HnswGraph<type> & graph) { size_t count(0); - size_t num_nodes = graph.node_refs.get_size(); // Called from writer only + size_t num_nodes = graph.nodes.get_size(); // Called from writer only for (size_t i = 0; i < num_nodes; ++i) { - auto node_ref = graph.get_node_ref(i); - if (node_ref.valid()) { - count += graph.nodes.get(node_ref).size(); + auto levels_ref = graph.get_levels_ref(i); + if (levels_ref.valid()) { + count += graph.levels_store.get(levels_ref).size(); } } return count; @@ -42,23 +42,23 @@ HnswIndexSaver<type>::~HnswIndexSaver() = default; template <HnswIndexType type> HnswIndexSaver<type>::HnswIndexSaver(const HnswGraph<type> &graph) - : _graph_links(graph.links), _meta_data() + : _graph_links(graph.links_store), _meta_data() { auto entry = graph.get_entry_node(); _meta_data.entry_nodeid = entry.nodeid; _meta_data.entry_level = entry.level; - size_t num_nodes = graph.node_refs.get_size(); // Called from writer only + size_t num_nodes = graph.nodes.get_size(); // Called from writer only assert (num_nodes <= (std::numeric_limits<uint32_t>::max() - 1)); size_t link_array_count = count_valid_link_arrays(graph); assert (link_array_count <= std::numeric_limits<uint32_t>::max()); _meta_data.refs.reserve(link_array_count); _meta_data.nodes.reserve(num_nodes+1); for (size_t i = 0; i < num_nodes; ++i) { - auto& node = graph.node_refs.get_elem_ref(i); + auto& node = graph.nodes.get_elem_ref(i); _meta_data.nodes.emplace_back(_meta_data.refs.size(), node); - auto node_ref = node.ref().load_relaxed(); - if (node_ref.valid()) { - auto levels = graph.nodes.get(node_ref); + auto levels_ref = node.levels_ref().load_relaxed(); + if (levels_ref.valid()) { + auto levels = graph.levels_store.get(levels_ref); for (const auto& links_ref : levels) { _meta_data.refs.push_back(links_ref.load_relaxed()); } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h index 2884ee9b494..76a97274cc6 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h @@ -34,7 +34,7 @@ private: MetaData(); ~MetaData(); }; - const typename HnswGraph<type>::LinkStore &_graph_links; + const typename HnswGraph<type>::LinkArrayStore &_graph_links; MetaData _meta_data; }; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h index a88b805f198..95aace19e69 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h @@ -15,14 +15,14 @@ namespace search::tensor { */ struct HnswTraversalCandidate { uint32_t nodeid; - vespalib::datastore::EntryRef node_ref; + vespalib::datastore::EntryRef levels_ref; double distance; HnswTraversalCandidate(uint32_t nodeid_in, double distance_in) noexcept - : nodeid(nodeid_in), node_ref(), distance(distance_in) {} - HnswTraversalCandidate(uint32_t nodeid_in, vespalib::datastore::EntryRef node_ref_in, double distance_in) noexcept - : nodeid(nodeid_in), node_ref(node_ref_in), distance(distance_in) {} - HnswTraversalCandidate(uint32_t nodeid_in, uint32_t docid_in, vespalib::datastore::EntryRef node_ref_in, double distance_in) noexcept - : nodeid(nodeid_in), node_ref(node_ref_in), distance(distance_in) + : nodeid(nodeid_in), levels_ref(), distance(distance_in) {} + HnswTraversalCandidate(uint32_t nodeid_in, vespalib::datastore::EntryRef levels_ref_in, double distance_in) noexcept + : nodeid(nodeid_in), levels_ref(levels_ref_in), distance(distance_in) {} + HnswTraversalCandidate(uint32_t nodeid_in, uint32_t docid_in, vespalib::datastore::EntryRef levels_ref_in, double distance_in) noexcept + : nodeid(nodeid_in), levels_ref(levels_ref_in), distance(distance_in) { (void) docid_in; } @@ -35,8 +35,8 @@ struct HnswTraversalCandidate { struct HnswCandidate : public HnswTraversalCandidate { uint32_t docid; - HnswCandidate(uint32_t nodeid_in, uint32_t docid_in, vespalib::datastore::EntryRef node_ref_in, double distance_in) noexcept - : HnswTraversalCandidate(nodeid_in, docid_in, node_ref_in, distance_in), + HnswCandidate(uint32_t nodeid_in, uint32_t docid_in, vespalib::datastore::EntryRef levels_ref_in, double distance_in) noexcept + : HnswTraversalCandidate(nodeid_in, docid_in, levels_ref_in, distance_in), docid(docid_in) { } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h index 2e14f363bba..e5e2910fe81 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_node.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h @@ -14,19 +14,19 @@ class HnswNode { using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; using EntryRef = vespalib::datastore::EntryRef; - AtomicEntryRef _ref; + AtomicEntryRef _levels_ref; vespalib::datastore::AtomicValueWrapper<uint32_t> _docid; vespalib::datastore::AtomicValueWrapper<uint32_t> _subspace; public: HnswNode() noexcept - : _ref(), + : _levels_ref(), _docid(), _subspace() { } - AtomicEntryRef& ref() noexcept { return _ref; } - const AtomicEntryRef& ref() const noexcept { return _ref; } + AtomicEntryRef& levels_ref() noexcept { return _levels_ref; } + const AtomicEntryRef& levels_ref() const noexcept { return _levels_ref; } void store_docid(uint32_t docid) noexcept { _docid.store_release(docid); } void store_subspace(uint32_t subspace) noexcept { _subspace.store_release(subspace); } // Mapping from nodeid to docid and subspace. diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp index 9983bf6d97b..787abf4ad14 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp @@ -122,7 +122,7 @@ get_docid_limit(vespalib::ConstArrayRef<HnswNode> nodes) { uint32_t max_docid = 0; for (auto& node : nodes) { - if (node.ref().load_relaxed().valid()) { + if (node.levels_ref().load_relaxed().valid()) { max_docid = std::max(node.acquire_docid(), max_docid); } } @@ -135,7 +135,7 @@ make_subspaces_histogram(vespalib::ConstArrayRef<HnswNode> nodes, uint32_t docid // Make histogram std::vector<uint32_t> histogram(docid_limit); for (auto& node : nodes) { - if (node.ref().load_relaxed().valid()) { + if (node.levels_ref().load_relaxed().valid()) { auto docid = node.acquire_docid(); auto subspace = node.acquire_subspace(); auto &num_subspaces = histogram[docid]; @@ -171,7 +171,7 @@ HnswNodeidMapping::populate_docid_to_nodeids_mapping_and_free_list(vespalib::Con { uint32_t nodeid = 0; for (auto& node : nodes) { - if (node.ref().load_relaxed().valid()) { + if (node.levels_ref().load_relaxed().valid()) { auto docid = node.acquire_docid(); auto subspace = node.acquire_subspace(); auto nodeids = _nodeids.get_writable(_refs[docid]); @@ -207,7 +207,7 @@ HnswNodeidMapping::on_load(vespalib::ConstArrayRef<HnswNode> nodes) return; } // Check that reserved nodeid is not used - assert(!nodes[0].ref().load_relaxed().valid()); + assert(!nodes[0].levels_ref().load_relaxed().valid()); auto docid_limit = get_docid_limit(nodes); auto histogram = make_subspaces_histogram(nodes, docid_limit); // Allocate mapping from docid to nodeids allocate_docid_to_nodeids_mapping(std::move(histogram)); diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h index 9740cab5c31..4d775f61a8d 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h @@ -13,15 +13,15 @@ class HnswSimpleNode { using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; using EntryRef = vespalib::datastore::EntryRef; - AtomicEntryRef _ref; + AtomicEntryRef _levels_ref; public: HnswSimpleNode() - : _ref() + : _levels_ref() { } - AtomicEntryRef& ref() noexcept { return _ref; } - const AtomicEntryRef& ref() const noexcept { return _ref; } + AtomicEntryRef& levels_ref() noexcept { return _levels_ref; } + const AtomicEntryRef& levels_ref() const noexcept { return _levels_ref; } void store_docid(uint32_t docid) noexcept { (void) docid; } void store_subspace(uint32_t subspace) noexcept { (void) subspace; } // Mapping from nodeid to docid and subspace. |