Merge pull request #31547 from vespa-engine/balder/deinline-large-functions

Deinline some large and expensive functions. Also add noexcept.

4 files changed, 37 insertions, 80 deletions

diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h
index a1a9e9632be..88b7681f23c 100644
--- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h
@@ -82,7 +82,7 @@ struct HnswGraph {
         if (levels_ref.valid()) {
             return levels_store.get(levels_ref);
         }
-        return LevelArrayRef();
+        return {};
     }
 
     LevelArrayRef get_level_array(uint32_t nodeid) const {
@@ -102,7 +102,7 @@ struct HnswGraph {
                 return links_store.get(links_ref);
             }
         }
-        return LinkArrayRef();
+        return {};
     }
 
     LinkArrayRef get_link_array(uint32_t nodeid, uint32_t level) const {
@@ -126,12 +126,12 @@ struct HnswGraph {
         uint32_t nodeid;
         LevelsRef levels_ref;
         int32_t level;
-        EntryNode()
+        EntryNode() noexcept
           : nodeid(0), // Note that nodeid 0 is reserved and never used
             levels_ref(),
             level(-1)
         {}
-        EntryNode(uint32_t nodeid_in, LevelsRef levels_ref_in, int32_t level_in)
+        EntryNode(uint32_t nodeid_in, LevelsRef levels_ref_in, int32_t level_in) noexcept
           : nodeid(nodeid_in),
             levels_ref(levels_ref_in),
             level(level_in)
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
index b542c422f50..f9ab5d705c9 100644
--- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
@@ -155,8 +155,8 @@ HnswIndex<type>::make_default_level_array_store_config()
                                                        vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE,
                                                        vespalib::alloc::MemoryAllocator::PAGE_SIZE,
                                                        ArrayStoreConfig::default_max_buffer_size,
-                                                 min_num_arrays_for_new_buffer,
-                                                 alloc_grow_factor).enable_free_lists(true);
+                                                       min_num_arrays_for_new_buffer,
+                                                       alloc_grow_factor).enable_free_lists(true);
 }
 
 template <HnswIndexType type>
@@ -373,9 +373,7 @@ HnswIndex<type>::estimate_visited_nodes(uint32_t level, uint32_t nodeid_limit, u
 
 template <HnswIndexType type>
 HnswCandidate
-HnswIndex<type>::find_nearest_in_layer(
-        const BoundDistanceFunction &df,
-        const HnswCandidate& entry_point, uint32_t level) const
+HnswIndex<type>::find_nearest_in_layer(const BoundDistanceFunction &df, const HnswCandidate& entry_point, uint32_t level) const
 {
     HnswCandidate nearest = entry_point;
     bool keep_searching = true;
@@ -401,12 +399,10 @@ HnswIndex<type>::find_nearest_in_layer(
 template <HnswIndexType type>
 template <class VisitedTracker, class BestNeighbors>
 void
-HnswIndex<type>::search_layer_helper(
-        const BoundDistanceFunction &df,
-        uint32_t neighbors_to_find,
-        BestNeighbors& best_neighbors, uint32_t level, const GlobalFilter *filter,
-        uint32_t nodeid_limit, const vespalib::Doom* const doom,
-        uint32_t estimated_visited_nodes) const
+HnswIndex<type>::search_layer_helper(const BoundDistanceFunction &df, uint32_t neighbors_to_find,
+                                     BestNeighbors& best_neighbors, uint32_t level, const GlobalFilter *filter,
+                                     uint32_t nodeid_limit, const vespalib::Doom* const doom,
+                                     uint32_t estimated_visited_nodes) const
 {
     NearestPriQ candidates;
     GlobalFilterWrapper<type> filter_wrapper(filter);
@@ -471,11 +467,8 @@ HnswIndex<type>::search_layer_helper(
 template <HnswIndexType type>
 template <class BestNeighbors>
 void
-HnswIndex<type>::search_layer(
-        const BoundDistanceFunction &df,
-        uint32_t neighbors_to_find,
-        BestNeighbors& best_neighbors, uint32_t level,
-        const vespalib::Doom* const doom, const GlobalFilter *filter) const
+HnswIndex<type>::search_layer(const BoundDistanceFunction &df, uint32_t neighbors_to_find, BestNeighbors& best_neighbors,
+                              uint32_t level, const vespalib::Doom* const doom, const GlobalFilter *filter) const
 {
     uint32_t nodeid_limit = _graph.nodes_size.load(std::memory_order_acquire);
     uint32_t estimated_visited_nodes = estimate_visited_nodes(level, nodeid_limit, neighbors_to_find, filter);
@@ -488,7 +481,7 @@ HnswIndex<type>::search_layer(
 
 template <HnswIndexType type>
 HnswIndex<type>::HnswIndex(const DocVectorAccess& vectors, DistanceFunctionFactory::UP distance_ff,
-                     RandomLevelGenerator::UP level_generator, const HnswIndexConfig& cfg)
+                           RandomLevelGenerator::UP level_generator, const HnswIndexConfig& cfg)
     : _graph(),
       _vectors(vectors),
       _distance_ff(std::move(distance_ff)),
@@ -633,9 +626,7 @@ HnswIndex<type>::internal_complete_add_node(uint32_t nodeid, uint32_t docid, uin
 
 template <HnswIndexType type>
 std::unique_ptr<PrepareResult>
-HnswIndex<type>::prepare_add_document(uint32_t docid,
-                                VectorBundle vectors,
-                                vespalib::GenerationHandler::Guard read_guard) const
+HnswIndex<type>::prepare_add_document(uint32_t docid, VectorBundle vectors, vespalib::GenerationHandler::Guard read_guard) const
 {
     uint32_t active_nodes = _graph.get_active_nodes();
     if (active_nodes < _cfg.min_size_before_two_phase()) {
@@ -930,12 +921,8 @@ struct NeighborsByDocId {
 
 template <HnswIndexType type>
 std::vector<NearestNeighborIndex::Neighbor>
-HnswIndex<type>::top_k_by_docid(
-        uint32_t k,
-        const BoundDistanceFunction &df,
-        const GlobalFilter *filter, uint32_t explore_k,
-        const vespalib::Doom& doom,
-        double distance_threshold) const
+HnswIndex<type>::top_k_by_docid(uint32_t k, const BoundDistanceFunction &df, const GlobalFilter *filter,
+                                uint32_t explore_k, const vespalib::Doom& doom, double distance_threshold) const
 {
     SearchBestNeighbors candidates = top_k_candidates(df, std::max(k, explore_k), filter, doom);
     auto result = candidates.get_neighbors(k, distance_threshold);
@@ -945,34 +932,23 @@ HnswIndex<type>::top_k_by_docid(
 
 template <HnswIndexType type>
 std::vector<NearestNeighborIndex::Neighbor>
-HnswIndex<type>::find_top_k(
-        uint32_t k,
-        const BoundDistanceFunction &df,
-        uint32_t explore_k,
-        const vespalib::Doom& doom,
-        double distance_threshold) const
+HnswIndex<type>::find_top_k(uint32_t k, const BoundDistanceFunction &df, uint32_t explore_k,
+                            const vespalib::Doom& doom, double distance_threshold) const
 {
     return top_k_by_docid(k, df, nullptr, explore_k, doom, distance_threshold);
 }
 
 template <HnswIndexType type>
 std::vector<NearestNeighborIndex::Neighbor>
-HnswIndex<type>::find_top_k_with_filter(
-        uint32_t k,
-        const BoundDistanceFunction &df,
-        const GlobalFilter &filter, uint32_t explore_k,
-        const vespalib::Doom& doom,
-        double distance_threshold) const
+HnswIndex<type>::find_top_k_with_filter(uint32_t k, const BoundDistanceFunction &df, const GlobalFilter &filter,
+                                        uint32_t explore_k, const vespalib::Doom& doom, double distance_threshold) const
 {
     return top_k_by_docid(k, df, &filter, explore_k, doom, distance_threshold);
 }
 
 template <HnswIndexType type>
 typename HnswIndex<type>::SearchBestNeighbors
-HnswIndex<type>::top_k_candidates(
-        const BoundDistanceFunction &df,
-        uint32_t k, const GlobalFilter *filter,
-        const vespalib::Doom& doom) const
+HnswIndex<type>::top_k_candidates(const BoundDistanceFunction &df, uint32_t k, const GlobalFilter *filter, const vespalib::Doom& doom) const
 {
     SearchBestNeighbors best_neighbors;
     auto entry = _graph.get_entry_node();
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h
index 4d4440c1bcb..8c74f1e5264 100644
--- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h
@@ -171,24 +171,19 @@ protected:
     /**
      * Performs a greedy search in the given layer to find the candidate that is nearest the input vector.
      */
-    HnswCandidate find_nearest_in_layer(const BoundDistanceFunction &df, const HnswCandidate& entry_point, uint32_t level) const;
+    HnswCandidate find_nearest_in_layer(const BoundDistanceFunction &df, const HnswCandidate& entry_point, uint32_t level) const __attribute__((noinline));
     template <class VisitedTracker, class BestNeighbors>
     void search_layer_helper(const BoundDistanceFunction &df, uint32_t neighbors_to_find, BestNeighbors& best_neighbors,
-                             uint32_t level, const GlobalFilter *filter,
-                             uint32_t nodeid_limit,
-                             const vespalib::Doom* const doom,
-                             uint32_t estimated_visited_nodes) const;
+                             uint32_t level, const GlobalFilter *filter, uint32_t nodeid_limit,
+                             const vespalib::Doom* const doom, uint32_t estimated_visited_nodes) const __attribute__((noinline));
     template <class BestNeighbors>
     void search_layer(const BoundDistanceFunction &df, uint32_t neighbors_to_find, BestNeighbors& best_neighbors,
-                      uint32_t level, const vespalib::Doom* const doom,
-                      const GlobalFilter *filter = nullptr) const;
-    std::vector<Neighbor> top_k_by_docid(uint32_t k, const BoundDistanceFunction &df,
-                                         const GlobalFilter *filter, uint32_t explore_k,
-                                         const vespalib::Doom& doom,
-                                         double distance_threshold) const;
+                      uint32_t level, const vespalib::Doom* const doom, const GlobalFilter *filter = nullptr) const;
+    std::vector<Neighbor> top_k_by_docid(uint32_t k, const BoundDistanceFunction &df, const GlobalFilter *filter,
+                                         uint32_t explore_k, const vespalib::Doom& doom, double distance_threshold) const;
 
     internal::PreparedAddDoc internal_prepare_add(uint32_t docid, VectorBundle input_vectors,
-                                        vespalib::GenerationHandler::Guard read_guard) const;
+                                                  vespalib::GenerationHandler::Guard read_guard) const;
     void internal_prepare_add_node(internal::PreparedAddDoc& op, TypedCells input_vector, const typename GraphType::EntryNode& entry) const;
     LinkArray filter_valid_nodeids(uint32_t level, const internal::PreparedAddNode::Links &neighbors, uint32_t self_nodeid);
     void internal_complete_add(uint32_t docid, internal::PreparedAddDoc &op);
@@ -205,8 +200,7 @@ public:
 
     // Implements NearestNeighborIndex
     void add_document(uint32_t docid) override;
-    std::unique_ptr<PrepareResult> prepare_add_document(uint32_t docid,
-                                                        VectorBundle vectors,
+    std::unique_ptr<PrepareResult> prepare_add_document(uint32_t docid, VectorBundle vectors,
                                                         vespalib::GenerationHandler::Guard read_guard) const override;
     void complete_add_document(uint32_t docid, std::unique_ptr<PrepareResult> prepare_result) override;
     void remove_node(uint32_t nodeid);
@@ -225,26 +219,16 @@ public:
     std::unique_ptr<NearestNeighborIndexSaver> make_saver(vespalib::GenericHeader& header) const override;
     std::unique_ptr<NearestNeighborIndexLoader> make_loader(FastOS_FileInterface& file, const vespalib::GenericHeader& header) override;
 
-    std::vector<Neighbor> find_top_k(
-            uint32_t k,
-            const BoundDistanceFunction &df,
-            uint32_t explore_k,
-            const vespalib::Doom& doom,
-            double distance_threshold) const override;
+    std::vector<Neighbor> find_top_k(uint32_t k, const BoundDistanceFunction &df, uint32_t explore_k,
+                                     const vespalib::Doom& doom, double distance_threshold) const override;
 
-    std::vector<Neighbor> find_top_k_with_filter(
-            uint32_t k,
-            const BoundDistanceFunction &df,
-            const GlobalFilter &filter, uint32_t explore_k,
-            const vespalib::Doom& doom,
-            double distance_threshold) const override;
+    std::vector<Neighbor> find_top_k_with_filter(uint32_t k, const BoundDistanceFunction &df, const GlobalFilter &filter,
+                                                 uint32_t explore_k, const vespalib::Doom& doom, double distance_threshold) const override;
 
     DistanceFunctionFactory &distance_function_factory() const override { return *_distance_ff; }
 
-    SearchBestNeighbors top_k_candidates(
-            const BoundDistanceFunction &df,
-            uint32_t k, const GlobalFilter *filter,
-            const vespalib::Doom& doom) const;
+    SearchBestNeighbors top_k_candidates(const BoundDistanceFunction &df, uint32_t k, const GlobalFilter *filter,
+                                         const vespalib::Doom& doom) const;
 
     uint32_t get_entry_nodeid() const { return _graph.get_entry_node().nodeid; }
     int32_t get_entry_level() const { return _graph.get_entry_node().level; }
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h
index b8189090079..33a9fa2503f 100644
--- a/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h
@@ -16,10 +16,7 @@ class HnswSimpleNode {
     AtomicEntryRef _levels_ref;
 
 public:
-    HnswSimpleNode()
-        : _levels_ref()
-    {
-    }
+    HnswSimpleNode() noexcept : _levels_ref() { }
     AtomicEntryRef& levels_ref() noexcept { return _levels_ref; }
     const AtomicEntryRef& levels_ref() const noexcept { return _levels_ref; }
     void store_docid(uint32_t docid) noexcept { (void) docid; }