diff options
Diffstat (limited to 'searchlib/src')
12 files changed, 67 insertions, 2 deletions
diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 084bdf90a14..db76d8bad25 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -34,6 +34,7 @@ LOG_SETUP("tensorattribute_test"); using document::WrongTensorTypeException; +using search::AddressSpaceUsage; using search::AttributeGuard; using search::AttributeVector; using search::CompactionStrategy; @@ -210,6 +211,7 @@ public: ++_memory_usage_cnt; return vespalib::MemoryUsage(); } + void populate_address_space_usage(AddressSpaceUsage&) const override {} void get_state(const vespalib::slime::Inserter&) const override {} void shrink_lid_space(uint32_t) override { } std::unique_ptr<NearestNeighborIndexSaver> make_saver() const override { @@ -517,6 +519,7 @@ struct Fixture { void testTensorTypeFileHeaderTag(); void testEmptyTensor(); void testOnHoldAccounting(); + void test_populate_address_space_usage(); }; @@ -708,6 +711,15 @@ Fixture::testOnHoldAccounting() EXPECT_EQUAL(0u, getStatus().getOnHold()); } +void +Fixture::test_populate_address_space_usage() +{ + search::AddressSpaceUsage usage = _attr->getAddressSpaceUsage(); + const auto& all = usage.get_all(); + EXPECT_EQUAL(1u, all.size()); + EXPECT_EQUAL(1u, all.count("tensor-store")); +} + template <class MakeFixture> void testAll(MakeFixture &&f) { @@ -718,6 +730,7 @@ void testAll(MakeFixture &&f) TEST_DO(f()->testTensorTypeFileHeaderTag()); TEST_DO(f()->testEmptyTensor()); TEST_DO(f()->testOnHoldAccounting()); + TEST_DO(f()->test_populate_address_space_usage()); } TEST("Test sparse tensors with generic tensor attribute") @@ -790,6 +803,16 @@ TEST_F("Hnsw index is integrated in dense tensor attribute and can be saved and expect_level_0(1, index_b.get_node(2)); } +TEST_F("Populates address space usage", DenseTensorAttributeHnswIndex) +{ + search::AddressSpaceUsage usage = f._attr->getAddressSpaceUsage(); + const auto& all = usage.get_all(); + EXPECT_EQUAL(3u, all.size()); + EXPECT_EQUAL(1u, all.count("tensor-store")); + EXPECT_EQUAL(1u, all.count("hnsw-node-store")); + EXPECT_EQUAL(1u, all.count("hnsw-link-store")); +} + class DenseTensorAttributeMockIndex : public Fixture { public: diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp index 74075367da9..1e6ad8f4ef7 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp @@ -17,5 +17,8 @@ AddressSpace AddressSpaceComponents::default_multi_value_usage() { const vespalib::string AddressSpaceComponents::enum_store = "enum-store"; const vespalib::string AddressSpaceComponents::multi_value = "multi-value"; +const vespalib::string AddressSpaceComponents::tensor_store = "tensor-store"; +const vespalib::string AddressSpaceComponents::hnsw_node_store = "hnsw-node-store"; +const vespalib::string AddressSpaceComponents::hnsw_link_store = "hnsw-link-store"; } diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.h b/searchlib/src/vespa/searchlib/attribute/address_space_components.h index d52fe67abbd..213d86fa803 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_components.h +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.h @@ -16,6 +16,9 @@ public: static vespalib::AddressSpace default_multi_value_usage(); static const vespalib::string enum_store; static const vespalib::string multi_value; + static const vespalib::string tensor_store; + static const vespalib::string hnsw_node_store; + static const vespalib::string hnsw_link_store; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 1639f5f8113..7c05699b8e1 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -125,6 +125,15 @@ DenseTensorAttribute::memory_usage() const return result; } +void +DenseTensorAttribute::populate_address_space_usage(AddressSpaceUsage& usage) const +{ + TensorAttribute::populate_address_space_usage(usage); + if (_index) { + _index->populate_address_space_usage(usage); + } +} + DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, const NearestNeighborIndexFactory& index_factory) : TensorAttribute(baseFileName, cfg, _denseTensorStore), diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h index 8899b1e4bd1..1f5d6849dfd 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -25,6 +25,7 @@ private: void consider_remove_from_index(DocId docid); vespalib::MemoryUsage update_stat() override; vespalib::MemoryUsage memory_usage() const override; + void populate_address_space_usage(AddressSpaceUsage& usage) const override; class ThreadedLoader; class ForegroundLoader; public: diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h index 57826088ca5..ae36e47cb42 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h @@ -10,7 +10,7 @@ namespace search::tensor { /** - * Stroage of a hierarchical navigable small world graph (HNSW) + * Storage of a hierarchical navigable small world graph (HNSW) * that is used for approximate K-nearest neighbor search. */ struct HnswGraph { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index ca5f522457a..51c12d8af6d 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -6,6 +6,8 @@ #include "hnsw_index_saver.h" #include "random_level_generator.h" #include <vespa/searchcommon/common/compaction_strategy.h> +#include <vespa/searchlib/attribute/address_space_components.h> +#include <vespa/searchlib/attribute/address_space_usage.h> #include <vespa/searchlib/util/state_explorer_utils.h> #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> @@ -20,6 +22,7 @@ LOG_SETUP(".searchlib.tensor.hnsw_index"); namespace search::tensor { +using search::AddressSpaceComponents; using search::StateExplorerUtils; using vespalib::datastore::EntryRef; @@ -579,6 +582,13 @@ HnswIndex::memory_usage() const } void +HnswIndex::populate_address_space_usage(search::AddressSpaceUsage& usage) const +{ + usage.set(AddressSpaceComponents::hnsw_node_store, _graph.nodes.addressSpaceUsage()); + usage.set(AddressSpaceComponents::hnsw_link_store, _graph.links.addressSpaceUsage()); +} + +void HnswIndex::get_state(const vespalib::slime::Inserter& inserter) const { auto& object = inserter.insertObject(); diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 6f6d213d6cc..eb874193ae9 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -172,6 +172,7 @@ public: bool consider_compact(const CompactionStrategy& compaction_strategy) override; vespalib::MemoryUsage update_stat() override; vespalib::MemoryUsage memory_usage() const override; + void populate_address_space_usage(search::AddressSpaceUsage& usage) const override; void get_state(const vespalib::slime::Inserter& inserter) const override; void shrink_lid_space(uint32_t doc_id_limit) override; diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h index 0122738e173..b8f30a53ddf 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -15,6 +15,7 @@ namespace vespalib::slime { struct Inserter; } namespace search::fileutil { class LoadedBuffer; } namespace search { +class AddressSpaceUsage; class BitVector; class CompactionStrategy; } @@ -65,6 +66,7 @@ public: virtual bool consider_compact(const CompactionStrategy& compaction_strategy) = 0; virtual vespalib::MemoryUsage update_stat() = 0; virtual vespalib::MemoryUsage memory_usage() const = 0; + virtual void populate_address_space_usage(search::AddressSpaceUsage& usage) const = 0; virtual void get_state(const vespalib::slime::Inserter& inserter) const = 0; virtual void shrink_lid_space(uint32_t doc_id_limit) = 0; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 5044a989203..d61fc544839 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -3,6 +3,7 @@ #include "tensor_attribute.h" #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/tensor_data_type.h> +#include <vespa/searchlib/attribute/address_space_components.h> #include <vespa/searchlib/util/state_explorer_utils.h> #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> @@ -15,11 +16,12 @@ using document::TensorDataType; using document::TensorUpdate; using document::WrongTensorTypeException; +using search::AddressSpaceComponents; +using search::StateExplorerUtils; using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::TensorSpec; using vespalib::eval::Value; using vespalib::eval::ValueType; -using search::StateExplorerUtils; namespace search::tensor { @@ -189,6 +191,12 @@ TensorAttribute::populate_state(vespalib::slime::Cursor& object) const object.setObject("tensor_store").setObject("memory_usage")); } +void +TensorAttribute::populate_address_space_usage(AddressSpaceUsage& usage) const +{ + usage.set(AddressSpaceComponents::tensor_store, _tensorStore.get_address_space_usage()); +} + vespalib::eval::Value::UP TensorAttribute::getEmptyTensor() const { diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index ea0f7391120..115a8d34058 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -35,6 +35,7 @@ protected: virtual vespalib::MemoryUsage update_stat(); virtual vespalib::MemoryUsage memory_usage() const; void populate_state(vespalib::slime::Cursor& object) const; + void populate_address_space_usage(AddressSpaceUsage& usage) const override; public: DECLARE_IDENTIFIABLE_ABSTRACT(TensorAttribute); diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_store.h b/searchlib/src/vespa/searchlib/tensor/tensor_store.h index 4c6314bfe3e..89451ac72b9 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_store.h @@ -53,6 +53,10 @@ public: return _store.getMemoryUsage(); } + vespalib::AddressSpace get_address_space_usage() const { + return _store.getAddressSpaceUsage(); + } + uint32_t startCompactWorstBuffer() { return _store.startCompactWorstBuffer(_typeId); } |