diff options
Diffstat (limited to 'searchlib/src/tests')
11 files changed, 262 insertions, 37 deletions
diff --git a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp index 24919fb2341..04d2dfe4d52 100644 --- a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp @@ -491,7 +491,7 @@ BitVectorTest::test(BasicType bt, v->asDocumentWeightAttribute(); if (dwa != NULL) { search::IDocumentWeightAttribute::LookupResult lres = - dwa->lookup(getSearchStr<VectorType>()); + dwa->lookup(getSearchStr<VectorType>(), dwa->get_dictionary_snapshot()); typedef search::queryeval::DocumentWeightSearchIterator DWSI; typedef search::queryeval::SearchIterator SI; TermFieldMatchData md; diff --git a/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp b/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp index cf1506a9118..d8a1d03f1a8 100644 --- a/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp +++ b/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributeguard.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/attributememorysavetarget.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/attributevector.hpp> @@ -22,6 +23,7 @@ #include <vespa/searchlib/test/searchiteratorverifier.h> #include <vespa/searchlib/util/randomgenerator.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/test/insertion_operators.h> #include <vespa/log/log.h> LOG_SETUP("document_weight_iterator_test"); @@ -124,17 +126,17 @@ void verify_invalid_lookup(IDocumentWeightAttribute::LookupResult result) { } TEST_F("require that integer lookup works correctly", LongFixture) { - verify_valid_lookup(f1.api->lookup("111")); - verify_invalid_lookup(f1.api->lookup("222")); + verify_valid_lookup(f1.api->lookup("111", f1.api->get_dictionary_snapshot())); + verify_invalid_lookup(f1.api->lookup("222", f1.api->get_dictionary_snapshot())); } TEST_F("require string lookup works correctly", StringFixture) { - verify_valid_lookup(f1.api->lookup("foo")); - verify_invalid_lookup(f1.api->lookup("bar")); + verify_valid_lookup(f1.api->lookup("foo", f1.api->get_dictionary_snapshot())); + verify_invalid_lookup(f1.api->lookup("bar", f1.api->get_dictionary_snapshot())); } void verify_posting(const IDocumentWeightAttribute &api, const char *term) { - auto result = api.lookup(term); + auto result = api.lookup(term, api.get_dictionary_snapshot()); ASSERT_TRUE(result.posting_idx.valid()); std::vector<DocumentWeightIterator> itr_store; api.create(result.posting_idx, itr_store); @@ -168,6 +170,53 @@ TEST_F("require that string iterators are created correctly", StringFixture) { verify_posting(*f1.api, "foo"); } +TEST_F("require that dictionary snapshot works", LongFixture) +{ + auto read_guard = f1.attr->makeReadGuard(false); + auto dictionary_snapshot = f1.api->get_dictionary_snapshot(); + auto lookup1 = f1.api->lookup("111", dictionary_snapshot); + EXPECT_TRUE(lookup1.enum_idx.valid()); + f1.attr->clearDoc(1); + f1.attr->clearDoc(5); + f1.attr->clearDoc(7); + f1.attr->commit(); + auto lookup2 = f1.api->lookup("111", f1.api->get_dictionary_snapshot()); + EXPECT_FALSE(lookup2.enum_idx.valid()); + auto lookup3 = f1.api->lookup("111", dictionary_snapshot); + EXPECT_TRUE(lookup3.enum_idx.valid()); + EXPECT_EQUAL(lookup1.enum_idx.ref(), lookup3.enum_idx.ref()); +} + +TEST_F("require that collect_folded works for string", StringFixture) +{ + StringAttribute *attr = static_cast<StringAttribute *>(f1.attr.get()); + set_doc(attr, 2, "bar", 30); + attr->commit(); + set_doc(attr, 3, "FOO", 30); + attr->commit(); + auto dictionary_snapshot = f1.api->get_dictionary_snapshot(); + auto lookup1 = f1.api->lookup("foo", dictionary_snapshot); + std::vector<vespalib::string> folded; + std::function<void(vespalib::datastore::EntryRef)> save_folded = [&folded,attr](vespalib::datastore::EntryRef enum_idx) { folded.emplace_back(attr->getFromEnum(enum_idx.ref())); }; + f1.api->collect_folded(lookup1.enum_idx, dictionary_snapshot, save_folded); + std::vector<vespalib::string> expected_folded{"FOO", "foo"}; + EXPECT_EQUAL(expected_folded, folded); +} + +TEST_F("require that collect_folded works for integers", LongFixture) +{ + IntegerAttributeTemplate<int64_t> *attr = dynamic_cast<IntegerAttributeTemplate<int64_t> *>(f1.attr.get()); + set_doc(attr, 2, int64_t(112), 30); + attr->commit(); + auto dictionary_snapshot = f1.api->get_dictionary_snapshot(); + auto lookup1 = f1.api->lookup("111", dictionary_snapshot); + std::vector<int64_t> folded; + std::function<void(vespalib::datastore::EntryRef)> save_folded = [&folded,attr](vespalib::datastore::EntryRef enum_idx) { folded.emplace_back(attr->getFromEnum(enum_idx.ref())); }; + f1.api->collect_folded(lookup1.enum_idx, dictionary_snapshot, save_folded); + std::vector<int64_t> expected_folded{int64_t(111)}; + EXPECT_EQUAL(expected_folded, folded); +} + class Verifier : public search::test::SearchIteratorVerifier { public: Verifier(); @@ -176,7 +225,7 @@ public: (void) strict; const IDocumentWeightAttribute *api(_attr->asDocumentWeightAttribute()); ASSERT_TRUE(api != nullptr); - auto dict_entry = api->lookup("123"); + auto dict_entry = api->lookup("123", api->get_dictionary_snapshot()); ASSERT_TRUE(dict_entry.posting_idx.valid()); return std::make_unique<queryeval::DocumentWeightSearchIterator>(_tfmd, *api, dict_entry); } diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index c698a1d612b..089a2a8476e 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -10,6 +10,7 @@ #include <vespa/searchlib/queryeval/nearest_neighbor_blueprint.h> #include <vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> +#include <vespa/searchlib/tensor/direct_tensor_attribute.h> #include <vespa/searchlib/tensor/doc_vector_access.h> #include <vespa/searchlib/tensor/generic_tensor_attribute.h> #include <vespa/searchlib/tensor/hnsw_index.h> @@ -37,6 +38,7 @@ using search::queryeval::GlobalFilter; using search::queryeval::NearestNeighborBlueprint; using search::tensor::DefaultNearestNeighborIndexFactory; using search::tensor::DenseTensorAttribute; +using search::tensor::DirectTensorAttribute; using search::tensor::DocVectorAccess; using search::tensor::GenericTensorAttribute; using search::tensor::HnswIndex; @@ -256,6 +258,40 @@ class MockNearestNeighborIndexFactory : public NearestNeighborIndexFactory { const vespalib::string test_dir = "test_data/"; const vespalib::string attr_name = test_dir + "my_attr"; +struct FixtureTraits { + bool use_dense_tensor_attribute = false; + bool use_direct_tensor_attribute = false; + bool enable_hnsw_index = false; + bool use_mock_index = false; + + FixtureTraits dense() && { + use_dense_tensor_attribute = true; + enable_hnsw_index = false; + return *this; + } + + FixtureTraits hnsw() && { + use_dense_tensor_attribute = true; + enable_hnsw_index = true; + use_mock_index = false; + return *this; + } + + FixtureTraits mock_hnsw() && { + use_dense_tensor_attribute = true; + enable_hnsw_index = true; + use_mock_index = true; + return *this; + } + + FixtureTraits direct() && { + use_dense_tensor_attribute = false; + use_direct_tensor_attribute = true; + return *this; + } + +}; + struct Fixture { using BasicType = search::attribute::BasicType; using CollectionType = search::attribute::CollectionType; @@ -270,24 +306,21 @@ struct Fixture { std::shared_ptr<TensorAttribute> _tensorAttr; std::shared_ptr<AttributeVector> _attr; bool _denseTensors; - bool _useDenseTensorAttribute; + FixtureTraits _traits; Fixture(const vespalib::string &typeSpec, - bool useDenseTensorAttribute = false, - bool enable_hnsw_index = false, - bool use_mock_index = false) + FixtureTraits traits = FixtureTraits()) : _dir_handler(test_dir), _cfg(BasicType::TENSOR, CollectionType::SINGLE), _name(attr_name), _typeSpec(typeSpec), - _use_mock_index(use_mock_index), _index_factory(), _tensorAttr(), _attr(), _denseTensors(false), - _useDenseTensorAttribute(useDenseTensorAttribute) + _traits(traits) { - if (enable_hnsw_index) { + if (traits.enable_hnsw_index) { _cfg.set_distance_metric(DistanceMetric::Euclidean); _cfg.set_hnsw_index_params(HnswIndexParams(4, 20, DistanceMetric::Euclidean)); } @@ -301,7 +334,7 @@ struct Fixture { if (_cfg.tensorType().is_dense()) { _denseTensors = true; } - if (_use_mock_index) { + if (_traits.use_mock_index) { _index_factory = std::make_unique<MockNearestNeighborIndexFactory>(); } else { _index_factory = std::make_unique<DefaultNearestNeighborIndexFactory>(); @@ -322,9 +355,11 @@ struct Fixture { } std::shared_ptr<TensorAttribute> makeAttr() { - if (_useDenseTensorAttribute) { + if (_traits.use_dense_tensor_attribute) { assert(_denseTensors); return std::make_shared<DenseTensorAttribute>(_name, _cfg, *_index_factory); + } else if (_traits.use_direct_tensor_attribute) { + return std::make_shared<DirectTensorAttribute>(_name, _cfg); } else { return std::make_shared<GenericTensorAttribute>(_name, _cfg); } @@ -543,7 +578,7 @@ Fixture::testSaveLoad() void Fixture::testCompaction() { - if (_useDenseTensorAttribute && _denseTensors) { + if (_traits.use_dense_tensor_attribute && _denseTensors) { LOG(info, "Skipping compaction test for tensor '%s' which is using free-lists", _cfg.tensorType().to_spec().c_str()); return; } @@ -607,7 +642,7 @@ Fixture::testTensorTypeFileHeaderTag() auto header = get_file_header(); EXPECT_TRUE(header.hasTag("tensortype")); EXPECT_EQUAL(_typeSpec, header.getTag("tensortype").asString()); - if (_useDenseTensorAttribute) { + if (_traits.use_dense_tensor_attribute) { EXPECT_EQUAL(1u, header.getTag("version").asInteger()); } else { EXPECT_EQUAL(0u, header.getTag("version").asInteger()); @@ -644,6 +679,11 @@ TEST("Test sparse tensors with generic tensor attribute") testAll([]() { return std::make_shared<Fixture>(sparseSpec); }); } +TEST("Test sparse tensors with direct tensor attribute") +{ + testAll([]() { return std::make_shared<Fixture>(sparseSpec, FixtureTraits().direct()); }); +} + TEST("Test dense tensors with generic tensor attribute") { testAll([]() { return std::make_shared<Fixture>(denseSpec); }); @@ -651,11 +691,11 @@ TEST("Test dense tensors with generic tensor attribute") TEST("Test dense tensors with dense tensor attribute") { - testAll([]() { return std::make_shared<Fixture>(denseSpec, true); }); + testAll([]() { return std::make_shared<Fixture>(denseSpec, FixtureTraits().dense()); }); } TEST_F("Hnsw index is NOT instantiated in dense tensor attribute by default", - Fixture(vec_2d_spec, true, false)) + Fixture(vec_2d_spec, FixtureTraits().dense())) { const auto& tensor = f.as_dense_tensor(); EXPECT_TRUE(tensor.nearest_neighbor_index() == nullptr); @@ -663,7 +703,7 @@ TEST_F("Hnsw index is NOT instantiated in dense tensor attribute by default", class DenseTensorAttributeHnswIndex : public Fixture { public: - DenseTensorAttributeHnswIndex() : Fixture(vec_2d_spec, true, true, false) {} + DenseTensorAttributeHnswIndex() : Fixture(vec_2d_spec, FixtureTraits().hnsw()) {} }; TEST_F("Hnsw index is instantiated in dense tensor attribute when specified in config", DenseTensorAttributeHnswIndex) @@ -704,9 +744,10 @@ TEST_F("Hnsw index is integrated in dense tensor attribute and can be saved and expect_level_0(1, index_b.get_node(2)); } + class DenseTensorAttributeMockIndex : public Fixture { public: - DenseTensorAttributeMockIndex() : Fixture(vec_2d_spec, true, true, true) {} + DenseTensorAttributeMockIndex() : Fixture(vec_2d_spec, FixtureTraits().mock_hnsw()) {} }; TEST_F("setTensor() updates nearest neighbor index", DenseTensorAttributeMockIndex) diff --git a/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp b/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp index cc6b8e0ce29..7a200a46ab2 100644 --- a/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp +++ b/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp @@ -25,7 +25,8 @@ std::string get_source_dir() { } std::string source_dir = get_source_dir(); std::string vespa_dir = source_dir + "/" + "../../../../.."; -std::string simple_model = vespa_dir + "/" + "model-integration/src/test/models/onnx/simple/simple.onnx"; +std::string simple_model = vespa_dir + "/" + "eval/src/tests/tensor/onnx_wrapper/simple.onnx"; +std::string dynamic_model = vespa_dir + "/" + "eval/src/tests/tensor/onnx_wrapper/dynamic.onnx"; uint32_t default_docid = 1; @@ -97,4 +98,16 @@ TEST_F(OnnxFeatureTest, simple_onnx_model_can_be_calculated) { EXPECT_EQ(get(3), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 89.0)); } +TEST_F(OnnxFeatureTest, dynamic_onnx_model_can_be_calculated) { + add_expr("query_tensor", "tensor<float>(a[1],b[4]):[[docid,2,3,4]]"); + add_expr("attribute_tensor", "tensor<float>(a[4],b[1]):[[5],[6],[7],[8]]"); + add_expr("bias_tensor", "tensor<float>(a[1],b[2]):[[4,5]]"); + add_onnx("dynamic", dynamic_model); + compile(onnx_feature("dynamic")); + EXPECT_EQ(get(1), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 79.0)); + EXPECT_EQ(get("onnxModel(dynamic).output", 1), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 79.0)); + EXPECT_EQ(get(2), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 84.0)); + EXPECT_EQ(get(3), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 89.0)); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp index d4915ed29f4..18efd69d0b9 100644 --- a/searchlib/src/tests/features/tensor/tensor_test.cpp +++ b/searchlib/src/tests/features/tensor/tensor_test.cpp @@ -10,6 +10,7 @@ #include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> #include <vespa/searchlib/fef/test/queryenvironment.h> #include <vespa/searchlib/tensor/tensor_attribute.h> +#include <vespa/searchlib/tensor/direct_tensor_attribute.h> #include <vespa/eval/eval/function.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/tensor/tensor.h> @@ -25,6 +26,7 @@ using namespace search::fef::test; using namespace search::features; using search::AttributeFactory; using search::tensor::TensorAttribute; +using search::tensor::DirectTensorAttribute; using search::AttributeVector; using vespalib::eval::Function; using vespalib::eval::Value; @@ -70,10 +72,14 @@ struct ExecFixture addAttributeField(attrName); return AttributeFactory::createAttribute(attrName, AVC(AVBT::STRING, AVCT::SINGLE)); } - AttributeVector::SP createTensorAttribute(const vespalib::string &attrName, const vespalib::string &type) { + AttributeVector::SP createTensorAttribute(const vespalib::string &attrName, + const vespalib::string &type, + bool direct = false) + { addAttributeField(attrName); AVC config(AVBT::TENSOR, AVCT::SINGLE); config.setTensorType(ValueType::from_spec(type)); + config.setFastSearch(direct); return AttributeFactory::createAttribute(attrName, config); } void setAttributeTensorType(const vespalib::string &attrName, const vespalib::string &type) { @@ -85,10 +91,12 @@ struct ExecFixture void setupAttributeVectors() { std::vector<AttributePtr> attrs; attrs.push_back(createTensorAttribute("tensorattr", "tensor(x{})")); + attrs.push_back(createTensorAttribute("directattr", "tensor(x{})", true)); attrs.push_back(createStringAttribute("singlestr")); attrs.push_back(createTensorAttribute("wrongtype", "tensor(y{})")); addAttributeField("null"); setAttributeTensorType("tensorattr", "tensor(x{})"); + setAttributeTensorType("directattr", "tensor(x{})"); setAttributeTensorType("wrongtype", "tensor(x{})"); setAttributeTensorType("null", "tensor(x{})"); @@ -103,11 +111,16 @@ struct ExecFixture TensorAttribute *tensorAttr = dynamic_cast<TensorAttribute *>(attrs[0].get()); + DirectTensorAttribute *directAttr = + dynamic_cast<DirectTensorAttribute *>(attrs[1].get()); + + auto doc_tensor = makeTensor<Tensor>(TensorSpec("tensor(x{})") + .add({{"x", "a"}}, 3) + .add({{"x", "b"}}, 5) + .add({{"x", "c"}}, 7)); + tensorAttr->setTensor(1, *doc_tensor); + directAttr->set_tensor(1, std::move(doc_tensor)); - tensorAttr->setTensor(1, *makeTensor<Tensor>(TensorSpec("tensor(x{})") - .add({{"x", "a"}}, 3) - .add({{"x", "b"}}, 5) - .add({{"x", "c"}}, 7))); for (const auto &attr : attrs) { attr->commit(); } @@ -157,6 +170,15 @@ TEST_F("require that tensor attribute can be extracted as tensor in attribute fe .add({{"x", "a"}}, 3)), f.execute()); } +TEST_F("require that direct tensor attribute can be extracted in attribute feature", + ExecFixture("attribute(directattr)")) +{ + EXPECT_EQUAL(*makeTensor<Tensor>(TensorSpec("tensor(x{})") + .add({{"x", "b"}}, 5) + .add({{"x", "c"}}, 7) + .add({{"x", "a"}}, 3)), f.execute()); +} + TEST_F("require that tensor from query can be extracted as tensor in query feature", ExecFixture("query(tensorquery)")) { @@ -189,6 +211,11 @@ TEST_F("require that empty tensor with correct type is created if document has n EXPECT_EQUAL(*make_empty("tensor(x{})"), f.execute(2)); } +TEST_F("require that empty tensor with correct type is returned by direct tensor attribute", + ExecFixture("attribute(directattr)")) { + EXPECT_EQUAL(*make_empty("tensor(x{})"), f.execute(2)); +} + TEST_F("require that wrong tensor type from query tensor gives empty tensor", ExecFixture("query(mappedtensorquery)")) { EXPECT_EQUAL(*makeTensor<Tensor>(TensorSpec("tensor(x[2])") diff --git a/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp index 1eb9890cf2e..764ce6ab1b1 100644 --- a/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp @@ -6,7 +6,6 @@ #include <vespa/log/log.h> LOG_SETUP("feature_store_test"); -using namespace vespalib::btree; using namespace vespalib::datastore; using namespace search::index; diff --git a/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp b/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp index 9761b0da2d7..f2c02d02080 100644 --- a/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp +++ b/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp @@ -674,7 +674,7 @@ private: MatchParams match_params(_dummy_heap, _dummy_heap.getMinScore(), 1.0, 1); std::vector<IDocumentWeightAttribute::LookupResult> dict_entries; for (size_t i = 0; i < _num_children; ++i) { - dict_entries.push_back(_helper.dwa().lookup(vespalib::make_string("%zu", i).c_str())); + dict_entries.push_back(_helper.dwa().lookup(vespalib::make_string("%zu", i).c_str(), _helper.dwa().get_dictionary_snapshot())); } return create_wand(_use_dwa, _tfmd, match_params, _weights, dict_entries, _helper.dwa(), strict); } diff --git a/searchlib/src/tests/tensor/direct_tensor_store/CMakeLists.txt b/searchlib/src/tests/tensor/direct_tensor_store/CMakeLists.txt new file mode 100644 index 00000000000..14a70f25e3c --- /dev/null +++ b/searchlib/src/tests/tensor/direct_tensor_store/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_direct_tensor_store_test_app TEST + SOURCES + direct_tensor_store_test.cpp + DEPENDS + searchlib + GTest::GTest +) +vespa_add_test(NAME searchlib_direct_tensor_store_test_app COMMAND searchlib_direct_tensor_store_test_app) diff --git a/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp b/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp new file mode 100644 index 00000000000..1932e6d7d1f --- /dev/null +++ b/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp @@ -0,0 +1,89 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/tensor/direct_tensor_store.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/tensor.h> + +using namespace search::tensor; + +using vespalib::datastore::EntryRef; +using vespalib::eval::TensorSpec; +using vespalib::tensor::DefaultTensorEngine; +using vespalib::tensor::Tensor; + +vespalib::string tensor_spec("tensor(x{})"); + +Tensor::UP +make_tensor(const TensorSpec& spec) +{ + auto value = DefaultTensorEngine::ref().from_spec(spec); + auto* tensor = dynamic_cast<Tensor*>(value.get()); + assert(tensor != nullptr); + value.release(); + return Tensor::UP(tensor); +} + +Tensor::UP +make_tensor(double value) +{ + return make_tensor(TensorSpec(tensor_spec).add({{"x", "a"}}, value)); +} + +class DirectTensorStoreTest : public ::testing::Test { +public: + DirectTensorStore store; + + DirectTensorStoreTest() : store() {} + + virtual ~DirectTensorStoreTest() { + store.clearHoldLists(); + } + + void expect_tensor(const Tensor* exp, EntryRef ref) { + const auto* act = store.get_tensor(ref); + ASSERT_TRUE(act); + EXPECT_EQ(exp, act); + } +}; + +TEST_F(DirectTensorStoreTest, can_set_and_get_tensor) +{ + auto t = make_tensor(5); + auto* exp = t.get(); + auto ref = store.store_tensor(std::move(t)); + expect_tensor(exp, ref); +} + +TEST_F(DirectTensorStoreTest, invalid_ref_returns_nullptr) +{ + const auto* t = store.get_tensor(EntryRef()); + EXPECT_FALSE(t); +} + +TEST_F(DirectTensorStoreTest, hold_adds_entry_to_hold_list) +{ + auto ref = store.store_tensor(make_tensor(5)); + auto mem_1 = store.getMemoryUsage(); + store.holdTensor(ref); + auto mem_2 = store.getMemoryUsage(); + EXPECT_GT(mem_2.allocatedBytesOnHold(), mem_1.allocatedBytesOnHold()); +} + +TEST_F(DirectTensorStoreTest, move_allocates_new_entry_and_puts_old_entry_on_hold) +{ + auto t = make_tensor(5); + auto* exp = t.get(); + auto ref_1 = store.store_tensor(std::move(t)); + auto mem_1 = store.getMemoryUsage(); + + auto ref_2 = store.move(ref_1); + auto mem_2 = store.getMemoryUsage(); + EXPECT_NE(ref_1, ref_2); + expect_tensor(exp, ref_1); + expect_tensor(exp, ref_2); + EXPECT_GT(mem_2.allocatedBytesOnHold(), mem_1.allocatedBytesOnHold()); +} + +GTEST_MAIN_RUN_ALL_TESTS() + diff --git a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp index 9c896396de3..a5e0e1e2b6a 100644 --- a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp @@ -1,9 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <sys/types.h> -#include <sys/stat.h> #include <fcntl.h> -#include <stdio.h> +#include <cstdio> #include <unistd.h> #include <chrono> #include <cstdlib> @@ -24,6 +22,7 @@ #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/util/lambdatask.h> +#include <vespa/vespalib/data/simple_buffer.h> #include <vespa/log/log.h> LOG_SETUP("stress_hnsw_mt"); diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index 81a3006dbff..013ca81dcc9 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -8,7 +8,6 @@ #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/fastos/app.h> #include <iostream> -#include <stdexcept> #include <sstream> #include <thread> @@ -223,7 +222,6 @@ FeederThread::~FeederThread() = default; void FeederThread::commitPacket() { - _packet.close(); const vespalib::nbostream& stream = _packet.getHandle(); if (!_session->commit(ConstBufferRef(stream.data(), stream.size()))) { throw std::runtime_error(vespalib::make_string @@ -238,8 +236,9 @@ FeederThread::commitPacket() bool FeederThread::addEntry(const Packet::Entry & e) { - //LOG(info, "FeederThread: add %s", EntryPrinter::toStr(e).c_str()); - return _packet.add(e); + if (_packet.sizeBytes() > 0xf000) return false; + _packet.add(e); + return true; } void |