aboutsummaryrefslogtreecommitdiffstats
path: root/searchlib/src/tests
diff options
context:
space:
mode:
Diffstat (limited to 'searchlib/src/tests')
-rw-r--r--searchlib/src/tests/attribute/bitvector/bitvector_test.cpp2
-rw-r--r--searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp61
-rw-r--r--searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp71
-rw-r--r--searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp15
-rw-r--r--searchlib/src/tests/features/tensor/tensor_test.cpp37
-rw-r--r--searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp1
-rw-r--r--searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp2
-rw-r--r--searchlib/src/tests/tensor/direct_tensor_store/CMakeLists.txt9
-rw-r--r--searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp89
-rw-r--r--searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp5
-rw-r--r--searchlib/src/tests/transactionlogstress/translogstress.cpp7
11 files changed, 262 insertions, 37 deletions
diff --git a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp
index 24919fb2341..04d2dfe4d52 100644
--- a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp
+++ b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp
@@ -491,7 +491,7 @@ BitVectorTest::test(BasicType bt,
v->asDocumentWeightAttribute();
if (dwa != NULL) {
search::IDocumentWeightAttribute::LookupResult lres =
- dwa->lookup(getSearchStr<VectorType>());
+ dwa->lookup(getSearchStr<VectorType>(), dwa->get_dictionary_snapshot());
typedef search::queryeval::DocumentWeightSearchIterator DWSI;
typedef search::queryeval::SearchIterator SI;
TermFieldMatchData md;
diff --git a/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp b/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp
index cf1506a9118..d8a1d03f1a8 100644
--- a/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp
+++ b/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp
@@ -3,6 +3,7 @@
#include <vespa/searchlib/attribute/attribute.h>
#include <vespa/searchlib/attribute/attributefactory.h>
#include <vespa/searchlib/attribute/attributeguard.h>
+#include <vespa/searchlib/attribute/attribute_read_guard.h>
#include <vespa/searchlib/attribute/attributememorysavetarget.h>
#include <vespa/searchlib/attribute/attributevector.h>
#include <vespa/searchlib/attribute/attributevector.hpp>
@@ -22,6 +23,7 @@
#include <vespa/searchlib/test/searchiteratorverifier.h>
#include <vespa/searchlib/util/randomgenerator.h>
#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/test/insertion_operators.h>
#include <vespa/log/log.h>
LOG_SETUP("document_weight_iterator_test");
@@ -124,17 +126,17 @@ void verify_invalid_lookup(IDocumentWeightAttribute::LookupResult result) {
}
TEST_F("require that integer lookup works correctly", LongFixture) {
- verify_valid_lookup(f1.api->lookup("111"));
- verify_invalid_lookup(f1.api->lookup("222"));
+ verify_valid_lookup(f1.api->lookup("111", f1.api->get_dictionary_snapshot()));
+ verify_invalid_lookup(f1.api->lookup("222", f1.api->get_dictionary_snapshot()));
}
TEST_F("require string lookup works correctly", StringFixture) {
- verify_valid_lookup(f1.api->lookup("foo"));
- verify_invalid_lookup(f1.api->lookup("bar"));
+ verify_valid_lookup(f1.api->lookup("foo", f1.api->get_dictionary_snapshot()));
+ verify_invalid_lookup(f1.api->lookup("bar", f1.api->get_dictionary_snapshot()));
}
void verify_posting(const IDocumentWeightAttribute &api, const char *term) {
- auto result = api.lookup(term);
+ auto result = api.lookup(term, api.get_dictionary_snapshot());
ASSERT_TRUE(result.posting_idx.valid());
std::vector<DocumentWeightIterator> itr_store;
api.create(result.posting_idx, itr_store);
@@ -168,6 +170,53 @@ TEST_F("require that string iterators are created correctly", StringFixture) {
verify_posting(*f1.api, "foo");
}
+TEST_F("require that dictionary snapshot works", LongFixture)
+{
+ auto read_guard = f1.attr->makeReadGuard(false);
+ auto dictionary_snapshot = f1.api->get_dictionary_snapshot();
+ auto lookup1 = f1.api->lookup("111", dictionary_snapshot);
+ EXPECT_TRUE(lookup1.enum_idx.valid());
+ f1.attr->clearDoc(1);
+ f1.attr->clearDoc(5);
+ f1.attr->clearDoc(7);
+ f1.attr->commit();
+ auto lookup2 = f1.api->lookup("111", f1.api->get_dictionary_snapshot());
+ EXPECT_FALSE(lookup2.enum_idx.valid());
+ auto lookup3 = f1.api->lookup("111", dictionary_snapshot);
+ EXPECT_TRUE(lookup3.enum_idx.valid());
+ EXPECT_EQUAL(lookup1.enum_idx.ref(), lookup3.enum_idx.ref());
+}
+
+TEST_F("require that collect_folded works for string", StringFixture)
+{
+ StringAttribute *attr = static_cast<StringAttribute *>(f1.attr.get());
+ set_doc(attr, 2, "bar", 30);
+ attr->commit();
+ set_doc(attr, 3, "FOO", 30);
+ attr->commit();
+ auto dictionary_snapshot = f1.api->get_dictionary_snapshot();
+ auto lookup1 = f1.api->lookup("foo", dictionary_snapshot);
+ std::vector<vespalib::string> folded;
+ std::function<void(vespalib::datastore::EntryRef)> save_folded = [&folded,attr](vespalib::datastore::EntryRef enum_idx) { folded.emplace_back(attr->getFromEnum(enum_idx.ref())); };
+ f1.api->collect_folded(lookup1.enum_idx, dictionary_snapshot, save_folded);
+ std::vector<vespalib::string> expected_folded{"FOO", "foo"};
+ EXPECT_EQUAL(expected_folded, folded);
+}
+
+TEST_F("require that collect_folded works for integers", LongFixture)
+{
+ IntegerAttributeTemplate<int64_t> *attr = dynamic_cast<IntegerAttributeTemplate<int64_t> *>(f1.attr.get());
+ set_doc(attr, 2, int64_t(112), 30);
+ attr->commit();
+ auto dictionary_snapshot = f1.api->get_dictionary_snapshot();
+ auto lookup1 = f1.api->lookup("111", dictionary_snapshot);
+ std::vector<int64_t> folded;
+ std::function<void(vespalib::datastore::EntryRef)> save_folded = [&folded,attr](vespalib::datastore::EntryRef enum_idx) { folded.emplace_back(attr->getFromEnum(enum_idx.ref())); };
+ f1.api->collect_folded(lookup1.enum_idx, dictionary_snapshot, save_folded);
+ std::vector<int64_t> expected_folded{int64_t(111)};
+ EXPECT_EQUAL(expected_folded, folded);
+}
+
class Verifier : public search::test::SearchIteratorVerifier {
public:
Verifier();
@@ -176,7 +225,7 @@ public:
(void) strict;
const IDocumentWeightAttribute *api(_attr->asDocumentWeightAttribute());
ASSERT_TRUE(api != nullptr);
- auto dict_entry = api->lookup("123");
+ auto dict_entry = api->lookup("123", api->get_dictionary_snapshot());
ASSERT_TRUE(dict_entry.posting_idx.valid());
return std::make_unique<queryeval::DocumentWeightSearchIterator>(_tfmd, *api, dict_entry);
}
diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
index c698a1d612b..089a2a8476e 100644
--- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
+++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
@@ -10,6 +10,7 @@
#include <vespa/searchlib/queryeval/nearest_neighbor_blueprint.h>
#include <vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h>
#include <vespa/searchlib/tensor/dense_tensor_attribute.h>
+#include <vespa/searchlib/tensor/direct_tensor_attribute.h>
#include <vespa/searchlib/tensor/doc_vector_access.h>
#include <vespa/searchlib/tensor/generic_tensor_attribute.h>
#include <vespa/searchlib/tensor/hnsw_index.h>
@@ -37,6 +38,7 @@ using search::queryeval::GlobalFilter;
using search::queryeval::NearestNeighborBlueprint;
using search::tensor::DefaultNearestNeighborIndexFactory;
using search::tensor::DenseTensorAttribute;
+using search::tensor::DirectTensorAttribute;
using search::tensor::DocVectorAccess;
using search::tensor::GenericTensorAttribute;
using search::tensor::HnswIndex;
@@ -256,6 +258,40 @@ class MockNearestNeighborIndexFactory : public NearestNeighborIndexFactory {
const vespalib::string test_dir = "test_data/";
const vespalib::string attr_name = test_dir + "my_attr";
+struct FixtureTraits {
+ bool use_dense_tensor_attribute = false;
+ bool use_direct_tensor_attribute = false;
+ bool enable_hnsw_index = false;
+ bool use_mock_index = false;
+
+ FixtureTraits dense() && {
+ use_dense_tensor_attribute = true;
+ enable_hnsw_index = false;
+ return *this;
+ }
+
+ FixtureTraits hnsw() && {
+ use_dense_tensor_attribute = true;
+ enable_hnsw_index = true;
+ use_mock_index = false;
+ return *this;
+ }
+
+ FixtureTraits mock_hnsw() && {
+ use_dense_tensor_attribute = true;
+ enable_hnsw_index = true;
+ use_mock_index = true;
+ return *this;
+ }
+
+ FixtureTraits direct() && {
+ use_dense_tensor_attribute = false;
+ use_direct_tensor_attribute = true;
+ return *this;
+ }
+
+};
+
struct Fixture {
using BasicType = search::attribute::BasicType;
using CollectionType = search::attribute::CollectionType;
@@ -270,24 +306,21 @@ struct Fixture {
std::shared_ptr<TensorAttribute> _tensorAttr;
std::shared_ptr<AttributeVector> _attr;
bool _denseTensors;
- bool _useDenseTensorAttribute;
+ FixtureTraits _traits;
Fixture(const vespalib::string &typeSpec,
- bool useDenseTensorAttribute = false,
- bool enable_hnsw_index = false,
- bool use_mock_index = false)
+ FixtureTraits traits = FixtureTraits())
: _dir_handler(test_dir),
_cfg(BasicType::TENSOR, CollectionType::SINGLE),
_name(attr_name),
_typeSpec(typeSpec),
- _use_mock_index(use_mock_index),
_index_factory(),
_tensorAttr(),
_attr(),
_denseTensors(false),
- _useDenseTensorAttribute(useDenseTensorAttribute)
+ _traits(traits)
{
- if (enable_hnsw_index) {
+ if (traits.enable_hnsw_index) {
_cfg.set_distance_metric(DistanceMetric::Euclidean);
_cfg.set_hnsw_index_params(HnswIndexParams(4, 20, DistanceMetric::Euclidean));
}
@@ -301,7 +334,7 @@ struct Fixture {
if (_cfg.tensorType().is_dense()) {
_denseTensors = true;
}
- if (_use_mock_index) {
+ if (_traits.use_mock_index) {
_index_factory = std::make_unique<MockNearestNeighborIndexFactory>();
} else {
_index_factory = std::make_unique<DefaultNearestNeighborIndexFactory>();
@@ -322,9 +355,11 @@ struct Fixture {
}
std::shared_ptr<TensorAttribute> makeAttr() {
- if (_useDenseTensorAttribute) {
+ if (_traits.use_dense_tensor_attribute) {
assert(_denseTensors);
return std::make_shared<DenseTensorAttribute>(_name, _cfg, *_index_factory);
+ } else if (_traits.use_direct_tensor_attribute) {
+ return std::make_shared<DirectTensorAttribute>(_name, _cfg);
} else {
return std::make_shared<GenericTensorAttribute>(_name, _cfg);
}
@@ -543,7 +578,7 @@ Fixture::testSaveLoad()
void
Fixture::testCompaction()
{
- if (_useDenseTensorAttribute && _denseTensors) {
+ if (_traits.use_dense_tensor_attribute && _denseTensors) {
LOG(info, "Skipping compaction test for tensor '%s' which is using free-lists", _cfg.tensorType().to_spec().c_str());
return;
}
@@ -607,7 +642,7 @@ Fixture::testTensorTypeFileHeaderTag()
auto header = get_file_header();
EXPECT_TRUE(header.hasTag("tensortype"));
EXPECT_EQUAL(_typeSpec, header.getTag("tensortype").asString());
- if (_useDenseTensorAttribute) {
+ if (_traits.use_dense_tensor_attribute) {
EXPECT_EQUAL(1u, header.getTag("version").asInteger());
} else {
EXPECT_EQUAL(0u, header.getTag("version").asInteger());
@@ -644,6 +679,11 @@ TEST("Test sparse tensors with generic tensor attribute")
testAll([]() { return std::make_shared<Fixture>(sparseSpec); });
}
+TEST("Test sparse tensors with direct tensor attribute")
+{
+ testAll([]() { return std::make_shared<Fixture>(sparseSpec, FixtureTraits().direct()); });
+}
+
TEST("Test dense tensors with generic tensor attribute")
{
testAll([]() { return std::make_shared<Fixture>(denseSpec); });
@@ -651,11 +691,11 @@ TEST("Test dense tensors with generic tensor attribute")
TEST("Test dense tensors with dense tensor attribute")
{
- testAll([]() { return std::make_shared<Fixture>(denseSpec, true); });
+ testAll([]() { return std::make_shared<Fixture>(denseSpec, FixtureTraits().dense()); });
}
TEST_F("Hnsw index is NOT instantiated in dense tensor attribute by default",
- Fixture(vec_2d_spec, true, false))
+ Fixture(vec_2d_spec, FixtureTraits().dense()))
{
const auto& tensor = f.as_dense_tensor();
EXPECT_TRUE(tensor.nearest_neighbor_index() == nullptr);
@@ -663,7 +703,7 @@ TEST_F("Hnsw index is NOT instantiated in dense tensor attribute by default",
class DenseTensorAttributeHnswIndex : public Fixture {
public:
- DenseTensorAttributeHnswIndex() : Fixture(vec_2d_spec, true, true, false) {}
+ DenseTensorAttributeHnswIndex() : Fixture(vec_2d_spec, FixtureTraits().hnsw()) {}
};
TEST_F("Hnsw index is instantiated in dense tensor attribute when specified in config", DenseTensorAttributeHnswIndex)
@@ -704,9 +744,10 @@ TEST_F("Hnsw index is integrated in dense tensor attribute and can be saved and
expect_level_0(1, index_b.get_node(2));
}
+
class DenseTensorAttributeMockIndex : public Fixture {
public:
- DenseTensorAttributeMockIndex() : Fixture(vec_2d_spec, true, true, true) {}
+ DenseTensorAttributeMockIndex() : Fixture(vec_2d_spec, FixtureTraits().mock_hnsw()) {}
};
TEST_F("setTensor() updates nearest neighbor index", DenseTensorAttributeMockIndex)
diff --git a/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp b/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp
index cc6b8e0ce29..7a200a46ab2 100644
--- a/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp
+++ b/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp
@@ -25,7 +25,8 @@ std::string get_source_dir() {
}
std::string source_dir = get_source_dir();
std::string vespa_dir = source_dir + "/" + "../../../../..";
-std::string simple_model = vespa_dir + "/" + "model-integration/src/test/models/onnx/simple/simple.onnx";
+std::string simple_model = vespa_dir + "/" + "eval/src/tests/tensor/onnx_wrapper/simple.onnx";
+std::string dynamic_model = vespa_dir + "/" + "eval/src/tests/tensor/onnx_wrapper/dynamic.onnx";
uint32_t default_docid = 1;
@@ -97,4 +98,16 @@ TEST_F(OnnxFeatureTest, simple_onnx_model_can_be_calculated) {
EXPECT_EQ(get(3), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 89.0));
}
+TEST_F(OnnxFeatureTest, dynamic_onnx_model_can_be_calculated) {
+ add_expr("query_tensor", "tensor<float>(a[1],b[4]):[[docid,2,3,4]]");
+ add_expr("attribute_tensor", "tensor<float>(a[4],b[1]):[[5],[6],[7],[8]]");
+ add_expr("bias_tensor", "tensor<float>(a[1],b[2]):[[4,5]]");
+ add_onnx("dynamic", dynamic_model);
+ compile(onnx_feature("dynamic"));
+ EXPECT_EQ(get(1), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 79.0));
+ EXPECT_EQ(get("onnxModel(dynamic).output", 1), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 79.0));
+ EXPECT_EQ(get(2), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 84.0));
+ EXPECT_EQ(get(3), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 89.0));
+}
+
GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp
index d4915ed29f4..18efd69d0b9 100644
--- a/searchlib/src/tests/features/tensor/tensor_test.cpp
+++ b/searchlib/src/tests/features/tensor/tensor_test.cpp
@@ -10,6 +10,7 @@
#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h>
#include <vespa/searchlib/fef/test/queryenvironment.h>
#include <vespa/searchlib/tensor/tensor_attribute.h>
+#include <vespa/searchlib/tensor/direct_tensor_attribute.h>
#include <vespa/eval/eval/function.h>
#include <vespa/eval/eval/tensor_spec.h>
#include <vespa/eval/tensor/tensor.h>
@@ -25,6 +26,7 @@ using namespace search::fef::test;
using namespace search::features;
using search::AttributeFactory;
using search::tensor::TensorAttribute;
+using search::tensor::DirectTensorAttribute;
using search::AttributeVector;
using vespalib::eval::Function;
using vespalib::eval::Value;
@@ -70,10 +72,14 @@ struct ExecFixture
addAttributeField(attrName);
return AttributeFactory::createAttribute(attrName, AVC(AVBT::STRING, AVCT::SINGLE));
}
- AttributeVector::SP createTensorAttribute(const vespalib::string &attrName, const vespalib::string &type) {
+ AttributeVector::SP createTensorAttribute(const vespalib::string &attrName,
+ const vespalib::string &type,
+ bool direct = false)
+ {
addAttributeField(attrName);
AVC config(AVBT::TENSOR, AVCT::SINGLE);
config.setTensorType(ValueType::from_spec(type));
+ config.setFastSearch(direct);
return AttributeFactory::createAttribute(attrName, config);
}
void setAttributeTensorType(const vespalib::string &attrName, const vespalib::string &type) {
@@ -85,10 +91,12 @@ struct ExecFixture
void setupAttributeVectors() {
std::vector<AttributePtr> attrs;
attrs.push_back(createTensorAttribute("tensorattr", "tensor(x{})"));
+ attrs.push_back(createTensorAttribute("directattr", "tensor(x{})", true));
attrs.push_back(createStringAttribute("singlestr"));
attrs.push_back(createTensorAttribute("wrongtype", "tensor(y{})"));
addAttributeField("null");
setAttributeTensorType("tensorattr", "tensor(x{})");
+ setAttributeTensorType("directattr", "tensor(x{})");
setAttributeTensorType("wrongtype", "tensor(x{})");
setAttributeTensorType("null", "tensor(x{})");
@@ -103,11 +111,16 @@ struct ExecFixture
TensorAttribute *tensorAttr =
dynamic_cast<TensorAttribute *>(attrs[0].get());
+ DirectTensorAttribute *directAttr =
+ dynamic_cast<DirectTensorAttribute *>(attrs[1].get());
+
+ auto doc_tensor = makeTensor<Tensor>(TensorSpec("tensor(x{})")
+ .add({{"x", "a"}}, 3)
+ .add({{"x", "b"}}, 5)
+ .add({{"x", "c"}}, 7));
+ tensorAttr->setTensor(1, *doc_tensor);
+ directAttr->set_tensor(1, std::move(doc_tensor));
- tensorAttr->setTensor(1, *makeTensor<Tensor>(TensorSpec("tensor(x{})")
- .add({{"x", "a"}}, 3)
- .add({{"x", "b"}}, 5)
- .add({{"x", "c"}}, 7)));
for (const auto &attr : attrs) {
attr->commit();
}
@@ -157,6 +170,15 @@ TEST_F("require that tensor attribute can be extracted as tensor in attribute fe
.add({{"x", "a"}}, 3)), f.execute());
}
+TEST_F("require that direct tensor attribute can be extracted in attribute feature",
+ ExecFixture("attribute(directattr)"))
+{
+ EXPECT_EQUAL(*makeTensor<Tensor>(TensorSpec("tensor(x{})")
+ .add({{"x", "b"}}, 5)
+ .add({{"x", "c"}}, 7)
+ .add({{"x", "a"}}, 3)), f.execute());
+}
+
TEST_F("require that tensor from query can be extracted as tensor in query feature",
ExecFixture("query(tensorquery)"))
{
@@ -189,6 +211,11 @@ TEST_F("require that empty tensor with correct type is created if document has n
EXPECT_EQUAL(*make_empty("tensor(x{})"), f.execute(2));
}
+TEST_F("require that empty tensor with correct type is returned by direct tensor attribute",
+ ExecFixture("attribute(directattr)")) {
+ EXPECT_EQUAL(*make_empty("tensor(x{})"), f.execute(2));
+}
+
TEST_F("require that wrong tensor type from query tensor gives empty tensor",
ExecFixture("query(mappedtensorquery)")) {
EXPECT_EQUAL(*makeTensor<Tensor>(TensorSpec("tensor(x[2])")
diff --git a/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp
index 1eb9890cf2e..764ce6ab1b1 100644
--- a/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp
+++ b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp
@@ -6,7 +6,6 @@
#include <vespa/log/log.h>
LOG_SETUP("feature_store_test");
-using namespace vespalib::btree;
using namespace vespalib::datastore;
using namespace search::index;
diff --git a/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp b/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp
index 9761b0da2d7..f2c02d02080 100644
--- a/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp
+++ b/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp
@@ -674,7 +674,7 @@ private:
MatchParams match_params(_dummy_heap, _dummy_heap.getMinScore(), 1.0, 1);
std::vector<IDocumentWeightAttribute::LookupResult> dict_entries;
for (size_t i = 0; i < _num_children; ++i) {
- dict_entries.push_back(_helper.dwa().lookup(vespalib::make_string("%zu", i).c_str()));
+ dict_entries.push_back(_helper.dwa().lookup(vespalib::make_string("%zu", i).c_str(), _helper.dwa().get_dictionary_snapshot()));
}
return create_wand(_use_dwa, _tfmd, match_params, _weights, dict_entries, _helper.dwa(), strict);
}
diff --git a/searchlib/src/tests/tensor/direct_tensor_store/CMakeLists.txt b/searchlib/src/tests/tensor/direct_tensor_store/CMakeLists.txt
new file mode 100644
index 00000000000..14a70f25e3c
--- /dev/null
+++ b/searchlib/src/tests/tensor/direct_tensor_store/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_direct_tensor_store_test_app TEST
+ SOURCES
+ direct_tensor_store_test.cpp
+ DEPENDS
+ searchlib
+ GTest::GTest
+)
+vespa_add_test(NAME searchlib_direct_tensor_store_test_app COMMAND searchlib_direct_tensor_store_test_app)
diff --git a/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp b/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp
new file mode 100644
index 00000000000..1932e6d7d1f
--- /dev/null
+++ b/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp
@@ -0,0 +1,89 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchlib/tensor/direct_tensor_store.h>
+#include <vespa/vespalib/gtest/gtest.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
+#include <vespa/eval/tensor/tensor.h>
+
+using namespace search::tensor;
+
+using vespalib::datastore::EntryRef;
+using vespalib::eval::TensorSpec;
+using vespalib::tensor::DefaultTensorEngine;
+using vespalib::tensor::Tensor;
+
+vespalib::string tensor_spec("tensor(x{})");
+
+Tensor::UP
+make_tensor(const TensorSpec& spec)
+{
+ auto value = DefaultTensorEngine::ref().from_spec(spec);
+ auto* tensor = dynamic_cast<Tensor*>(value.get());
+ assert(tensor != nullptr);
+ value.release();
+ return Tensor::UP(tensor);
+}
+
+Tensor::UP
+make_tensor(double value)
+{
+ return make_tensor(TensorSpec(tensor_spec).add({{"x", "a"}}, value));
+}
+
+class DirectTensorStoreTest : public ::testing::Test {
+public:
+ DirectTensorStore store;
+
+ DirectTensorStoreTest() : store() {}
+
+ virtual ~DirectTensorStoreTest() {
+ store.clearHoldLists();
+ }
+
+ void expect_tensor(const Tensor* exp, EntryRef ref) {
+ const auto* act = store.get_tensor(ref);
+ ASSERT_TRUE(act);
+ EXPECT_EQ(exp, act);
+ }
+};
+
+TEST_F(DirectTensorStoreTest, can_set_and_get_tensor)
+{
+ auto t = make_tensor(5);
+ auto* exp = t.get();
+ auto ref = store.store_tensor(std::move(t));
+ expect_tensor(exp, ref);
+}
+
+TEST_F(DirectTensorStoreTest, invalid_ref_returns_nullptr)
+{
+ const auto* t = store.get_tensor(EntryRef());
+ EXPECT_FALSE(t);
+}
+
+TEST_F(DirectTensorStoreTest, hold_adds_entry_to_hold_list)
+{
+ auto ref = store.store_tensor(make_tensor(5));
+ auto mem_1 = store.getMemoryUsage();
+ store.holdTensor(ref);
+ auto mem_2 = store.getMemoryUsage();
+ EXPECT_GT(mem_2.allocatedBytesOnHold(), mem_1.allocatedBytesOnHold());
+}
+
+TEST_F(DirectTensorStoreTest, move_allocates_new_entry_and_puts_old_entry_on_hold)
+{
+ auto t = make_tensor(5);
+ auto* exp = t.get();
+ auto ref_1 = store.store_tensor(std::move(t));
+ auto mem_1 = store.getMemoryUsage();
+
+ auto ref_2 = store.move(ref_1);
+ auto mem_2 = store.getMemoryUsage();
+ EXPECT_NE(ref_1, ref_2);
+ expect_tensor(exp, ref_1);
+ expect_tensor(exp, ref_2);
+ EXPECT_GT(mem_2.allocatedBytesOnHold(), mem_1.allocatedBytesOnHold());
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
+
diff --git a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp
index 9c896396de3..a5e0e1e2b6a 100644
--- a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp
+++ b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp
@@ -1,9 +1,7 @@
// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <sys/types.h>
-#include <sys/stat.h>
#include <fcntl.h>
-#include <stdio.h>
+#include <cstdio>
#include <unistd.h>
#include <chrono>
#include <cstdlib>
@@ -24,6 +22,7 @@
#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
#include <vespa/vespalib/util/generationhandler.h>
#include <vespa/vespalib/util/lambdatask.h>
+#include <vespa/vespalib/data/simple_buffer.h>
#include <vespa/log/log.h>
LOG_SETUP("stress_hnsw_mt");
diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp
index 81a3006dbff..013ca81dcc9 100644
--- a/searchlib/src/tests/transactionlogstress/translogstress.cpp
+++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp
@@ -8,7 +8,6 @@
#include <vespa/searchlib/index/dummyfileheadercontext.h>
#include <vespa/fastos/app.h>
#include <iostream>
-#include <stdexcept>
#include <sstream>
#include <thread>
@@ -223,7 +222,6 @@ FeederThread::~FeederThread() = default;
void
FeederThread::commitPacket()
{
- _packet.close();
const vespalib::nbostream& stream = _packet.getHandle();
if (!_session->commit(ConstBufferRef(stream.data(), stream.size()))) {
throw std::runtime_error(vespalib::make_string
@@ -238,8 +236,9 @@ FeederThread::commitPacket()
bool
FeederThread::addEntry(const Packet::Entry & e)
{
- //LOG(info, "FeederThread: add %s", EntryPrinter::toStr(e).c_str());
- return _packet.add(e);
+ if (_packet.sizeBytes() > 0xf000) return false;
+ _packet.add(e);
+ return true;
}
void