diff options
Diffstat (limited to 'searchlib/src/tests')
9 files changed, 285 insertions, 93 deletions
diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 5089743a54a..1f3ad7c2fec 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -6,6 +6,7 @@ #include <vespa/eval/tensor/tensor.h> #include <vespa/fastos/file.h> #include <vespa/searchlib/attribute/attributeguard.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/doc_vector_access.h> @@ -41,6 +42,7 @@ using vespalib::tensor::DenseTensor; using vespalib::tensor::Tensor; using DoubleVector = std::vector<double>; +using generation_t = vespalib::GenerationHandler::generation_t; namespace vespalib::tensor { @@ -80,12 +82,16 @@ private: const DocVectorAccess& _vectors; EntryVector _adds; EntryVector _removes; + generation_t _transfer_gen; + generation_t _trim_gen; public: MockNearestNeighborIndex(const DocVectorAccess& vectors) : _vectors(vectors), _adds(), - _removes() + _removes(), + _transfer_gen(std::numeric_limits<generation_t>::max()), + _trim_gen(std::numeric_limits<generation_t>::max()) { } void clear() { @@ -111,6 +117,9 @@ public: EXPECT_EQUAL(exp_docid, _removes.back().first); EXPECT_EQUAL(exp_vector, _removes.back().second); } + generation_t get_transfer_gen() const { return _transfer_gen; } + generation_t get_trim_gen() const { return _trim_gen; } + void add_document(uint32_t docid) override { auto vector = _vectors.get_vector(docid).typify<double>(); _adds.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); @@ -119,6 +128,15 @@ public: auto vector = _vectors.get_vector(docid).typify<double>(); _removes.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); } + void transfer_hold_lists(generation_t current_gen) override { + _transfer_gen = current_gen; + } + void trim_hold_lists(generation_t first_used_gen) override { + _trim_gen = first_used_gen; + } + vespalib::MemoryUsage memory_usage() const override { + return vespalib::MemoryUsage(); + } std::vector<Neighbor> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) const override { (void) k; (void) vector; @@ -232,6 +250,10 @@ struct Fixture _attr->commit(); } + generation_t get_current_gen() const { + return _attr->getCurrentGeneration(); + } + search::attribute::Status getStatus() { _attr->commit(true); return _attr->getStatus(); @@ -531,4 +553,33 @@ TEST_F("onLoad() updates nearest neighbor index", DenseTensorAttributeMockIndex) index.expect_adds({{1, {3, 5}}, {2, {7, 9}}}); } + +TEST_F("commit() ensures transfer and trim hold lists on nearest neighbor index", DenseTensorAttributeMockIndex) +{ + auto& index = f.mock_index(); + TensorSpec spec = vec_2d(3, 5); + + f.set_tensor(1, spec); + generation_t gen_1 = f.get_current_gen(); + EXPECT_EQUAL(gen_1 - 1, index.get_transfer_gen()); + EXPECT_EQUAL(gen_1, index.get_trim_gen()); + + generation_t gen_2 = 0; + { + // Takes guard on gen_1 + auto guard = f._attr->makeReadGuard(false); + f.set_tensor(2, spec); + gen_2 = f.get_current_gen(); + EXPECT_GREATER(gen_2, gen_1); + EXPECT_EQUAL(gen_2 - 1, index.get_transfer_gen()); + EXPECT_EQUAL(gen_1, index.get_trim_gen()); + } + + f.set_tensor(3, spec); + generation_t gen_3 = f.get_current_gen(); + EXPECT_GREATER(gen_3, gen_2); + EXPECT_EQUAL(gen_3 - 1, index.get_transfer_gen()); + EXPECT_EQUAL(gen_3, index.get_trim_gen()); +} + TEST_MAIN() { TEST_RUN_ALL(); vespalib::unlink("test.dat"); } diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore b/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore index 35d038b0b7c..4bd94f124fb 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore +++ b/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore @@ -1 +1,2 @@ searchlib_sequencedtaskexecutor_test_app +searchlib_sequencedtaskexecutor_benchmark_app diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt index 6ba30a1647f..6c593d20683 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt +++ b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt @@ -1,4 +1,11 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_sequencedtaskexecutor_benchmark_app TEST + SOURCES + sequencedtaskexecutor_benchmark.cpp + DEPENDS + searchlib +) + vespa_add_executable(searchlib_sequencedtaskexecutor_test_app TEST SOURCES sequencedtaskexecutor_test.cpp diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp new file mode 100644 index 00000000000..a51becfbf13 --- /dev/null +++ b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp @@ -0,0 +1,24 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/vespalib/util/lambdatask.h> +#include <atomic> + +using search::SequencedTaskExecutor; +using ExecutorId = search::ISequencedTaskExecutor::ExecutorId; + +int main(int argc, char *argv[]) { + unsigned long numTasks = 1000000; + unsigned numThreads = 4; + std::atomic<long> counter(0); + if (argc > 1) + numTasks = atol(argv[1]); + if (argc > 2) + numThreads = atoi(argv[2]); + + SequencedTaskExecutor executor(numThreads); + for (unsigned long tid(0); tid < numTasks; tid++) { + executor.executeTask(ExecutorId(tid%numThreads), vespalib::makeLambdaTask([&counter] { counter++; })); + } + return 0; +} diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 56aaf2dcbc9..de436dffff1 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -9,16 +9,15 @@ #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/searchlib/attribute/singleboolattribute.h> #include <vespa/searchlib/features/agefeature.h> #include <vespa/searchlib/features/array_parser.hpp> #include <vespa/searchlib/features/attributefeature.h> -#include <vespa/searchlib/features/attributematchfeature.h> #include <vespa/searchlib/features/closenessfeature.h> #include <vespa/searchlib/features/distancefeature.h> #include <vespa/searchlib/features/dotproductfeature.h> #include <vespa/searchlib/features/fieldlengthfeature.h> #include <vespa/searchlib/features/fieldmatchfeature.h> -#include <vespa/searchlib/features/fieldtermmatchfeature.h> #include <vespa/searchlib/features/firstphasefeature.h> #include <vespa/searchlib/features/foreachfeature.h> #include <vespa/searchlib/features/freshnessfeature.h> @@ -35,7 +34,6 @@ #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/features/termfeature.h> #include <vespa/searchlib/features/utils.h> -#include <vespa/searchlib/features/valuefeature.h> #include <vespa/searchlib/features/weighted_set_parser.hpp> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/indexproperties.h> @@ -60,6 +58,7 @@ using search::AttributeFactory; using search::IntegerAttribute; using search::FloatingPointAttribute; using search::StringAttribute; +using search::SingleBoolAttribute; using search::WeightedSetStringExtAttribute; using search::attribute::WeightedEnumContent; @@ -212,7 +211,7 @@ Test::setupForAgeTest(FtFeatureTest & ft, uint64_t docTime) doctime->addReservedDoc(); doctime->addDocs(1); ft.getIndexEnv().getAttributeMap().add(doctime); - (static_cast<IntegerAttribute *>(doctime.get()))->update(1, docTime); + (dynamic_cast<IntegerAttribute *>(doctime.get()))->update(1, docTime); doctime->commit(); } @@ -240,7 +239,12 @@ Test::testAttribute() RankResult exp; exp.addScore("attribute(sint)", 10). addScore("attribute(sint,0)", 10). + addScore("attribute(slong)", 20). + addScore("attribute(sbyte)", 37). + addScore("attribute(sbool)", 1). + addScore("attribute(sebool)", 0). addScore("attribute(sfloat)", 60.5f). + addScore("attribute(sdouble)", 67.5f). addScore("attribute(sstr)", (feature_t)vespalib::hash_code("foo")). addScore("attribute(sint).count", 1). addScore("attribute(sfloat).count", 1). @@ -250,12 +254,18 @@ Test::testAttribute() addScore("attribute(udefstr)", (feature_t)vespalib::hash_code("")); FtFeatureTest ft(_factory, exp.getKeys()); - ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sfloat"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sstr"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr"); + ft.getIndexEnv().getBuilder() + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "slong") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sebool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sfloat") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sstr") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr"); setupForAttributeTest(ft); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(exp)); @@ -370,6 +380,11 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env) avs.push_back(AttributeFactory::createAttribute("udefint", AVC(AVBT::INT32, AVCT::SINGLE))); // 9 avs.push_back(AttributeFactory::createAttribute("udeffloat", AVC(AVBT::FLOAT, AVCT::SINGLE))); // 10 avs.push_back(AttributeFactory::createAttribute("udefstr", AVC(AVBT::STRING, AVCT::SINGLE))); // 11 + avs.push_back(AttributeFactory::createAttribute("sbyte", AVC(AVBT::INT64, AVCT::SINGLE))); // 12 + avs.push_back(AttributeFactory::createAttribute("slong", AVC(AVBT::INT64, AVCT::SINGLE))); // 13 + avs.push_back(AttributeFactory::createAttribute("sbool", AVC(AVBT::BOOL, AVCT::SINGLE))); // 14 + avs.push_back(AttributeFactory::createAttribute("sebool", AVC(AVBT::BOOL, AVCT::SINGLE))); // 15 + avs.push_back(AttributeFactory::createAttribute("sdouble", AVC(AVBT::DOUBLE, AVCT::SINGLE))); // 16 // simulate a unique only attribute as specified in sd AVC cfg(AVBT::INT32, AVCT::SINGLE); @@ -391,36 +406,46 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env) .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr") - .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "unique"); + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "unique") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "slong") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sebool"); } - for (uint32_t i = 0; i < avs.size(); ++i) { - avs[i]->addReservedDoc(); - avs[i]->addDocs(1); - ft.getIndexEnv().getAttributeMap().add(avs[i]); + for (const auto & attr : avs) { + attr->addReservedDoc(); + attr->addDocs(1); + ft.getIndexEnv().getAttributeMap().add(attr); } // integer attributes - (static_cast<IntegerAttribute *>(avs[0].get()))->update(1, 10); - (static_cast<IntegerAttribute *>(avs[1].get()))->append(1, 20, 0); - (static_cast<IntegerAttribute *>(avs[1].get()))->append(1, 30, 0); - (static_cast<IntegerAttribute *>(avs[2].get()))->append(1, 40, 10); - (static_cast<IntegerAttribute *>(avs[2].get()))->append(1, 50, 20); - (static_cast<IntegerAttribute *>(avs[9].get()))->update(1, search::attribute::getUndefined<int32_t>()); + (dynamic_cast<IntegerAttribute *>(avs[0].get()))->update(1, 10); + (dynamic_cast<IntegerAttribute *>(avs[12].get()))->update(1, 37); + (dynamic_cast<IntegerAttribute *>(avs[13].get()))->update(1, 20); + (dynamic_cast<SingleBoolAttribute *>(avs[14].get()))->update(1, 1); + (dynamic_cast<SingleBoolAttribute *>(avs[15].get()))->update(1, 0); + (dynamic_cast<IntegerAttribute *>(avs[1].get()))->append(1, 20, 0); + (dynamic_cast<IntegerAttribute *>(avs[1].get()))->append(1, 30, 0); + (dynamic_cast<IntegerAttribute *>(avs[2].get()))->append(1, 40, 10); + (dynamic_cast<IntegerAttribute *>(avs[2].get()))->append(1, 50, 20); + (dynamic_cast<IntegerAttribute *>(avs[9].get()))->update(1, search::attribute::getUndefined<int32_t>()); // feature_t attributes - (static_cast<FloatingPointAttribute *>(avs[3].get()))->update(1, 60.5f); - (static_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 70.5f, 0); - (static_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 80.5f, 0); - (static_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 90.5f, -30); - (static_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 100.5f, -40); - (static_cast<FloatingPointAttribute *>(avs[10].get()))->update(1, search::attribute::getUndefined<float>()); + (dynamic_cast<FloatingPointAttribute *>(avs[3].get()))->update(1, 60.5f); + (dynamic_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 70.5f, 0); + (dynamic_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 80.5f, 0); + (dynamic_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 90.5f, -30); + (dynamic_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 100.5f, -40); + (dynamic_cast<FloatingPointAttribute *>(avs[10].get()))->update(1, search::attribute::getUndefined<float>()); + (dynamic_cast<FloatingPointAttribute *>(avs[16].get()))->update(1, 67.5); // string attributes - (static_cast<StringAttribute *>(avs[6].get()))->update(1, "foo"); - (static_cast<StringAttribute *>(avs[7].get()))->append(1, "bar", 0); - (static_cast<StringAttribute *>(avs[7].get()))->append(1, "baz", 0); - (static_cast<StringAttribute *>(avs[8].get()))->append(1, "qux", 11); - (static_cast<StringAttribute *>(avs[8].get()))->append(1, "quux", 12); - (static_cast<StringAttribute *>(avs[11].get()))->update(1, ""); + (dynamic_cast<StringAttribute *>(avs[6].get()))->update(1, "foo"); + (dynamic_cast<StringAttribute *>(avs[7].get()))->append(1, "bar", 0); + (dynamic_cast<StringAttribute *>(avs[7].get()))->append(1, "baz", 0); + (dynamic_cast<StringAttribute *>(avs[8].get()))->append(1, "qux", 11); + (dynamic_cast<StringAttribute *>(avs[8].get()))->append(1, "quux", 12); + (dynamic_cast<StringAttribute *>(avs[11].get()))->update(1, ""); for (uint32_t i = 0; i < avs.size() - 1; ++i) { // do not commit the noupdate attribute avs[i]->commit(); @@ -475,7 +500,7 @@ Test::assertCloseness(feature_t exp, const vespalib::string & attr, double dista FtFeatureTest ft(_factory, feature); std::vector<std::pair<int32_t, int32_t> > positions; int32_t x = 0; - positions.push_back(std::make_pair(x, x)); + positions.emplace_back(x, x); setupForDistanceTest(ft, "pos", positions, false); ft.getQueryEnv().getLocation().setXPosition((int)distance); ft.getQueryEnv().getLocation().setValid(true); @@ -572,7 +597,7 @@ Test::assertFieldMatch(const vespalib::string & spec, const vespalib::string & field, uint32_t totalTermWeight) { - assertFieldMatch(spec, query, field, NULL, totalTermWeight); + assertFieldMatch(spec, query, field, nullptr, totalTermWeight); } void @@ -581,7 +606,7 @@ Test::assertFieldMatchTS(const vespalib::string & spec, const vespalib::string & field, feature_t totalSignificance) { - assertFieldMatch(spec, query, field, NULL, 0, totalSignificance); + assertFieldMatch(spec, query, field, nullptr, 0, totalSignificance); } @@ -871,12 +896,12 @@ Test::setupForDistanceTest(FtFeatureTest &ft, const vespalib::string & attrName, pos->addDocs(1); ft.getIndexEnv().getAttributeMap().add(pos); - IntegerAttribute * ia = static_cast<IntegerAttribute *>(pos.get()); - for (uint32_t i = 0; i < positions.size(); ++i) { + auto ia = dynamic_cast<IntegerAttribute *>(pos.get()); + for (const auto & p : positions) { if (zcurve) { - ia->append(1, vespalib::geo::ZCurve::encode(positions[i].first, positions[i].second), 0); + ia->append(1, vespalib::geo::ZCurve::encode(p.first, p.second), 0); } else { - ia->append(1, positions[i].first, 0); + ia->append(1, p.first, 0); } } @@ -891,11 +916,11 @@ Test::assert2DZDistance(feature_t exp, const vespalib::string & positions, FtFeatureTest ft(_factory, "distance(pos)"); std::vector<vespalib::string> ta = FtUtil::tokenize(positions, ","); std::vector<std::pair<int32_t, int32_t> > pos; - for (uint32_t i = 0; i < ta.size(); ++i) { - std::vector<vespalib::string> tb = FtUtil::tokenize(ta[i], ":"); - int32_t x = util::strToNum<int32_t>(tb[0]); - int32_t y = util::strToNum<int32_t>(tb[1]); - pos.push_back(std::make_pair(x, y)); + for (const auto & s : ta) { + std::vector<vespalib::string> tb = FtUtil::tokenize(s, ":"); + auto x = util::strToNum<int32_t>(tb[0]); + auto y = util::strToNum<int32_t>(tb[1]); + pos.emplace_back(x, y); } setupForDistanceTest(ft, "pos", pos, true); ft.getQueryEnv().getLocation().setXPosition(xquery); @@ -927,7 +952,7 @@ Test::testDistanceToPath() { // Test executor. std::vector<std::pair<int32_t, int32_t> > pos; - pos.push_back(std::make_pair(0, 0)); + pos.emplace_back(0, 0); // invalid path assertDistanceToPath(pos, "a"); @@ -965,7 +990,7 @@ Test::testDistanceToPath() assertDistanceToPath(pos, "(-3,2,2,2,2,-1,0,-1)", 1, 1, 2); // multiple document locations - pos.push_back(std::make_pair(0, 1)); + pos.emplace_back(0, 1); assertDistanceToPath(pos, "(-1,1,1,1)", 0, 0.5); assertDistanceToPath(pos, "(-2,-1,-1,1)", 1, 1, 2); assertDistanceToPath(pos, "(-1,0.25,1,0.25)", 0.25, 0.5, 0.5); @@ -1017,7 +1042,7 @@ Test::testDistanceToPath() } void -Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, +Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > & pos, const vespalib::string &path, feature_t distance, feature_t traveled, feature_t product) { LOG(info, "Testing distance to path '%s' with %zd document locations.", path.c_str(), pos.size()); @@ -1033,20 +1058,6 @@ Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, .addScore("distanceToPath(pos).product", product))); } -void -Test::setupForDocumentTest(FtFeatureTest &ft, const vespalib::string & attrName, const vespalib::string & docType) -{ - AttributePtr type = AttributeFactory::createAttribute(attrName, AVC(AVBT::STRING, AVCT::SINGLE)); - - type->addReservedDoc(); - type->addDocs(1); - ft.getIndexEnv().getAttributeMap().add(type); - - (static_cast<StringAttribute *>(type.get()))->update(1, docType); - type->commit(); -} - - namespace { void @@ -1264,6 +1275,8 @@ void Test::setupForDotProductTest(FtFeatureTest & ft) { struct Config { + Config() : name(nullptr), dataType(AVBT::BOOL), collectionType(AVCT::SINGLE), fastSearch(false) {} + Config(const char *n, AVBT dt, AVCT ct, bool fs) : name(n), dataType(dt), collectionType(ct), fastSearch(fs) {} const char * name; AVBT dataType; AVCT collectionType; @@ -1296,11 +1309,11 @@ Test::setupForDotProductTest(FtFeatureTest & ft) baf->addDocs(2); ft.getIndexEnv().getAttributeMap().add(baf); for (size_t i(1); i < 6; i++) { - IntegerAttribute * ia = dynamic_cast<IntegerAttribute *>(baf.get()); + auto ia = dynamic_cast<IntegerAttribute *>(baf.get()); if (ia) { ia->append(1, i, i); } else { - FloatingPointAttribute * fa = dynamic_cast<FloatingPointAttribute *>(baf.get()); + auto fa = dynamic_cast<FloatingPointAttribute *>(baf.get()); fa->append(1, i, i); } } @@ -1315,14 +1328,14 @@ Test::setupForDotProductTest(FtFeatureTest & ft) ft.getIndexEnv().getAttributeMap().add(c); ft.getIndexEnv().getAttributeMap().add(d); - StringAttribute * sa = static_cast<StringAttribute *>(a.get()); + auto sa = dynamic_cast<StringAttribute *>(a.get()); sa->append(1, "a", 1); sa->append(1, "b", 2); sa->append(1, "c", 3); sa->append(1, "d", 4); sa->append(1, "e", 5); - WeightedSetStringExtAttribute * ea = static_cast<WeightedSetStringExtAttribute *>(d.get()); + auto ea = dynamic_cast<WeightedSetStringExtAttribute *>(d.get()); EXPECT_TRUE(!ea->hasEnum()); uint32_t docId; ea->addDoc(docId); // reserved doc @@ -1517,9 +1530,9 @@ Test::testMatchCount() ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz"); - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 0, hit in foo - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL); // query term 1, hit in bar - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 2, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 0, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != nullptr); // query term 1, hit in bar + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 2, hit in foo ASSERT_TRUE(ft.setup()); MatchDataBuilder::UP mdb = ft.createMatchDataBuilder(); @@ -1577,9 +1590,9 @@ Test::testMatches() ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz"); - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 0, hit in foo - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL); // query term 1, hit in bar - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 2, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 0, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != nullptr); // query term 1, hit in bar + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 2, hit in foo ASSERT_TRUE(ft.setup()); MatchDataBuilder::UP mdb = ft.createMatchDataBuilder(); @@ -1613,11 +1626,9 @@ Test::assertMatches(uint32_t output, ASSERT_TRUE(ft.execute(output, EPS, docId)); // Execute and compare results. - if (!EXPECT_TRUE(ft.execute(output, EPS, docId))) return false; - return true; + return EXPECT_TRUE(ft.execute(output, EPS, docId)); } - void Test::testQuery() { @@ -1721,7 +1732,7 @@ Test::testRandom() search::Rand48 rnd; rnd.srand48(100); for (uint32_t i = 0; i < 5; ++i) { - feature_t exp = rnd.lrand48() / (feature_t)0x80000000u; + feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u); ASSERT_TRUE(ft.execute(exp, EPS, i + 1)); } } @@ -1744,7 +1755,7 @@ Test::testRandom() search::Rand48 rnd; for (uint32_t i = 1; i <= 5; ++i) { rnd.srand48(100 + i); // seed + lid - feature_t exp = rnd.lrand48() / (feature_t)0x80000000u; + feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u); ASSERT_TRUE(ft.execute(exp, EPS, i)); } } @@ -2067,10 +2078,7 @@ Test::assertTermDistance(const TermDistanceCalculator::Result & exp, rr.addScore(feature + ".forwardTermPosition", exp.forwardTermPos); rr.addScore(feature + ".reverse", exp.reverseDist); rr.addScore(feature + ".reverseTermPosition", exp.reverseTermPos); - if (!EXPECT_TRUE(ft.execute(rr, docId))) { - return false; - } - return true; + return EXPECT_TRUE(ft.execute(rr, docId)); } void diff --git a/searchlib/src/tests/features/prod_features.h b/searchlib/src/tests/features/prod_features.h index 8e6578f34bd..6d30dd3fcd8 100644 --- a/searchlib/src/tests/features/prod_features.h +++ b/searchlib/src/tests/features/prod_features.h @@ -10,10 +10,10 @@ class Test : public FtTestApp { public: Test(); - ~Test(); + ~Test() override; int Main() override; void testFramework(); - void testFtLib(); + static void testFtLib(); void testAge(); void testAttribute(); void testAttributeMatch(); @@ -39,10 +39,9 @@ public: void testRankingExpression(); void testTerm(); void testTermDistance(); - void testUtils(); + static void testUtils(); static void setupForDotProductTest(FtFeatureTest & ft); - static void setupForDocumentTest(FtFeatureTest &ft, const vespalib::string & attrName, const vespalib::string & docType); private: void testFieldMatchBluePrint(); @@ -81,21 +80,21 @@ private: void testFieldMatchExecutorRemaining(); void assertAge(feature_t expAge, const vespalib::string & attr, uint64_t now, uint64_t docTime); - void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime); - void setupForAttributeTest(FtFeatureTest &ft, bool setup_env = true); + static void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime); + static void setupForAttributeTest(FtFeatureTest &ft, bool setup_env = true); void assertCloseness(feature_t exp, const vespalib::string & attr, double distance, double maxDistance = 0, double halfResponse = 0); - void setupForDistanceTest(FtFeatureTest & ft, const vespalib::string & attrName, - const std::vector<std::pair<int32_t, int32_t> > & positions, bool zcurve); + static void setupForDistanceTest(FtFeatureTest & ft, const vespalib::string & attrName, + const std::vector<std::pair<int32_t, int32_t> > & positions, bool zcurve); void assert2DZDistance(feature_t exp, const vespalib::string & positions, int32_t xquery, int32_t yquery, uint32_t xAspect = 0); - void assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, const vespalib::string &path, + void assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > & pos, const vespalib::string &path, feature_t distance = search::features::DistanceToPathExecutor::DEFAULT_DISTANCE, feature_t traveled = 1, feature_t product = 0); void assertDotProduct(feature_t exp, const vespalib::string & vector, uint32_t docId = 1, const vespalib::string & attribute = "wsstr", const vespalib::string & attributeOverride=""); void assertFieldMatch(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field, - const search::features::fieldmatch::Params * params = NULL, uint32_t totalTermWeight = 0, feature_t totalSignificance = 0.0f); + const search::features::fieldmatch::Params * params = nullptr, uint32_t totalTermWeight = 0, feature_t totalSignificance = 0.0f); void assertFieldMatch(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field, uint32_t totalTermWeight); void assertFieldMatchTS(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field, diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index 691e80aeb9f..ee8d4b787bd 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -195,7 +195,7 @@ TEST("require that NnsIndexIterator works as expected") { std::vector<NnsIndexIterator::Hit> hits{{2,4.0}, {3,9.0}, {5,1.0}, {8,16.0}, {9,36.0}}; auto md = MatchData::makeTestInstance(2, 2); auto &tfmd = *(md->resolveTermField(0)); - auto search = NnsIndexIterator::create(true, tfmd, hits); + auto search = NnsIndexIterator::create(tfmd, hits); uint32_t docid = 1; search->initFullRange(); bool match = search->seek(docid); diff --git a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp index b10da86dd8c..861af3527ca 100644 --- a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp +++ b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/vespalib/util/stash.h> #include <set> using namespace search::features::rankingexpression; diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index 1204ae1e9bc..37c4d02017f 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -6,11 +6,14 @@ #include <vespa/searchlib/tensor/hnsw_index.h> #include <vespa/searchlib/tensor/random_level_generator.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/generationhandler.h> #include <vector> #include <vespa/log/log.h> LOG_SETUP("hnsw_index_test"); +using vespalib::GenerationHandler; +using vespalib::MemoryUsage; using namespace search::tensor; template <typename FloatType> @@ -49,11 +52,13 @@ class HnswIndexTest : public ::testing::Test { public: FloatVectors vectors; LevelGenerator* level_generator; + GenerationHandler gen_handler; HnswIndexUP index; HnswIndexTest() : vectors(), level_generator(), + gen_handler(), index() { vectors.set(1, {2, 2}).set(2, {3, 2}).set(3, {2, 3}) @@ -70,9 +75,23 @@ public: void add_document(uint32_t docid, uint32_t max_level = 0) { level_generator->level = max_level; index->add_document(docid); + commit(); } void remove_document(uint32_t docid) { index->remove_document(docid); + commit(); + } + void commit() { + index->transfer_hold_lists(gen_handler.getCurrentGeneration()); + gen_handler.incGeneration(); + gen_handler.updateFirstUsedGeneration(); + index->trim_hold_lists(gen_handler.getFirstUsedGeneration()); + } + GenerationHandler::Guard take_read_guard() { + return gen_handler.takeGuard(); + } + MemoryUsage memory_usage() const { + return index->memory_usage(); } void expect_entry_point(uint32_t exp_docid, uint32_t exp_level) { EXPECT_EQ(exp_docid, index->get_entry_docid()); @@ -83,6 +102,10 @@ public: ASSERT_EQ(1, node.size()); EXPECT_EQ(exp_links, node.level(0)); } + void expect_empty_level_0(uint32_t docid) { + auto node = index->get_node(docid); + EXPECT_TRUE(node.empty()); + } void expect_levels(uint32_t docid, const HnswNode::LevelArray& exp_levels) { auto act_node = index->get_node(docid); ASSERT_EQ(exp_levels.size(), act_node.size()); @@ -266,5 +289,83 @@ TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_sel expect_level_0(7, {3, 6}); } +TEST_F(HnswIndexTest, manual_insert) +{ + init(false); + + std::vector<uint32_t> nbl; + HnswNode empty{nbl}; + index->set_node(1, empty); + index->set_node(2, empty); + + HnswNode three{{1,2}}; + index->set_node(3, three); + expect_level_0(1, {3}); + expect_level_0(2, {3}); + expect_level_0(3, {1,2}); + + expect_entry_point(1, 0); + + HnswNode twolevels{{{1},nbl}}; + index->set_node(4, twolevels); + + expect_entry_point(4, 1); + expect_level_0(1, {3,4}); + + HnswNode five{{{1,2}, {4}}}; + index->set_node(5, five); + + expect_levels(1, {{3,4,5}}); + expect_levels(2, {{3,5}}); + expect_levels(3, {{1,2}}); + expect_levels(4, {{1}, {5}}); + expect_levels(5, {{1,2}, {4}}); +} + +TEST_F(HnswIndexTest, memory_is_reclaimed_when_doing_changes_to_graph) +{ + init(true); + + add_document(1); + add_document(3); + auto mem_1 = memory_usage(); + + add_document(2); + expect_level_0(1, {2,3}); + expect_level_0(2, {1,3}); + expect_level_0(3, {1,2}); + auto mem_2 = memory_usage(); + // We should use more memory with larger link arrays and extra document. + EXPECT_GT((mem_2.usedBytes() - mem_2.deadBytes()), (mem_1.usedBytes() - mem_1.deadBytes())); + EXPECT_EQ(0, mem_2.allocatedBytesOnHold()); + + remove_document(2); + expect_level_0(1, {3}); + expect_empty_level_0(2); + expect_level_0(3, {1}); + auto mem_3 = memory_usage(); + // We end up in the same state as before document 2 was added and effectively use the same amount of memory. + EXPECT_EQ((mem_1.usedBytes() - mem_1.deadBytes()), (mem_3.usedBytes() - mem_3.deadBytes())); + EXPECT_EQ(0, mem_3.allocatedBytesOnHold()); +} + +TEST_F(HnswIndexTest, memory_is_put_on_hold_while_read_guard_is_held) +{ + init(true); + + add_document(1); + add_document(3); + { + auto guard = take_read_guard(); + add_document(2); + auto mem = memory_usage(); + // As read guard is held memory to reclaim is put on hold + EXPECT_GT(mem.allocatedBytesOnHold(), 0); + } + commit(); + auto mem = memory_usage(); + EXPECT_EQ(0, mem.allocatedBytesOnHold()); +} + GTEST_MAIN_RUN_ALL_TESTS() |