diff options
Diffstat (limited to 'searchlib/src/tests')
8 files changed, 241 insertions, 122 deletions
diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 5148ab2be34..32ac836302a 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -1297,6 +1297,9 @@ AttributeTest::testWeightedSet() testWeightedSet<IntegerAttribute, AttributeVector::WeightedInt>(ptr, values); IAttributeVector::EnumHandle e; EXPECT_TRUE(ptr->findEnum("1", e)); + EXPECT_EQUAL(1u, ptr->findFoldedEnums("1").size()); + EXPECT_EQUAL(e, ptr->findFoldedEnums("1")[0]); + } } { // FloatingPointAttribute @@ -1320,6 +1323,8 @@ AttributeTest::testWeightedSet() testWeightedSet<FloatingPointAttribute, AttributeVector::WeightedFloat>(ptr, values); IAttributeVector::EnumHandle e; EXPECT_TRUE(ptr->findEnum("1", e)); + EXPECT_EQUAL(1u, ptr->findFoldedEnums("1").size()); + EXPECT_EQUAL(e, ptr->findFoldedEnums("1")[0]); } } { // StringAttribute @@ -1345,6 +1350,8 @@ AttributeTest::testWeightedSet() testWeightedSet<StringAttribute, AttributeVector::WeightedString>(ptr, values); IAttributeVector::EnumHandle e; EXPECT_TRUE(ptr->findEnum("string00", e)); + EXPECT_EQUAL(1u, ptr->findFoldedEnums("StRiNg00").size()); + EXPECT_EQUAL(e, ptr->findFoldedEnums("StRiNg00")[0]); } } } diff --git a/searchlib/src/tests/attribute/benchmark/attributesearcher.h b/searchlib/src/tests/attribute/benchmark/attributesearcher.h index f8cd614c48c..37f33803d27 100644 --- a/searchlib/src/tests/attribute/benchmark/attributesearcher.h +++ b/searchlib/src/tests/attribute/benchmark/attributesearcher.h @@ -126,7 +126,7 @@ AttributeFindSearcher<T>::doRun() // build simple term query vespalib::asciistream ss; ss << _values[i % _values.size()].getValue(); - this->buildTermQuery(_query, _attrPtr->getName(), ss.str().c_str()); + this->buildTermQuery(_query, _attrPtr->getName(), ss.str().data()); AttributeGuard guard(_attrPtr); std::unique_ptr<AttributeVector::SearchContext> searchContext = @@ -204,7 +204,7 @@ AttributeRangeSearcher::doRun() // build simple range term query vespalib::asciistream ss; ss << "[" << iter.a() << ";" << iter.b() << "]"; - buildTermQuery(_query, _attrPtr->getName(), ss.str().c_str()); + buildTermQuery(_query, _attrPtr->getName(), ss.str().data()); AttributeGuard guard(_attrPtr); std::unique_ptr<AttributeVector::SearchContext> searchContext = diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index daff432d68d..89dd1cfdab4 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -41,6 +41,7 @@ private: void testFloatEnumStore(EnumStoreType & es); void testFloatEnumStore(); + void testFindFolded(); void testAddEnum(); template <typename EnumStoreType> void testAddEnum(bool hasPostings); @@ -275,6 +276,40 @@ EnumStoreTest::testFloatEnumStore() } void +EnumStoreTest::testFindFolded() +{ + StringEnumStore ses(100, false); + std::vector<EnumIndex> indices; + std::vector<std::string> unique({"", "one", "two", "TWO", "Two", "three"}); + for (std::string &str : unique) { + EnumIndex idx; + ses.addEnum(str.c_str(), idx); + EXPECT_TRUE(ses.getLastEnum() == indices.size()); + indices.push_back(idx); + ses.incRefCount(idx); + EXPECT_EQUAL(1u, ses.getRefCount(idx)); + } + ses.freezeTree(); + for (uint32_t i = 0; i < indices.size(); ++i) { + uint32_t e = ses.getEnum(indices[i]); + EXPECT_EQUAL(i, e); + EnumIndex idx; + EXPECT_TRUE(ses.findIndex(unique[i].c_str(), idx)); + } + EXPECT_EQUAL(1u, ses.findFoldedEnums("").size()); + EXPECT_EQUAL(0u, ses.findFoldedEnums("foo").size()); + EXPECT_EQUAL(1u, ses.findFoldedEnums("one").size()); + EXPECT_EQUAL(3u, ses.findFoldedEnums("two").size()); + EXPECT_EQUAL(3u, ses.findFoldedEnums("TWO").size()); + EXPECT_EQUAL(3u, ses.findFoldedEnums("tWo").size()); + const auto v = ses.findFoldedEnums("Two"); + EXPECT_EQUAL(std::string("TWO"), ses.getValue(v[0])); + EXPECT_EQUAL(std::string("Two"), ses.getValue(v[1])); + EXPECT_EQUAL(std::string("two"), ses.getValue(v[2])); + EXPECT_EQUAL(1u, ses.findFoldedEnums("three").size()); +} + +void EnumStoreTest::testAddEnum() { testAddEnum<StringEnumStore>(false); @@ -319,6 +354,8 @@ EnumStoreTest::testAddEnum(bool hasPostings) uint32_t e = ses.getEnum(indices[i]); EXPECT_EQUAL(i, e); EXPECT_TRUE(ses.findEnum(unique[i].c_str(), e)); + EXPECT_EQUAL(1u, ses.findFoldedEnums(unique[i].c_str()).size()); + EXPECT_EQUAL(e, ses.findFoldedEnums(unique[i].c_str())[0]); EXPECT_TRUE(ses.getEnum(datastore::EntryRef(e)) == i); EXPECT_TRUE(ses.findIndex(unique[i].c_str(), idx)); EXPECT_TRUE(idx == indices[i]); @@ -874,6 +911,7 @@ EnumStoreTest::Main() testStringEntry(); testNumericEntry(); testFloatEnumStore(); + testFindFolded(); testAddEnum(); testCompaction(); testReset(); diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp index 0a02824e77a..f8d696b3203 100644 --- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp +++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp @@ -348,7 +348,7 @@ TEST_F("Multiple iterators can be created from the same context", SingleValueFix // implemented at all for (single) numeric attributes. Intentional? TEST_F("queryTerm() returns term context was created with", WsetValueFixture) { auto ctx = f.create_context(word_term("helloworld")); - EXPECT_EQUAL(std::string("helloworld"), std::string(ctx->queryTerm().getTerm())); + EXPECT_EQUAL(std::string("helloworld"), std::string(ctx->queryTerm()->getTerm())); } struct SearchCacheFixture : Fixture { diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index b25abdfd2e5..2adfdd135df 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -153,6 +153,8 @@ StringAttributeTest::testMultiValue(Attribute & attr, uint32_t numDocs) EXPECT_TRUE(strcmp(attr.get(doc), uniqueStrings[0].c_str()) == 0); uint32_t e; EXPECT_TRUE(attr.findEnum(uniqueStrings[0].c_str(), e)); + EXPECT_EQUAL(1u, attr.findFoldedEnums(uniqueStrings[0].c_str()).size()); + EXPECT_EQUAL(e, attr.findFoldedEnums(uniqueStrings[0].c_str())[0]); EXPECT_TRUE(attr.getEnum(doc) == e); } diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index a22c44e843c..34046f551d9 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -282,13 +282,13 @@ TEST("testTruncatedIdxFile"){ { // Files comes from the 'growing test'. LogDataStore datastore(executor, TEST_PATH("bug-7257706"), config, GrowStrategy(), - TuneFileSummary(), fileHeaderContext, tlSyncer, NULL); + TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); EXPECT_EQUAL(354ul, datastore.lastSyncToken()); } const char * magic = "mumbo jumbo"; { LogDataStore datastore(executor, "bug-7257706-truncated", config, GrowStrategy(), - TuneFileSummary(), fileHeaderContext, tlSyncer, NULL); + TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); EXPECT_EQUAL(331ul, datastore.lastSyncToken()); datastore.write(332, 7, magic, strlen(magic)); datastore.write(333, 8, magic, strlen(magic)); @@ -296,7 +296,7 @@ TEST("testTruncatedIdxFile"){ } { LogDataStore datastore(executor, "bug-7257706-truncated", config, GrowStrategy(), - TuneFileSummary(), fileHeaderContext, tlSyncer, NULL); + TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); EXPECT_EQUAL(334ul, datastore.lastSyncToken()); } } @@ -308,7 +308,7 @@ TEST("testThatEmptyIdxFilesAndDanglingDatFilesAreRemoved") { MyTlSyncer tlSyncer; LogDataStore datastore(executor, "dangling-test", config, GrowStrategy(), TuneFileSummary(), - fileHeaderContext, tlSyncer, NULL); + fileHeaderContext, tlSyncer, nullptr); EXPECT_EQUAL(354ul, datastore.lastSyncToken()); EXPECT_EQUAL(4096u + 480u, datastore.getDiskHeaderFootprint()); EXPECT_EQUAL(datastore.getDiskHeaderFootprint() + 94016u, datastore.getDiskFootprint()); @@ -321,7 +321,7 @@ TEST("testThatIncompleteCompactedFilesAreRemoved") { MyTlSyncer tlSyncer; LogDataStore datastore(executor, "incompletecompact-test", config, GrowStrategy(), TuneFileSummary(), - fileHeaderContext, tlSyncer, NULL); + fileHeaderContext, tlSyncer, nullptr); EXPECT_EQUAL(354ul, datastore.lastSyncToken()); EXPECT_EQUAL(3*(4096u + 480u), datastore.getDiskHeaderFootprint()); LogDataStore::NameIdSet files = datastore.getAllActiveFiles(); @@ -340,7 +340,7 @@ public: _executor(1, 128*1024), _tlSyncer(), _datastore(_executor, _myDir.getDir(), _config, GrowStrategy(), - TuneFileSummary(), _fileHeaderContext, _tlSyncer, NULL) + TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr) { } ~VisitStore(); IDataStore & getStore() { return _datastore; } @@ -414,7 +414,7 @@ makeDocTypeRepoConfig() Document::UP -makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool extra_field) +makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool extra_field, size_t numReps=0) { asciistream idstr; idstr << "id:test:test:: " << i; @@ -424,7 +424,7 @@ makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool extra_field) ASSERT_TRUE(doc.get()); asciistream mainstr; mainstr << "static text" << i << " body something"; - for (uint32_t j = 0; j < 10; ++j) { + for (uint32_t j = 0; j < 10+numReps; ++j) { mainstr << (j + i * 1000) << " "; } mainstr << " and end field"; @@ -432,7 +432,6 @@ makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool extra_field) if (extra_field) { doc->set("extra", "foo"); } - return doc; } @@ -440,11 +439,15 @@ makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool extra_field) class VisitCacheStore { public: - VisitCacheStore(); + using UpdateStrategy=DocumentStore::Config::UpdateStrategy; + VisitCacheStore(UpdateStrategy strategy); ~VisitCacheStore(); - IDocumentStore & getStore() { return _datastore; } + IDocumentStore & getStore() { return *_datastore; } void write(uint32_t id) { - write(id, makeDoc(_repo, id, true)); + write(id, 0); + } + void write(uint32_t lid, uint32_t numReps) { + write(lid, makeDoc(_repo, lid, true, numReps)); } void rewrite(uint32_t id) { write(id, makeDoc(_repo, id, false)); @@ -458,7 +461,10 @@ public: _inserted.erase(id); } void verifyRead(uint32_t id) { - verifyDoc(*_datastore.read(id, _repo), id); + verifyDoc(*_datastore->read(id, _repo), id); + } + void read(uint32_t id) { + *_datastore->read(id, _repo); } void verifyDoc(const Document & doc, uint32_t id) { EXPECT_TRUE(doc == *_inserted[id]); @@ -468,8 +474,10 @@ public: } void verifyVisit(const std::vector<uint32_t> & lids, const std::vector<uint32_t> & expected, bool allowCaching) { VerifyVisitor vv(*this, expected, allowCaching); - _datastore.visit(lids, _repo, vv); + _datastore->visit(lids, _repo, vv); } + void recreate(); + private: class VerifyVisitor : public IDocumentVisitor { public: @@ -494,7 +502,7 @@ private: DummyFileHeaderContext _fileHeaderContext; vespalib::ThreadStackExecutor _executor; MyTlSyncer _tlSyncer; - LogDocumentStore _datastore; + std::unique_ptr<LogDocumentStore> _datastore; std::map<uint32_t, Document::UP> _inserted; SerialNum _serial; }; @@ -510,21 +518,33 @@ VisitCacheStore::VerifyVisitor::~VerifyVisitor() { EXPECT_EQUAL(_expected.size(), _actual.size()); } -VisitCacheStore::VisitCacheStore() : + +VisitCacheStore::VisitCacheStore(UpdateStrategy strategy) : _myDir("visitcache"), _repo(makeDocTypeRepoConfig()), - _config(DocumentStore::Config(CompressionConfig::LZ4, 1000000, 0).allowVisitCaching(true), + _config(DocumentStore::Config(CompressionConfig::LZ4, 1000000, 0) + .allowVisitCaching(true).updateStrategy(strategy), LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0) .setFileConfig(WriteableFileChunk::Config(CompressionConfig(), 16384))), _fileHeaderContext(), _executor(1, 128*1024), _tlSyncer(), - _datastore(_executor, _myDir.getDir(), _config, GrowStrategy(), - TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr), + _datastore(std::make_unique<LogDocumentStore>(_executor, _myDir.getDir(), _config, GrowStrategy(), + TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr)), _inserted(), _serial(1) { } -VisitCacheStore::~VisitCacheStore() {} + +VisitCacheStore::~VisitCacheStore() = default; + +void +VisitCacheStore::recreate() { + _datastore->flush(_datastore->initFlush(_datastore->tentativeLastSyncToken())); + _datastore.reset(); + _datastore = std::make_unique<LogDocumentStore>(_executor, _myDir.getDir(), _config, GrowStrategy(), + TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr); + +} void verifyCacheStats(CacheStats cs, size_t hits, size_t misses, size_t elements, size_t memory_used) { @@ -535,8 +555,64 @@ verifyCacheStats(CacheStats cs, size_t hits, size_t misses, size_t elements, siz EXPECT_GREATER_EQUAL(memory_used+20, cs.memory_used); } +TEST("test the update cache strategy") { + VisitCacheStore vcs(DocumentStore::Config::UpdateStrategy::UPDATE); + IDocumentStore & ds = vcs.getStore(); + for (size_t i(1); i <= 10; i++) { + vcs.write(i); + } + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 0, 0, 0)); + vcs.verifyRead(7); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 1, 1, 221)); + vcs.write(8); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 1, 1, 221)); + vcs.write(7, 17); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 1, 1, 282)); + vcs.verifyRead(7); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 1, 1, 1, 282)); + vcs.remove(8); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 1, 1, 1, 282)); + vcs.remove(7); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 1, 1, 0, 0)); + vcs.write(7); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 1, 1, 0, 0)); + vcs.verifyRead(7); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 1, 2, 1, 221)); + vcs.write(7, 17); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 1, 2, 1, 282)); + vcs.recreate(); + IDocumentStore & ds2 = vcs.getStore(); + vcs.verifyRead(7); + TEST_DO(verifyCacheStats(ds2.getCacheStats(), 0, 1, 1, 282)); +} + +TEST("test the invalidate cache strategy") { + VisitCacheStore vcs(DocumentStore::Config::UpdateStrategy::INVALIDATE); + IDocumentStore & ds = vcs.getStore(); + for (size_t i(1); i <= 10; i++) { + vcs.write(i); + } + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 0, 0, 0)); + vcs.verifyRead(7); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 1, 1, 221)); + vcs.write(8); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 1, 1, 221)); + vcs.write(7); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 1, 0, 0)); + vcs.verifyRead(7); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 2, 1, 221)); + vcs.remove(8); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 2, 1, 221)); + vcs.remove(7); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 2, 0, 0)); + vcs.write(7); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 2, 0, 0)); + vcs.verifyRead(7); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 3, 1, 221)); +} + TEST("test that the integrated visit cache works.") { - VisitCacheStore vcs; + VisitCacheStore vcs(DocumentStore::Config::UpdateStrategy::INVALIDATE); IDocumentStore & ds = vcs.getStore(); for (size_t i(1); i <= 100; i++) { vcs.write(i); @@ -546,41 +622,42 @@ TEST("test that the integrated visit cache works.") { for (size_t i(1); i <= 100; i++) { vcs.verifyRead(i); } - TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 100, 100, 20574)); + constexpr size_t BASE_SZ = 21374; + TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 100, 100, BASE_SZ)); for (size_t i(1); i <= 100; i++) { vcs.verifyRead(i); } - TEST_DO(verifyCacheStats(ds.getCacheStats(), 100, 100, 100, 20574)); // From the individual cache. + TEST_DO(verifyCacheStats(ds.getCacheStats(), 100, 100, 100, BASE_SZ)); // From the individual cache. vcs.verifyVisit({7,9,17,19,67,88}, false); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 100, 100, 100, 20574)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 100, 100, 100, BASE_SZ)); vcs.verifyVisit({7,9,17,19,67,88}, true); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 100, 101, 101, 21135)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 100, 101, 101, BASE_SZ+557)); vcs.verifyVisit({7,9,17,19,67,88}, true); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 101, 101, 21135)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 101, 101, BASE_SZ+557)); vcs.rewrite(8); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 101, 100, 20922)); // From the individual cache. + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 101, 100, BASE_SZ+328)); // From the individual cache. vcs.rewrite(7); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 101, 98, 20148)); // From the both caches. + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 101, 98, BASE_SZ-442)); // From the both caches. vcs.verifyVisit({7,9,17,19,67,88}, true); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 102, 99, 20732)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 102, 99, BASE_SZ+130)); vcs.verifyVisit({7,9,17,19,67,88,89}, true); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 103, 99, 20783)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 103, 99, BASE_SZ+201)); vcs.rewrite(17); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 103, 97, 19943)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 103, 97, BASE_SZ-657)); vcs.verifyVisit({7,9,17,19,67,88,89}, true); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 104, 98, 20587)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 104, 98, BASE_SZ-3)); vcs.remove(17); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 104, 97, 19943)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 104, 97, BASE_SZ-657)); vcs.verifyVisit({7,9,17,19,67,88,89}, {7,9,19,67,88,89}, true); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 105, 98, 20526)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 105, 98, BASE_SZ-64)); vcs.verifyVisit({41, 42}, true); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 106, 99, 20820)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 106, 99, BASE_SZ+238)); vcs.verifyVisit({43, 44}, true); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 107, 100, 21124)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 107, 100, BASE_SZ+540)); vcs.verifyVisit({41, 42, 43, 44}, true); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 108, 99, 20944)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 108, 99, BASE_SZ+362)); } TEST("testWriteRead") { @@ -595,7 +672,7 @@ TEST("testWriteRead") { vespalib::ThreadStackExecutor executor(1, 128*1024); MyTlSyncer tlSyncer; LogDataStore datastore(executor, "empty", config, GrowStrategy(), - TuneFileSummary(), fileHeaderContext, tlSyncer, NULL); + TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); ASSERT_TRUE(datastore.lastSyncToken() == 0); size_t headerFootprint = datastore.getDiskHeaderFootprint(); EXPECT_LESS(0u, headerFootprint); @@ -606,7 +683,7 @@ TEST("testWriteRead") { fetchAndTest(datastore, 0, a[0].c_str(), a[0].size()); datastore.write(2, 0, a[1].c_str(), a[1].size()); fetchAndTest(datastore, 0, a[1].c_str(), a[1].size()); - fetchAndTest(datastore, 1, NULL, 0); + fetchAndTest(datastore, 1, nullptr, 0); datastore.remove(3, 0); fetchAndTest(datastore, 0, "", 0); @@ -632,7 +709,7 @@ TEST("testWriteRead") { MyTlSyncer tlSyncer; LogDataStore datastore(executor, "empty", config, GrowStrategy(), TuneFileSummary(), - fileHeaderContext, tlSyncer, NULL); + fileHeaderContext, tlSyncer, nullptr); size_t headerFootprint = datastore.getDiskHeaderFootprint(); EXPECT_LESS(0u, headerFootprint); EXPECT_EQUAL(4944ul + headerFootprint, datastore.getDiskFootprint()); diff --git a/searchlib/src/tests/hitcollector/hitcollector_test.cpp b/searchlib/src/tests/hitcollector/hitcollector_test.cpp index 3f49c6969a0..f9f977f4093 100644 --- a/searchlib/src/tests/hitcollector/hitcollector_test.cpp +++ b/searchlib/src/tests/hitcollector/hitcollector_test.cpp @@ -1,14 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("hitcollector_test"); -#include <vespa/vespalib/testkit/testapp.h> - -#include <iostream> +#include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/queryeval/hitcollector.h> -#include <vespa/searchlib/queryeval/scores.h> + +#include <vespa/log/log.h> +LOG_SETUP("hitcollector_test"); using namespace search; using namespace search::fef; @@ -19,8 +17,8 @@ typedef std::map<uint32_t, feature_t> ScoreMap; struct BasicScorer : public HitCollector::DocumentScorer { feature_t _scoreDelta; - BasicScorer(feature_t scoreDelta) : _scoreDelta(scoreDelta) {} - virtual feature_t score(uint32_t docId) override { + explicit BasicScorer(feature_t scoreDelta) : _scoreDelta(scoreDelta) {} + feature_t score(uint32_t docId) override { return docId + _scoreDelta; } }; @@ -28,8 +26,8 @@ struct BasicScorer : public HitCollector::DocumentScorer struct PredefinedScorer : public HitCollector::DocumentScorer { ScoreMap _scores; - PredefinedScorer(const ScoreMap &scores) : _scores(scores) {} - virtual feature_t score(uint32_t docId) override { + explicit PredefinedScorer(ScoreMap scores) : _scores(std::move(scores)) {} + feature_t score(uint32_t docId) override { feature_t retval = default_rank_value; auto itr = _scores.find(docId); if (itr != _scores.end()) { @@ -41,38 +39,32 @@ struct PredefinedScorer : public HitCollector::DocumentScorer void checkResult(const ResultSet & rs, const std::vector<RankedHit> & exp) { - if (exp.size() > 0) { + if ( ! exp.empty()) { const RankedHit * rh = rs.getArray(); - ASSERT_TRUE(rh != NULL); + ASSERT_TRUE(rh != nullptr); ASSERT_EQUAL(rs.getArrayUsed(), exp.size()); for (uint32_t i = 0; i < exp.size(); ++i) { -#if 0 - std::cout << " rh[" << i << "]._docId = " << rh[i]._docId << std::endl; - std::cout << "exp[" << i << "]._docId = " << exp[i]._docId << std::endl; - std::cout << " rh[" << i << "]._rankValue = " << rh[i]._rankValue << std::endl; - std::cout << "exp[" << i << "]._rankValue = " << exp[i]._rankValue << std::endl; -#endif EXPECT_EQUAL(rh[i]._docId, exp[i]._docId); EXPECT_EQUAL(rh[i]._rankValue, exp[i]._rankValue); } } else { - ASSERT_TRUE(rs.getArray() == NULL); + ASSERT_TRUE(rs.getArray() == nullptr); } } void checkResult(ResultSet & rs, BitVector * exp) { - if (exp != NULL) { + if (exp != nullptr) { BitVector * bv = rs.getBitOverflow(); - ASSERT_TRUE(bv != NULL); + ASSERT_TRUE(bv != nullptr); bv->invalidateCachedCount(); exp->invalidateCachedCount(); LOG(info, "bv.hits: %u, exp.hits: %u", bv->countTrueBits(), exp->countTrueBits()); ASSERT_TRUE(bv->countTrueBits() == exp->countTrueBits()); EXPECT_TRUE(*bv == *exp); } else { - ASSERT_TRUE(rs.getBitOverflow() == NULL); + ASSERT_TRUE(rs.getBitOverflow() == nullptr); } } @@ -85,8 +77,8 @@ void testAddHit(uint32_t numDocs, uint32_t maxHitsSize, uint32_t maxHeapSize) std::vector<RankedHit> expRh; std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), NULL)); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, nullptr)); } LOG(info, "testAddHit: only ranked hits"); @@ -98,14 +90,14 @@ void testAddHit(uint32_t numDocs, uint32_t maxHitsSize, uint32_t maxHeapSize) hc.addHit(i, i + 100); // build expected result set as we go along - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; expRh.back()._rankValue = i + 100; } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), NULL)); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, nullptr)); } LOG(info, "testAddHit: both ranked hits and bit vector hits"); @@ -120,15 +112,15 @@ void testAddHit(uint32_t numDocs, uint32_t maxHitsSize, uint32_t maxHeapSize) // build expected result set as we go along expBv->setBit(i); if (i >= (numDocs - maxHitsSize)) { - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; expRh.back()._rankValue = i + 100; } } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), expBv.get())); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, expBv.get())); } } @@ -157,23 +149,27 @@ struct Fixture { } } size_t reRank() { - return hc.reRank(scorer); + return hc.reRank(scorer, hc.getSortedHeapHits()); } size_t reRank(size_t count) { - return hc.reRank(scorer, count); + auto hits = hc.getSortedHeapHits(); + if (hits.size() > count) { + hits.resize(count); + } + return hc.reRank(scorer, std::move(hits)); } }; struct AscendingScoreFixture : Fixture { AscendingScoreFixture() : Fixture() {} - virtual HitRank calculateScore(uint32_t i) override { + HitRank calculateScore(uint32_t i) override { return i + 100; } }; struct DescendingScoreFixture : Fixture { DescendingScoreFixture() : Fixture() {} - virtual HitRank calculateScore(uint32_t i) override { + HitRank calculateScore(uint32_t i) override { return 100 - i; } }; @@ -197,8 +193,8 @@ TEST_F("testReRank - ascending", AscendingScoreFixture) EXPECT_EQUAL(expRh.size(), 10u); std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), f.expBv.get())); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, f.expBv.get())); } TEST_F("testReRank - descending", DescendingScoreFixture) @@ -216,8 +212,8 @@ TEST_F("testReRank - descending", DescendingScoreFixture) EXPECT_EQUAL(expRh.size(), 10u); std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), f.expBv.get())); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, f.expBv.get())); } TEST_F("testReRank - partial", AscendingScoreFixture) @@ -235,25 +231,24 @@ TEST_F("testReRank - partial", AscendingScoreFixture) EXPECT_EQUAL(expRh.size(), 10u); std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), f.expBv.get())); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, f.expBv.get())); } -TEST_F("require that scores for 2nd phase candidates can be retrieved", DescendingScoreFixture) +TEST_F("require that hits for 2nd phase candidates can be retrieved", DescendingScoreFixture) { f.addHits(); - std::vector<feature_t> scores = f.hc.getSortedHeapScores(); + std::vector<HitCollector::Hit> scores = f.hc.getSortedHeapHits(); ASSERT_EQUAL(5u, scores.size()); - EXPECT_EQUAL(100, scores[0]); - EXPECT_EQUAL(99, scores[1]); - EXPECT_EQUAL(98, scores[2]); - EXPECT_EQUAL(97, scores[3]); - EXPECT_EQUAL(96, scores[4]); + EXPECT_EQUAL(100, scores[0].second); + EXPECT_EQUAL(99, scores[1].second); + EXPECT_EQUAL(98, scores[2].second); + EXPECT_EQUAL(97, scores[3].second); + EXPECT_EQUAL(96, scores[4].second); } TEST("require that score ranges can be read and set.") { - std::pair<Scores, Scores> ranges = - std::make_pair(Scores(1.0, 2.0), Scores(3.0, 4.0)); + std::pair<Scores, Scores> ranges = std::make_pair(Scores(1.0, 2.0), Scores(3.0, 4.0)); HitCollector hc(20, 10, 5); hc.setRanges(ranges); EXPECT_EQUAL(ranges.first.low, hc.getRanges().first.low); @@ -275,19 +270,19 @@ TEST("testNoHitsToReRank") { hc.addHit(i, i + 100); // build expected result set as we go along - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; expRh.back()._rankValue = i + 100; } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), NULL)); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, nullptr)); } } void testScaling(const std::vector<feature_t> &initScores, - const ScoreMap &finalScores, + ScoreMap finalScores, const std::vector<RankedHit> &expected) { HitCollector hc(5, 5, 2); @@ -297,13 +292,13 @@ void testScaling(const std::vector<feature_t> &initScores, hc.addHit(i, initScores[i]); } - PredefinedScorer scorer(finalScores); + PredefinedScorer scorer(std::move(finalScores)); // perform second phase ranking - EXPECT_EQUAL(2u, hc.reRank(scorer)); + EXPECT_EQUAL(2u, hc.reRank(scorer, hc.getSortedHeapHits())); // check results std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expected)); + TEST_DO(checkResult(*rs, expected)); } TEST("testScaling") { @@ -332,7 +327,7 @@ TEST("testScaling") { finalScores[3] = 300; finalScores[4] = 400; - testScaling(initScores, finalScores, exp); + testScaling(initScores, std::move(finalScores), exp); } { // scale down and adjust up exp[0]._rankValue = 200; // scaled @@ -346,7 +341,7 @@ TEST("testScaling") { finalScores[3] = 500; finalScores[4] = 600; - testScaling(initScores, finalScores, exp); + testScaling(initScores, std::move(finalScores), exp); } { // scale up and adjust down @@ -361,7 +356,7 @@ TEST("testScaling") { finalScores[3] = 3250; finalScores[4] = 4500; - testScaling(initScores, finalScores, exp); + testScaling(initScores, std::move(finalScores), exp); } { // minimal scale (second phase range = 0 (4 - 4) -> 1) exp[0]._rankValue = 1; // scaled @@ -375,7 +370,7 @@ TEST("testScaling") { finalScores[3] = 4; finalScores[4] = 4; - testScaling(initScores, finalScores, exp); + testScaling(initScores, std::move(finalScores), exp); } { // minimal scale (first phase range = 0 (4000 - 4000) -> 1) std::vector<feature_t> is(initScores); @@ -391,7 +386,7 @@ TEST("testScaling") { finalScores[3] = 400; finalScores[4] = 500; - testScaling(is, finalScores, exp); + testScaling(is, std::move(finalScores), exp); } } @@ -410,8 +405,8 @@ TEST("testOnlyBitVector") { std::unique_ptr<ResultSet> rs = hc.getResultSet(); std::vector<RankedHit> expRh; - TEST_DO(checkResult(*rs.get(), expRh)); // no ranked hits - TEST_DO(checkResult(*rs.get(), expBv.get())); // only bit vector + TEST_DO(checkResult(*rs, expRh)); // no ranked hits + TEST_DO(checkResult(*rs, expBv.get())); // only bit vector } } @@ -433,19 +428,19 @@ TEST_F("require that result set is merged correctly with first phase ranking", f.hc.addHit(i, i + 1000); // build expected result set - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; // only the maxHitsSize best hits gets a score expRh.back()._rankValue = (i < f.numDocs - f.maxHitsSize) ? default_rank_value : i + 1000; } std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); + TEST_DO(checkResult(*rs, expRh)); } void addExpectedHitForMergeTest(const MergeResultSetFixture &f, std::vector<RankedHit> &expRh, uint32_t docId) { - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = docId; if (docId < f.numDocs - f.maxHitsSize) { // only the maxHitsSize best hits gets a score expRh.back()._rankValue = default_rank_value; @@ -466,9 +461,9 @@ TEST_F("require that result set is merged correctly with second phase ranking (d f.hc.addHit(i, i + 1000); addExpectedHitForMergeTest(f, expRh, i); } - EXPECT_EQUAL(f.maxHeapSize, f.hc.reRank(scorer)); + EXPECT_EQUAL(f.maxHeapSize, f.hc.reRank(scorer, f.hc.getSortedHeapHits())); std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); + TEST_DO(checkResult(*rs, expRh)); } TEST("require that hits can be added out of order") { @@ -476,7 +471,7 @@ TEST("require that hits can be added out of order") { std::vector<RankedHit> expRh; // produce expected result in normal order for (uint32_t i = 0; i < 5; ++i) { - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; expRh.back()._rankValue = i + 100; } @@ -485,8 +480,8 @@ TEST("require that hits can be added out of order") { hc.addHit(i, i + 100); } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), nullptr)); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, nullptr)); } TEST("require that hits can be added out of order when passing array limit") { @@ -495,7 +490,7 @@ TEST("require that hits can be added out of order when passing array limit") { // produce expected result in normal order const size_t numHits = 150; for (uint32_t i = 0; i < numHits; ++i) { - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; expRh.back()._rankValue = (i < 50) ? default_rank_value : (i + 100); } @@ -504,8 +499,8 @@ TEST("require that hits can be added out of order when passing array limit") { hc.addHit(i, i + 100); } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), nullptr)); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, nullptr)); } TEST("require that hits can be added out of order only after passing array limit") { @@ -514,7 +509,7 @@ TEST("require that hits can be added out of order only after passing array limit // produce expected result in normal order const size_t numHits = 150; for (uint32_t i = 0; i < numHits; ++i) { - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; expRh.back()._rankValue = (i < 50) ? default_rank_value : (i + 100); } @@ -527,8 +522,8 @@ TEST("require that hits can be added out of order only after passing array limit hc.addHit(i, i + 100); } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), nullptr)); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, nullptr)); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp index 8ad4578b6c1..37a63828667 100644 --- a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp +++ b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp @@ -172,12 +172,12 @@ StackDumpIteratorTest::ShowResult(int testNo, delete item; break; } - if (strncmp(item->_indexName.c_str(), idx.c_str(), idx.size()) != 0) { + if (strncmp(item->_indexName.c_str(), idx.data(), idx.size()) != 0) { results |= ITERATOR_ERROR_WRONG_INDEX; delete item; break; } - if (strncmp(item->_term.c_str(), term.c_str(), term.size()) != 0) { + if (strncmp(item->_term.c_str(), term.data(), term.size()) != 0) { results |= ITERATOR_ERROR_WRONG_TERM; delete item; break; |