diff options
author | Geir Storli <geirstorli@yahoo.no> | 2017-09-07 09:25:22 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-09-07 09:25:22 +0200 |
commit | a9b48c32108ed326d80ed0f6fac6ec4d6963df75 (patch) | |
tree | c83b9cb87fa06c5c9b945b885f3b9f7cb57809d2 | |
parent | e42cd7c1d9b13835163ee0fbf4876f9c86fcf998 (diff) | |
parent | 4119e49f014b01395fe67b9d95e4fd3b1616ca3d (diff) |
Merge pull request #3353 from vespa-engine/geirst/add-bitvector-search-cache-to-imported-attribute-vectors
Geirst/add bitvector search cache to imported attribute vectors
43 files changed, 666 insertions, 239 deletions
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 834d7145ed0..5b64fa80d39 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -133,7 +133,7 @@ struct ImportedAttributesRepoBuilder { void add(const vespalib::string &name) { auto refAttr = std::make_shared<ReferenceAttribute>(name + "_ref", AVConfig(BasicType::REFERENCE)); auto targetAttr = search::AttributeFactory::createAttribute(name + "_target", INT32_SINGLE); - auto importedAttr = std::make_shared<ImportedAttributeVector>(name, refAttr, targetAttr); + auto importedAttr = std::make_shared<ImportedAttributeVector>(name, refAttr, targetAttr, false); _repo->add(name, importedAttr); } ImportedAttributesRepo::UP build() { diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index e6052c02ee6..1112cf75570 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -2,44 +2,45 @@ #include <vespa/log/log.h> LOG_SETUP("attribute_test"); -#include <vespa/fastos/file.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/config-attributes.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/predicate/predicate_slime_builder.h> #include <vespa/document/update/arithmeticvalueupdate.h> +#include <vespa/document/update/assignvalueupdate.h> +#include <vespa/eval/tensor/default_tensor.h> +#include <vespa/eval/tensor/tensor.h> +#include <vespa/eval/tensor/tensor_factory.h> +#include <vespa/eval/tensor/types.h> +#include <vespa/fastos/file.h> #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> +#include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/test/attribute_utils.h> #include <vespa/searchcorespi/flush/iflushtarget.h> #include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/searchlib/attribute/bitvector_search_cache.h> +#include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/attribute/predicate_attribute.h> +#include <vespa/searchlib/attribute/singlenumericattribute.hpp> +#include <vespa/searchlib/common/foregroundtaskexecutor.h> #include <vespa/searchlib/common/idestructorcallback.h> +#include <vespa/searchlib/common/sequencedtaskexecutorobserver.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/predicate/predicate_hash.h> +#include <vespa/searchlib/predicate/predicate_index.h> +#include <vespa/searchlib/tensor/tensor_attribute.h> +#include <vespa/searchlib/test/directory_handler.h> #include <vespa/searchlib/util/filekit.h> -#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> - -#include <vespa/document/predicate/predicate_slime_builder.h> -#include <vespa/document/update/assignvalueupdate.h> -#include <vespa/searchlib/attribute/attributevector.hpp> -#include <vespa/searchlib/attribute/predicate_attribute.h> -#include <vespa/searchlib/predicate/predicate_index.h> -#include <vespa/searchlib/attribute/singlenumericattribute.hpp> -#include <vespa/searchlib/predicate/predicate_hash.h> -#include <vespa/searchlib/common/foregroundtaskexecutor.h> -#include <vespa/searchlib/common/sequencedtaskexecutorobserver.h> -#include <vespa/searchlib/test/directory_handler.h> -#include <vespa/eval/tensor/tensor.h> -#include <vespa/eval/tensor/types.h> -#include <vespa/eval/tensor/default_tensor.h> -#include <vespa/eval/tensor/tensor_factory.h> -#include <vespa/searchlib/tensor/tensor_attribute.h> -#include <vespa/config-attributes.h> - +#include <vespa/vespalib/testkit/testapp.h> namespace vespa { namespace config { namespace search {}}} @@ -50,8 +51,12 @@ using namespace search::index; using namespace search; using namespace vespa::config::search; +using proton::ImportedAttributesRepo; using proton::test::AttributeUtils; using search::TuneFileAttributes; +using search::attribute::BitVectorSearchCache; +using search::attribute::ImportedAttributeVector; +using search::attribute::ReferenceAttribute; using search::index::DummyFileHeaderContext; using search::index::schema::CollectionType; using search::predicate::PredicateHash; @@ -152,7 +157,7 @@ struct Fixture _aw->remove(serialNum, lid, immediateCommit, emptyCallback); } void commit(SerialNum serialNum) { - _aw->commit(serialNum, emptyCallback); + _aw->forceCommit(serialNum, emptyCallback); } void assertExecuteHistory(std::vector<uint32_t> expExecuteHistory) { EXPECT_EQUAL(expExecuteHistory, _attributeFieldWriter.getExecuteHistory()); @@ -345,14 +350,14 @@ TEST_F("require that visibilitydelay is honoured", Fixture) awDelayed.put(5, *doc, 4, false, emptyCallback); EXPECT_EQUAL(5u, a1->getNumDocs()); EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); - awDelayed.commit(6, emptyCallback); + awDelayed.forceCommit(6, emptyCallback); EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken()); AttributeWriter awDelayedShort(f._m); awDelayedShort.put(7, *doc, 2, false, emptyCallback); EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken()); awDelayedShort.put(8, *doc, 2, false, emptyCallback); - awDelayedShort.commit(8, emptyCallback); + awDelayedShort.forceCommit(8, emptyCallback); EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken()); verifyAttributeContent(*a1, 2, "10"); @@ -364,7 +369,7 @@ TEST_F("require that visibilitydelay is honoured", Fixture) 2, false, emptyCallback); EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken()); verifyAttributeContent(*a1, 2, "10"); - awDelayed.commit(12, emptyCallback); + awDelayed.forceCommit(12, emptyCallback); EXPECT_EQUAL(12u, a1->getStatus().getLastSyncToken()); verifyAttributeContent(*a1, 2, "30"); @@ -712,6 +717,34 @@ TEST_F("require that attribute writer spreads write over 3 write contexts", Fixt TEST_DO(putAttributes(f, {0, 1, 2})); } +ImportedAttributeVector::SP +createImportedAttribute(const vespalib::string &name) +{ + auto result = std::make_shared<ImportedAttributeVector>(name, + ReferenceAttribute::SP(), + AttributeVector::SP(), + true); + result->getSearchCache()->insert("foo", BitVectorSearchCache::Entry::SP()); + return result; +} + +ImportedAttributesRepo::UP +createImportedAttributesRepo() +{ + auto result = std::make_unique<ImportedAttributesRepo>(); + result->add("imported_a", createImportedAttribute("imported_a")); + result->add("imported_b", createImportedAttribute("imported_b")); + return result; +} + +TEST_F("require that AttributeWriter::forceCommit() clears search cache in imported attribute vectors", Fixture) +{ + f._m->setImportedAttributes(createImportedAttributesRepo()); + f.commit(10); + EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_a")->getSearchCache()->size()); + EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_b")->getSearchCache()->size()); +} + TEST_MAIN() { vespalib::rmdir(test_dir, true); diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp index 3b2c87b3840..d95d79fbd11 100644 --- a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp +++ b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp @@ -70,7 +70,8 @@ struct Fixture { Fixture &addAttribute(const vespalib::string &name) { auto attr = std::make_shared<ImportedAttributeVector>(name, createReferenceAttribute(name + "_ref"), - createTargetAttribute(name + "_target")); + createTargetAttribute(name + "_target"), + false); repo.add(name, attr); return *this; } diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp index 99b1d11e862..c41b88be2b4 100644 --- a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp +++ b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp @@ -21,7 +21,8 @@ createAttr(const vespalib::string &name) { return std::make_shared<ImportedAttributeVector>(name, ReferenceAttribute::SP(), - AttributeVector::SP()); + AttributeVector::SP(), + false); } struct Fixture { diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index d76117df4c2..45bda8c69cc 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -141,7 +141,9 @@ struct EmptyConstantValueFactory : public vespalib::eval::ConstantValueFactory { }; struct MyDocumentDBReferenceResolver : public IDocumentDBReferenceResolver { - std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &, const search::IAttributeManager &) override { + std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &, + const search::IAttributeManager &, + fastos::TimeStamp) override { return std::make_unique<ImportedAttributesRepo>(); } void teardown(const search::IAttributeManager &) override { } @@ -657,12 +659,18 @@ TEST_F("require that we can reconfigure matchers", Fixture) } } -TEST("require that attribute manager should change when imported fields has changed") +TEST("require that attribute manager (imported attributes) should change when imported fields has changed") { ReconfigParams params(CCR().setImportedFieldsChanged(true)); EXPECT_TRUE(params.shouldAttributeManagerChange()); } +TEST("require that attribute manager (imported attributes) should change when visibility delay has changed") +{ + ReconfigParams params(CCR().setVisibilityDelayChanged(true)); + EXPECT_TRUE(params.shouldAttributeManagerChange()); +} + void assertMaintenanceControllerShouldNotChange(DocumentDBConfig::ComparisonResult result) { diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index c3928e193e8..8cd57b1b0cb 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -108,7 +108,9 @@ struct MyMetricsWireService : public DummyWireService }; struct MyDocumentDBReferenceResolver : public IDocumentDBReferenceResolver { - std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &, const search::IAttributeManager &) override { + std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &, + const search::IAttributeManager &, + fastos::TimeStamp) override { return std::make_unique<ImportedAttributesRepo>(); } void teardown(const search::IAttributeManager &) override { } diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index 8af365d6327..c02aa033bb9 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -385,7 +385,7 @@ struct MyAttributeWriter : public IAttributeWriter virtual const proton::IAttributeManager::SP &getAttributeManager() const override { return _mgr; } - void commit(SerialNum serialNum, OnWriteDoneType) override { + void forceCommit(SerialNum serialNum, OnWriteDoneType) override { (void) serialNum; ++_commitCount; _tracer.traceCommit(attributeAdapterTypeName, serialNum); } diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 6eb447ac968..4b4e5b955a1 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -384,6 +384,9 @@ struct MyAttributeManager : public proton::IAttributeManager virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo>) override { abort(); } + virtual const ImportedAttributesRepo *getImportedAttributes() const override { + abort(); + } }; struct MockLidSpaceCompactionHandler : public ILidSpaceCompactionHandler diff --git a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp index 65e15a3efaa..a6576d11279 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp @@ -224,9 +224,12 @@ struct Fixture { oldAttrMgr.addReferenceAttribute("parent2_ref"); oldAttrMgr.addReferenceAttribute("parent3_ref"); } - ImportedAttributesRepo::UP resolve() { + ImportedAttributesRepo::UP resolve(fastos::TimeStamp visibilityDelay) { DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter); - return resolver.resolve(attrMgr, oldAttrMgr); + return resolver.resolve(attrMgr, oldAttrMgr, visibilityDelay); + } + ImportedAttributesRepo::UP resolve() { + return resolve(fastos::TimeStamp(0)); } void teardown() { DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter); @@ -238,11 +241,13 @@ struct Fixture { void assertImportedAttribute(const vespalib::string &name, const vespalib::string &referenceField, const vespalib::string &targetField, + bool useSearchCache, ImportedAttributeVector::SP attr) { ASSERT_TRUE(attr.get()); EXPECT_EQUAL(name, attr->getName()); EXPECT_EQUAL(attrMgr.getReferenceAttribute(referenceField), attr->getReferenceAttribute().get()); EXPECT_EQUAL(parentReference->getAttribute(targetField).get(), attr->getTargetAttribute().get()); + EXPECT_EQUAL(useSearchCache, attr->getSearchCache().get() != nullptr); } MockGidToLidChangeHandler &getGidToLidChangeHandler(const vespalib::string &referencedDocTypeName) { @@ -272,12 +277,20 @@ TEST_F("require that reference attributes are connected to gid mapper", Fixture) EXPECT_EQUAL(f.factory.get(), f.getMapperFactoryPtr("other_ref")); } -TEST_F("require that imported attributes are instantiated", Fixture) +TEST_F("require that imported attributes are instantiated without search cache as default", Fixture) { auto repo = f.resolve(); EXPECT_EQUAL(2u, repo->size()); - f.assertImportedAttribute("imported_a", "ref", "target_a", repo->get("imported_a")); - f.assertImportedAttribute("imported_b", "other_ref", "target_b", repo->get("imported_b")); + f.assertImportedAttribute("imported_a", "ref", "target_a", false, repo->get("imported_a")); + f.assertImportedAttribute("imported_b", "other_ref", "target_b", false, repo->get("imported_b")); +} + +TEST_F("require that imported attributes are instantiated with search cache if visibility delay > 0", Fixture) +{ + auto repo = f.resolve(fastos::TimeStamp::Seconds(1.0)); + EXPECT_EQUAL(2u, repo->size()); + f.assertImportedAttribute("imported_a", "ref", "target_a", true, repo->get("imported_a")); + f.assertImportedAttribute("imported_b", "other_ref", "target_b", true, repo->get("imported_b")); } TEST_F("require that listeners are added", Fixture) diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 6d4915d800d..906f71fecb4 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -2,18 +2,20 @@ #include "attribute_writer.h" #include "attributemanager.h" +#include <vespa/document/base/exceptions.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/attrupdate.h> #include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> -#include <vespa/document/datatype/documenttype.h> #include <vespa/log/log.h> -#include <vespa/document/base/exceptions.h> - LOG_SETUP(".proton.server.attributeadapter"); using namespace document; using namespace search; +using search::attribute::ImportedAttributeVector; namespace proton { @@ -459,8 +461,15 @@ AttributeWriter::heartBeat(SerialNum serialNum) void -AttributeWriter::commit(SerialNum serialNum, OnWriteDoneType onWriteDone) +AttributeWriter::forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) { + if (_mgr->getImportedAttributes() != nullptr) { + std::vector<std::shared_ptr<ImportedAttributeVector>> importedAttrs; + _mgr->getImportedAttributes()->getAll(importedAttrs); + for (const auto &attr : importedAttrs) { + attr->clearSearchCache(); + } + } for (const auto &wc : _writeContexts) { auto commitTask = std::make_unique<CommitTask>(wc, serialNum, onWriteDone); _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(commitTask)); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index 58b92459358..cbda11180c0 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -74,7 +74,7 @@ public: const proton::IAttributeManager::SP &getAttributeManager() const override { return _mgr; } - void commit(SerialNum serialNum, OnWriteDoneType onWriteDone) override; + void forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) override; virtual void onReplayDone(uint32_t docIdLimit) override; }; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index a7de9e2efa4..7148815ca9e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -155,7 +155,6 @@ public: static void padAttribute(search::AttributeVector &v, uint32_t docIdLimit); - const ImportedAttributesRepo *getImportedAttributes() const { return _importedAttributes.get(); } // Implements search::IAttributeManager virtual search::AttributeGuard::UP getAttribute(const vespalib::string &name) const override; @@ -200,6 +199,8 @@ public: virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override; virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override; + + virtual const ImportedAttributesRepo *getImportedAttributes() const override { return _importedAttributes.get(); } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index 9f78a431430..af64f47fc0a 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -213,4 +213,10 @@ FilterAttributeManager::setImportedAttributes(std::unique_ptr<ImportedAttributes throw vespalib::IllegalArgumentException("Not implemented"); } +const ImportedAttributesRepo * +FilterAttributeManager::getImportedAttributes() const +{ + throw vespalib::IllegalArgumentException("Not implemented"); +} + } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h index 958bf78eeb6..81fafa477ee 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h @@ -53,6 +53,7 @@ public: virtual void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const override; virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override; virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override; + virtual const ImportedAttributesRepo *getImportedAttributes() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h index eb9bde1048f..31bd7c27c85 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h @@ -101,6 +101,8 @@ struct IAttributeManager : public search::IAttributeManager virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const = 0; virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) = 0; + + virtual const ImportedAttributesRepo *getImportedAttributes() const = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h index 5903b4d76e4..7455ccda5a3 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h @@ -61,7 +61,7 @@ public: /** * Commit all underlying attribute vectors with the given serial number. */ - virtual void commit(SerialNum serialNum, OnWriteDoneType onWriteDone) = 0; + virtual void forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) = 0; virtual void onReplayDone(uint32_t docIdLimit) = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp index 5ac5cf3488e..b6c6c798506 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp @@ -13,159 +13,159 @@ using search::grouping::GroupingSession; namespace proton::matching { - namespace { - using Stats = SessionManager::Stats; - struct SessionCacheBase { - protected: - Stats _stats; - vespalib::Lock _lock; - - void entryDropped(const SessionId &id); - ~SessionCacheBase() {} - }; - - template <typename T> - struct SessionCache : SessionCacheBase { - typedef typename T::UP EntryUP; - vespalib::lrucache_map<vespalib::LruParam<SessionId, EntryUP> > _cache; - - SessionCache(uint32_t max_size) : _cache(max_size) {} - - void insert(EntryUP session) { - vespalib::LockGuard guard(_lock); - const SessionId &id(session->getSessionId()); - if (_cache.size() >= _cache.capacity()) { - entryDropped(id); - } - _cache.insert(id, std::move(session)); - _stats.numInsert++; - } - EntryUP pick(const SessionId & id) { - vespalib::LockGuard guard(_lock); - EntryUP ret; - if (_cache.hasKey(id)) { - _stats.numPick++; - ret = std::move(_cache[id]); - _cache.erase(id); - } - return ret; - } - void pruneTimedOutSessions(fastos::TimeStamp currentTime) { - std::vector<EntryUP> toDestruct = stealTimedOutSessions(currentTime); - toDestruct.clear(); - } - std::vector<EntryUP> stealTimedOutSessions(fastos::TimeStamp currentTime) { - std::vector<EntryUP> toDestruct; - vespalib::LockGuard guard(_lock); - toDestruct.reserve(_cache.size()); - for (auto it(_cache.begin()), mt(_cache.end()); it != mt;) { - auto &session = *it; - if (session->getTimeOfDoom() < currentTime) { - toDestruct.push_back(std::move(session)); - it = _cache.erase(it); - _stats.numTimedout++; - } else { - it++; - } - } - return toDestruct; - } - Stats getStats() { - vespalib::LockGuard guard(_lock); - Stats stats = _stats; - stats.numCached = _cache.size(); - _stats = Stats(); - return stats; - } - bool empty() const { - vespalib::LockGuard guard(_lock); - return _cache.empty(); - } - }; - - template <typename T> - struct SessionMap : SessionCacheBase { - typedef typename T::SP EntrySP; - vespalib::hash_map<SessionId, EntrySP> _map; - - void insert(EntrySP session) { - vespalib::LockGuard guard(_lock); - const SessionId &id(session->getSessionId()); - _map.insert(std::make_pair(id, session)); - _stats.numInsert++; - } - EntrySP pick(const SessionId & id) { - vespalib::LockGuard guard(_lock); - auto it = _map.find(id); - if (it != _map.end()) { - _stats.numPick++; - return it->second; - } - return EntrySP(); - } - void pruneTimedOutSessions(fastos::TimeStamp currentTime) { - std::vector<EntrySP> toDestruct = stealTimedOutSessions(currentTime); - toDestruct.clear(); - } - std::vector<EntrySP> stealTimedOutSessions(fastos::TimeStamp currentTime) { - std::vector<EntrySP> toDestruct; - std::vector<SessionId> keys; - vespalib::LockGuard guard(_lock); - keys.reserve(_map.size()); - toDestruct.reserve(_map.size()); - for (auto & it : _map) { - EntrySP &session = it.second; - if (session->getTimeOfDoom() < currentTime) { - keys.push_back(it.first); - toDestruct.push_back(EntrySP()); - toDestruct.back().swap(session); - } - } - for (auto key : keys) { - _map.erase(key); - _stats.numTimedout++; - } - return toDestruct; - } - Stats getStats() { - vespalib::LockGuard guard(_lock); - Stats stats = _stats; - stats.numCached = _map.size(); - _stats = Stats(); - return stats; - } - size_t size() const { - vespalib::LockGuard guard(_lock); - return _map.size(); - } - bool empty() const { - vespalib::LockGuard guard(_lock); - return _map.empty(); +namespace { +using Stats = SessionManager::Stats; +struct SessionCacheBase { +protected: + Stats _stats; + vespalib::Lock _lock; + + void entryDropped(const SessionId &id); + ~SessionCacheBase() {} +}; + +template <typename T> +struct SessionCache : SessionCacheBase { + typedef typename T::UP EntryUP; + vespalib::lrucache_map<vespalib::LruParam<SessionId, EntryUP> > _cache; + + SessionCache(uint32_t max_size) : _cache(max_size) {} + + void insert(EntryUP session) { + vespalib::LockGuard guard(_lock); + const SessionId &id(session->getSessionId()); + if (_cache.size() >= _cache.capacity()) { + entryDropped(id); + } + _cache.insert(id, std::move(session)); + _stats.numInsert++; + } + EntryUP pick(const SessionId & id) { + vespalib::LockGuard guard(_lock); + EntryUP ret; + if (_cache.hasKey(id)) { + _stats.numPick++; + ret = std::move(_cache[id]); + _cache.erase(id); + } + return ret; + } + void pruneTimedOutSessions(fastos::TimeStamp currentTime) { + std::vector<EntryUP> toDestruct = stealTimedOutSessions(currentTime); + toDestruct.clear(); + } + std::vector<EntryUP> stealTimedOutSessions(fastos::TimeStamp currentTime) { + std::vector<EntryUP> toDestruct; + vespalib::LockGuard guard(_lock); + toDestruct.reserve(_cache.size()); + for (auto it(_cache.begin()), mt(_cache.end()); it != mt;) { + auto &session = *it; + if (session->getTimeOfDoom() < currentTime) { + toDestruct.push_back(std::move(session)); + it = _cache.erase(it); + _stats.numTimedout++; + } else { + it++; } - template <typename F> - void each(F f) const { - vespalib::LockGuard guard(_lock); - for (const auto &entry: _map) { - f(*entry.second); - } + } + return toDestruct; + } + Stats getStats() { + vespalib::LockGuard guard(_lock); + Stats stats = _stats; + stats.numCached = _cache.size(); + _stats = Stats(); + return stats; + } + bool empty() const { + vespalib::LockGuard guard(_lock); + return _cache.empty(); + } +}; + +template <typename T> +struct SessionMap : SessionCacheBase { + typedef typename T::SP EntrySP; + vespalib::hash_map<SessionId, EntrySP> _map; + + void insert(EntrySP session) { + vespalib::LockGuard guard(_lock); + const SessionId &id(session->getSessionId()); + _map.insert(std::make_pair(id, session)); + _stats.numInsert++; + } + EntrySP pick(const SessionId & id) { + vespalib::LockGuard guard(_lock); + auto it = _map.find(id); + if (it != _map.end()) { + _stats.numPick++; + return it->second; + } + return EntrySP(); + } + void pruneTimedOutSessions(fastos::TimeStamp currentTime) { + std::vector<EntrySP> toDestruct = stealTimedOutSessions(currentTime); + toDestruct.clear(); + } + std::vector<EntrySP> stealTimedOutSessions(fastos::TimeStamp currentTime) { + std::vector<EntrySP> toDestruct; + std::vector<SessionId> keys; + vespalib::LockGuard guard(_lock); + keys.reserve(_map.size()); + toDestruct.reserve(_map.size()); + for (auto & it : _map) { + EntrySP &session = it.second; + if (session->getTimeOfDoom() < currentTime) { + keys.push_back(it.first); + toDestruct.push_back(EntrySP()); + toDestruct.back().swap(session); } - }; - - void SessionCacheBase::entryDropped(const SessionId &id) { - LOG(debug, "Session cache is full, dropping entry to fit session '%s'", id.c_str()); - _stats.numDropped++; } - + for (auto key : keys) { + _map.erase(key); + _stats.numTimedout++; + } + return toDestruct; } + Stats getStats() { + vespalib::LockGuard guard(_lock); + Stats stats = _stats; + stats.numCached = _map.size(); + _stats = Stats(); + return stats; + } + size_t size() const { + vespalib::LockGuard guard(_lock); + return _map.size(); + } + bool empty() const { + vespalib::LockGuard guard(_lock); + return _map.empty(); + } + template <typename F> + void each(F f) const { + vespalib::LockGuard guard(_lock); + for (const auto &entry: _map) { + f(*entry.second); + } + } +}; + +void SessionCacheBase::entryDropped(const SessionId &id) { + LOG(debug, "Session cache is full, dropping entry to fit session '%s'", id.c_str()); + _stats.numDropped++; +} + +} - struct GroupingSessionCache : public SessionCache<search::grouping::GroupingSession> { - using Parent = SessionCache<search::grouping::GroupingSession>; - using Parent::Parent; - }; +struct GroupingSessionCache : public SessionCache<search::grouping::GroupingSession> { + using Parent = SessionCache<search::grouping::GroupingSession>; + using Parent::Parent; +}; - struct SearchSessionCache : public SessionMap<SearchSession> { +struct SearchSessionCache : public SessionMap<SearchSession> { - }; +}; SessionManager::SessionManager(uint32_t maxSize) diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp index 5147840d5a3..c56aa18c1f7 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp @@ -135,13 +135,15 @@ DocumentDBReferenceResolver::listenToGidToLidChanges(const IAttributeManager &at } ImportedAttributesRepo::UP -DocumentDBReferenceResolver::createImportedAttributesRepo(const IAttributeManager &attrMgr) +DocumentDBReferenceResolver::createImportedAttributesRepo(const IAttributeManager &attrMgr, + bool useSearchCache) { auto result = std::make_unique<ImportedAttributesRepo>(); for (const auto &attr : _importedFieldsCfg.attribute) { ReferenceAttribute::SP refAttr = getReferenceAttribute(attr.referencefield, attrMgr); AttributeVector::SP targetAttr = getTargetDocumentDB(refAttr->getName())->getAttribute(attr.targetfield); - ImportedAttributeVector::SP importedAttr = std::make_shared<ImportedAttributeVector>(attr.name, refAttr, targetAttr); + ImportedAttributeVector::SP importedAttr = + std::make_shared<ImportedAttributeVector>(attr.name, refAttr, targetAttr, useSearchCache); result->add(importedAttr->getName(), importedAttr); } return result; @@ -169,12 +171,14 @@ DocumentDBReferenceResolver::~DocumentDBReferenceResolver() } ImportedAttributesRepo::UP -DocumentDBReferenceResolver::resolve(const IAttributeManager &newAttrMgr, const IAttributeManager &oldAttrMgr) +DocumentDBReferenceResolver::resolve(const IAttributeManager &newAttrMgr, + const IAttributeManager &oldAttrMgr, + fastos::TimeStamp visibilityDelay) { connectReferenceAttributesToGidMapper(newAttrMgr); detectOldListeners(oldAttrMgr); listenToGidToLidChanges(newAttrMgr); - return createImportedAttributesRepo(newAttrMgr); + return createImportedAttributesRepo(newAttrMgr, (visibilityDelay > 0)); } void diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h index cd58d4b5365..8e0a4178622 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h +++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h @@ -36,7 +36,8 @@ private: GidToLidChangeRegistrator &getRegistrator(const vespalib::string &docTypeName); std::shared_ptr<IDocumentDBReference> getTargetDocumentDB(const vespalib::string &refAttrName) const; void connectReferenceAttributesToGidMapper(const search::IAttributeManager &attrMgr); - std::unique_ptr<ImportedAttributesRepo> createImportedAttributesRepo(const search::IAttributeManager &attrMgr); + std::unique_ptr<ImportedAttributesRepo> createImportedAttributesRepo(const search::IAttributeManager &attrMgr, + bool useSearchCache); void detectOldListeners(const search::IAttributeManager &attrMgr); void listenToGidToLidChanges(const search::IAttributeManager &attrMgr); @@ -49,7 +50,9 @@ public: search::ISequencedTaskExecutor &attributeFieldWriter); ~DocumentDBReferenceResolver(); - virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr, const search::IAttributeManager &oldAttrMgr) override; + virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr, + const search::IAttributeManager &oldAttrMgr, + fastos::TimeStamp visibilityDelay) override; virtual void teardown(const search::IAttributeManager &oldAttrMgr) override; }; diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h index c0d1ab6f68c..7e5b26ac34c 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h +++ b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/fastos/timestamp.h> #include <memory> namespace search { class IAttributeManager; } @@ -14,7 +15,9 @@ class ImportedAttributesRepo; */ struct IDocumentDBReferenceResolver { virtual ~IDocumentDBReferenceResolver() {} - virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr, const search::IAttributeManager &oldAttrMgr) = 0; + virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr, + const search::IAttributeManager &oldAttrMgr, + fastos::TimeStamp visibilityDelay) = 0; virtual void teardown(const search::IAttributeManager &oldAttrMgr) = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index d4e10b677a5..86d43517d00 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -40,7 +40,8 @@ DocumentDBConfig::ComparisonResult::ComparisonResult() importedFieldsChanged(false), tuneFileDocumentDBChanged(false), schemaChanged(false), - maintenanceChanged(false) + maintenanceChanged(false), + visibilityDelayChanged(false) { } DocumentDBConfig::DocumentDBConfig( @@ -171,6 +172,8 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const retval.maintenanceChanged = !equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get()); + retval.visibilityDelayChanged = + (_maintenance->getVisibilityDelay() != rhs._maintenance->getVisibilityDelay()); return retval; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index 69e9354b741..773450af620 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -49,6 +49,7 @@ public: bool tuneFileDocumentDBChanged; bool schemaChanged; bool maintenanceChanged; + bool visibilityDelayChanged; ComparisonResult(); ComparisonResult &setRankProfilesChanged(bool val) { rankProfilesChanged = val; return *this; } @@ -64,6 +65,7 @@ public: ComparisonResult &setTuneFileDocumentDBChanged(bool val) { tuneFileDocumentDBChanged = val; return *this; } ComparisonResult &setSchemaChanged(bool val) { schemaChanged = val; return *this; } ComparisonResult &setMaintenanceChanged(bool val) { maintenanceChanged = val; return *this; } + ComparisonResult &setVisibilityDelayChanged(bool val) { visibilityDelayChanged = val; return *this; } }; using SP = std::shared_ptr<DocumentDBConfig>; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index 8f2e6081b95..692bb12213e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -328,8 +328,7 @@ FastAccessDocSubDB::onReprocessDone(SerialNum serialNum) IFeedView::SP feedView = _iFeedView.get(); IAttributeWriter::SP attrWriter = static_cast<FastAccessFeedView &>(*feedView).getAttributeWriter(); - attrWriter->commit(serialNum, - std::shared_ptr<search::IDestructorCallback>()); + attrWriter->forceCommit(serialNum, std::shared_ptr<search::IDestructorCallback>()); _writeService.attributeFieldWriter().sync(); _writeService.summary().sync(); Parent::onReprocessDone(serialNum); diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp index a48c6373c00..2444223c96f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp @@ -106,7 +106,7 @@ void FastAccessFeedView::forceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone) { - _attributeWriter->commit(serialNum, onCommitDone); + _attributeWriter->forceCommit(serialNum, onCommitDone); onCommitDone->registerCommittedDocIdLimit(_metaStore.getCommittedDocIdLimit(), &_docIdLimit); Parent::forceCommit(serialNum, onCommitDone); } diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp index afe19a4ecc3..c1d139c7486 100644 --- a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp @@ -49,7 +49,7 @@ ReconfigParams::shouldIndexManagerChange() const bool ReconfigParams::shouldAttributeManagerChange() const { - return _res.attributesChanged || _res.importedFieldsChanged; + return _res.attributesChanged || _res.importedFieldsChanged || _res.visibilityDelayChanged; } bool diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index 571e64e104c..03ada12c4f2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -210,7 +210,8 @@ SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, IAttributeWriter::SP attrWriter = _feedView.get()->getAttributeWriter(); if (params.shouldAttributeManagerChange()) { IAttributeManager::SP newAttrMgr = attrMgr->create(attrSpec); - newAttrMgr->setImportedAttributes(resolver.resolve(*newAttrMgr, *attrMgr)); + newAttrMgr->setImportedAttributes(resolver.resolve(*newAttrMgr, *attrMgr, + newConfig.getMaintenanceConfigSP()->getVisibilityDelay())); IAttributeManager::SP oldAttrMgr = attrMgr; attrMgr = newAttrMgr; shouldMatchViewChange = true; diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index d079ef167b8..787ca6ed008 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -73,6 +73,7 @@ vespa_define_module( src/tests/attribute/attributemanager src/tests/attribute/benchmark src/tests/attribute/bitvector + src/tests/attribute/bitvector_search_cache src/tests/attribute/changevector src/tests/attribute/compaction src/tests/attribute/comparator diff --git a/searchlib/src/tests/attribute/bitvector_search_cache/CMakeLists.txt b/searchlib/src/tests/attribute/bitvector_search_cache/CMakeLists.txt new file mode 100644 index 00000000000..2442474bcea --- /dev/null +++ b/searchlib/src/tests/attribute/bitvector_search_cache/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_bitvector_search_cache_test_app TEST + SOURCES + bitvector_search_cache_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_bitvector_search_cache_test_app COMMAND searchlib_bitvector_search_cache_test_app) diff --git a/searchlib/src/tests/attribute/bitvector_search_cache/bitvector_search_cache_test.cpp b/searchlib/src/tests/attribute/bitvector_search_cache/bitvector_search_cache_test.cpp new file mode 100644 index 00000000000..1f738a209df --- /dev/null +++ b/searchlib/src/tests/attribute/bitvector_search_cache/bitvector_search_cache_test.cpp @@ -0,0 +1,62 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/searchlib/attribute/bitvector_search_cache.h> +#include <vespa/searchlib/common/bitvector.h> + +using namespace search; +using namespace search::attribute; + +using BitVectorSP = BitVectorSearchCache::BitVectorSP; +using Entry = BitVectorSearchCache::Entry; + +Entry::SP +makeEntry() +{ + return std::make_shared<Entry>(BitVector::create(5), 10); +} + +struct Fixture { + BitVectorSearchCache cache; + Entry::SP entry1; + Entry::SP entry2; + Fixture() + : cache(), + entry1(makeEntry()), + entry2(makeEntry()) + {} +}; + +TEST_F("require that bit vectors can be inserted and retrieved", Fixture) +{ + EXPECT_EQUAL(0u, f.cache.size()); + f.cache.insert("foo", f.entry1); + f.cache.insert("bar", f.entry2); + EXPECT_EQUAL(2u, f.cache.size()); + + EXPECT_EQUAL(f.entry1, f.cache.find("foo")); + EXPECT_EQUAL(f.entry2, f.cache.find("bar")); + EXPECT_TRUE(f.cache.find("baz").get() == nullptr); +} + +TEST_F("require that insert() doesn't replace existing bit vector", Fixture) +{ + f.cache.insert("foo", f.entry1); + f.cache.insert("foo", f.entry2); + EXPECT_EQUAL(1u, f.cache.size()); + EXPECT_EQUAL(f.entry1, f.cache.find("foo")); +} + +TEST_F("require that cache can be cleared", Fixture) +{ + f.cache.insert("foo", f.entry1); + f.cache.insert("bar", f.entry2); + EXPECT_EQUAL(2u, f.cache.size()); + f.cache.clear(); + + EXPECT_EQUAL(0u, f.cache.size()); + EXPECT_TRUE(f.cache.find("foo").get() == nullptr); + EXPECT_TRUE(f.cache.find("bar").get() == nullptr); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp index c9e0757ee16..f6ab6bb50bf 100644 --- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp +++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp @@ -1,21 +1,28 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/test/imported_attribute_fixture.h> +#include <vespa/searchcommon/attribute/search_context_params.h> #include <vespa/searchlib/attribute/imported_search_context.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> -#include <vespa/searchcommon/attribute/search_context_params.h> +#include <vespa/searchlib/queryeval/simpleresult.h> +#include <vespa/searchlib/test/imported_attribute_fixture.h> +#include <vespa/vespalib/test/insertion_operators.h> namespace search::attribute { using fef::TermFieldMatchData; +using queryeval::SearchIterator; +using queryeval::SimpleResult; using vespalib::Trinary; struct Fixture : ImportedAttributeFixture { + + Fixture(bool useSearchCache = false) : ImportedAttributeFixture(useSearchCache) {} + std::unique_ptr<ImportedSearchContext> create_context(std::unique_ptr<QueryTermSimple> term) { return std::make_unique<ImportedSearchContext>(std::move(term), SearchContextParams(), *imported_attr); } - std::unique_ptr<queryeval::SearchIterator> create_iterator( + std::unique_ptr<SearchIterator> create_iterator( ImportedSearchContext& ctx, TermFieldMatchData& match, bool strict) { @@ -25,17 +32,21 @@ struct Fixture : ImportedAttributeFixture { return iter; } - std::unique_ptr<queryeval::SearchIterator> create_non_strict_iterator( + std::unique_ptr<SearchIterator> create_non_strict_iterator( ImportedSearchContext& ctx, TermFieldMatchData& match) { return create_iterator(ctx, match, false); } - std::unique_ptr<queryeval::SearchIterator> create_strict_iterator( + std::unique_ptr<SearchIterator> create_strict_iterator( ImportedSearchContext& ctx, TermFieldMatchData& match) { return create_iterator(ctx, match, true); } + + void assertSearch(const std::vector<uint32_t> &expDocIds, SearchIterator &iter) { + EXPECT_EQUAL(SimpleResult(expDocIds), SimpleResult().searchStrict(iter, imported_attr->getNumDocs())); + } }; template <typename Iterator> @@ -339,6 +350,63 @@ TEST_F("queryTerm() returns term context was created with", WsetValueFixture) { EXPECT_EQUAL(std::string("helloworld"), std::string(ctx->queryTerm().getTerm())); } +struct SearchCacheFixture : Fixture { + SearchCacheFixture() : Fixture(true) { + reset_with_single_value_reference_mappings<IntegerAttribute, int32_t>( + BasicType::INT32, + {{DocId(3), dummy_gid(5), DocId(5), 5678}, + {DocId(4), dummy_gid(6), DocId(6), 1234}, + {DocId(5), dummy_gid(8), DocId(8), 5678}, + {DocId(7), dummy_gid(9), DocId(9), 4321}}, + FastSearchConfig::ExplicitlyEnabled, + FilterConfig::ExplicitlyEnabled); + } +}; + +BitVectorSearchCache::Entry::SP +makeSearchCacheEntry(const std::vector<uint32_t> docIds, uint32_t docIdLimit) +{ + std::shared_ptr<BitVector> bitVector = BitVector::create(docIdLimit); + for (uint32_t docId : docIds) { + bitVector->setBit(docId); + } + return std::make_shared<BitVectorSearchCache::Entry>(bitVector, docIdLimit); +} + +TEST_F("Bit vector from search cache is used if found", SearchCacheFixture) +{ + f.imported_attr->getSearchCache()->insert("5678", + makeSearchCacheEntry({2, 6}, f.imported_attr->getNumDocs())); + auto ctx = f.create_context(word_term("5678")); + ctx->fetchPostings(true); + TermFieldMatchData match; + auto iter = f.create_strict_iterator(*ctx, match); + TEST_DO(f.assertSearch({2, 6}, *iter)); // Note: would be {3, 5} if cache was not used +} + +void +assertBitVector(const std::vector<uint32_t> &expDocIds, const BitVector &bitVector) +{ + std::vector<uint32_t> actDocsIds; + bitVector.foreach_truebit([&](uint32_t docId){ actDocsIds.push_back(docId); }); + EXPECT_EQUAL(expDocIds, actDocsIds); +} + +TEST_F("Entry is inserted into search cache if bit vector posting list is used", SearchCacheFixture) +{ + EXPECT_EQUAL(0u, f.imported_attr->getSearchCache()->size()); + auto ctx = f.create_context(word_term("5678")); + ctx->fetchPostings(true); + TermFieldMatchData match; + auto iter = f.create_strict_iterator(*ctx, match); + TEST_DO(f.assertSearch({3, 5}, *iter)); + + EXPECT_EQUAL(1u, f.imported_attr->getSearchCache()->size()); + auto cacheEntry = f.imported_attr->getSearchCache()->find("5678"); + EXPECT_EQUAL(cacheEntry->docIdLimit, f.imported_attr->getNumDocs()); + TEST_DO(assertBitVector({3, 5}, *cacheEntry->bitVector)); +} + } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index aa0b63894c0..e99092f0206 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -21,6 +21,7 @@ vespa_add_library(searchlib_attribute OBJECT attributesaver.cpp attributevector.cpp attrvector.cpp + bitvector_search_cache.cpp changevector.cpp configconverter.cpp createarrayfastsearch.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.cpp b/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.cpp new file mode 100644 index 00000000000..cf49856234a --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.cpp @@ -0,0 +1,53 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "bitvector_search_cache.h" +#include <vespa/searchlib/common/bitvector.h> +#include <vespa/vespalib/stllike/hash_map.hpp> + +namespace search::attribute { + +using BitVectorSP = BitVectorSearchCache::BitVectorSP; + +BitVectorSearchCache::BitVectorSearchCache() + : _mutex(), + _cache() +{ +} + +BitVectorSearchCache::~BitVectorSearchCache() +{ +} + +void +BitVectorSearchCache::insert(const vespalib::string &term, Entry::SP entry) +{ + LockGuard guard(_mutex); + _cache.insert(std::make_pair(term, std::move(entry))); +} + +BitVectorSearchCache::Entry::SP +BitVectorSearchCache::find(const vespalib::string &term) const +{ + LockGuard guard(_mutex); + auto itr = _cache.find(term); + if (itr != _cache.end()) { + return itr->second; + } + return Entry::SP(); +} + +size_t +BitVectorSearchCache::size() const +{ + LockGuard guard(_mutex); + return _cache.size(); +} + +void +BitVectorSearchCache::clear() +{ + LockGuard guard(_mutex); + _cache.clear(); +} + +} diff --git a/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.h b/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.h new file mode 100644 index 00000000000..73fb95e1752 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.h @@ -0,0 +1,50 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/stllike/string.h> +#include <memory> +#include <mutex> + +namespace search { + +class BitVector; + +namespace attribute { + +/** + * Class that caches posting lists (as bit vectors) for a set of search terms. + * + * Lifetime of cached bit vectors is controlled by calling clear() at regular intervals. + */ +class BitVectorSearchCache { +public: + using BitVectorSP = std::shared_ptr<BitVector>; + + struct Entry { + using SP = std::shared_ptr<Entry>; + BitVectorSP bitVector; + uint32_t docIdLimit; + Entry(BitVectorSP bitVector_, uint32_t docIdLimit_) + : bitVector(std::move(bitVector_)), docIdLimit(docIdLimit_) {} + }; + +private: + using LockGuard = std::lock_guard<std::mutex>; + using Cache = vespalib::hash_map<vespalib::string, Entry::SP>; + + mutable std::mutex _mutex; + Cache _cache; + +public: + BitVectorSearchCache(); + ~BitVectorSearchCache(); + void insert(const vespalib::string &term, Entry::SP entry); + Entry::SP find(const vespalib::string &term) const; + size_t size() const; + void clear(); +}; + +} +} diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp index 036f482b2e8..9efac9ce541 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp @@ -3,7 +3,7 @@ #include "imported_attribute_vector.h" #include "imported_attribute_vector_read_guard.h" #include "imported_search_context.h" -#include "attributeguard.h" +#include "bitvector_search_cache.h" #include <vespa/searchlib/query/queryterm.h> #include <vespa/vespalib/util/exceptions.h> @@ -13,10 +13,24 @@ namespace attribute { ImportedAttributeVector::ImportedAttributeVector( vespalib::stringref name, std::shared_ptr<ReferenceAttribute> reference_attribute, - std::shared_ptr<AttributeVector> target_attribute) + std::shared_ptr<AttributeVector> target_attribute, + bool use_search_cache) : _name(name), _reference_attribute(std::move(reference_attribute)), - _target_attribute(std::move(target_attribute)) + _target_attribute(std::move(target_attribute)), + _search_cache(use_search_cache ? std::make_shared<BitVectorSearchCache>() : + std::shared_ptr<BitVectorSearchCache>()) +{ +} + +ImportedAttributeVector::ImportedAttributeVector(vespalib::stringref name, + std::shared_ptr<ReferenceAttribute> reference_attribute, + std::shared_ptr<AttributeVector> target_attribute, + std::shared_ptr<BitVectorSearchCache> search_cache) + : _name(name), + _reference_attribute(std::move(reference_attribute)), + _target_attribute(std::move(target_attribute)), + _search_cache(std::move(search_cache)) { } @@ -27,7 +41,7 @@ std::unique_ptr<ImportedAttributeVector> ImportedAttributeVector::makeReadGuard(bool stableEnumGuard) const { return std::make_unique<ImportedAttributeVectorReadGuard> - (getName(), getReferenceAttribute(), getTargetAttribute(), stableEnumGuard); + (getName(), getReferenceAttribute(), getTargetAttribute(), getSearchCache(), stableEnumGuard); } const vespalib::string& search::attribute::ImportedAttributeVector::getName() const { @@ -131,6 +145,12 @@ bool ImportedAttributeVector::hasEnum() const { return _target_attribute->hasEnum(); } +void ImportedAttributeVector::clearSearchCache() { + if (_search_cache) { + _search_cache->clear(); + } +} + long ImportedAttributeVector::onSerializeForAscendingSort(DocId doc, void *serTo, long available, diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h index bddd0fa9093..c94eda3d4db 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h @@ -14,6 +14,8 @@ class AttributeEnumGuard; namespace attribute { +class BitVectorSearchCache; + /** * Attribute vector which does not store values of its own, but rather serves as a * convenient indirection wrapper towards a target vector, usually in another @@ -29,7 +31,12 @@ public: using SP = std::shared_ptr<ImportedAttributeVector>; ImportedAttributeVector(vespalib::stringref name, std::shared_ptr<ReferenceAttribute> reference_attribute, - std::shared_ptr<AttributeVector> target_attribute); + std::shared_ptr<AttributeVector> target_attribute, + bool use_search_cache); + ImportedAttributeVector(vespalib::stringref name, + std::shared_ptr<ReferenceAttribute> reference_attribute, + std::shared_ptr<AttributeVector> target_attribute, + std::shared_ptr<BitVectorSearchCache> search_cache); ~ImportedAttributeVector(); const vespalib::string & getName() const override; @@ -65,6 +72,10 @@ public: const std::shared_ptr<AttributeVector>& getTargetAttribute() const noexcept { return _target_attribute; } + const std::shared_ptr<BitVectorSearchCache> &getSearchCache() const { + return _search_cache; + } + void clearSearchCache(); /* * Create an imported attribute with a snapshot of lid to lid mapping. @@ -77,9 +88,10 @@ protected: const common::BlobConverter * bc) const override; - vespalib::string _name; - std::shared_ptr<ReferenceAttribute> _reference_attribute; - std::shared_ptr<AttributeVector> _target_attribute; + vespalib::string _name; + std::shared_ptr<ReferenceAttribute> _reference_attribute; + std::shared_ptr<AttributeVector> _target_attribute; + std::shared_ptr<BitVectorSearchCache> _search_cache; }; } // attribute diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp index 563834e6cdb..9586e2be79b 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp @@ -9,8 +9,9 @@ ImportedAttributeVectorReadGuard::ImportedAttributeVectorReadGuard( vespalib::stringref name, std::shared_ptr<ReferenceAttribute> reference_attribute, std::shared_ptr<AttributeVector> target_attribute, + std::shared_ptr<BitVectorSearchCache> search_cache, bool stableEnumGuard) - : ImportedAttributeVector(name, std::move(reference_attribute), std::move(target_attribute)), + : ImportedAttributeVector(name, std::move(reference_attribute), std::move(target_attribute), std::move(search_cache)), _referencedLids(), _referencedLidLimit(0u), _reference_attribute_guard(_reference_attribute), diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h index f4db2b538d5..77554fdb849 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h @@ -7,6 +7,8 @@ namespace search::attribute { +class BitVectorSearchCache; + /* * Short lived attribute vector that does not store values on its own. * @@ -31,6 +33,7 @@ public: ImportedAttributeVectorReadGuard(vespalib::stringref name, std::shared_ptr<ReferenceAttribute> reference_attribute, std::shared_ptr<AttributeVector> target_attribute, + std::shared_ptr<BitVectorSearchCache> search_cache, bool stableEnumGuard); ~ImportedAttributeVectorReadGuard(); diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index 5a040941e67..26e51f77ccf 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "bitvector_search_cache.h" #include "imported_search_context.h" #include "imported_attribute_vector.h" #include <vespa/searchlib/common/bitvectoriterator.h> @@ -24,6 +25,10 @@ ImportedSearchContext::ImportedSearchContext( const SearchContextParams& params, const ImportedAttributeVector& imported_attribute) : _imported_attribute(imported_attribute), + _queryTerm(term->getTerm()), + _useSearchCache(_imported_attribute.getSearchCache().get() != nullptr), + _searchCacheLookup((_useSearchCache ? _imported_attribute.getSearchCache()->find(_queryTerm) : + std::shared_ptr<BitVectorSearchCache::Entry>())), _reference_attribute(*_imported_attribute.getReferenceAttribute()), _target_attribute(*_imported_attribute.getTargetAttribute()), _target_search_context(_target_attribute.getSearch(std::move(term), params)), @@ -43,6 +48,9 @@ unsigned int ImportedSearchContext::approximateHits() const { std::unique_ptr<queryeval::SearchIterator> ImportedSearchContext::createIterator(fef::TermFieldMatchData* matchData, bool strict) { + if (_searchCacheLookup) { + return BitVectorIterator::create(_searchCacheLookup->bitVector.get(), _searchCacheLookup->docIdLimit, *matchData, strict); + } if (_merger.hasArray()) { if (_merger.emptyArray()) { return SearchIterator::UP(new EmptySearch()); @@ -204,12 +212,24 @@ void ImportedSearchContext::makeMergedPostings(bool isFilter) _merger.merge(); } +void +ImportedSearchContext::considerAddSearchCacheEntry() +{ + if (_useSearchCache && _merger.hasBitVector()) { + auto cacheEntry = std::make_shared<BitVectorSearchCache::Entry>(_merger.getBitVectorSP(), _merger.getDocIdLimit()); + _imported_attribute.getSearchCache()->insert(_queryTerm, std::move(cacheEntry)); + } +} + void ImportedSearchContext::fetchPostings(bool strict) { assert(!_fetchPostingsDone); _fetchPostingsDone = true; - _target_search_context->fetchPostings(strict); - if (strict || _target_attribute.getConfig().fastSearch()) { - makeMergedPostings(_target_attribute.getConfig().getIsFilter()); + if (!_searchCacheLookup) { + _target_search_context->fetchPostings(strict); + if (strict || _target_attribute.getConfig().fastSearch()) { + makeMergedPostings(_target_attribute.getConfig().getIsFilter()); + considerAddSearchCacheEntry(); + } } } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h index c4a6101ac0a..f3f44332a2e 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h @@ -3,6 +3,7 @@ #pragma once #include "attributevector.h" +#include "bitvector_search_cache.h" #include <vespa/searchcommon/attribute/i_search_context.h> #include <vespa/searchlib/attribute/posting_list_merger.h> #include <vespa/vespalib/util/arrayref.h> @@ -26,6 +27,9 @@ class SearchContextParams; class ImportedSearchContext : public ISearchContext { using ReferencedLids = vespalib::ConstArrayRef<uint32_t>; const ImportedAttributeVector& _imported_attribute; + vespalib::string _queryTerm; + bool _useSearchCache; + BitVectorSearchCache::Entry::SP _searchCacheLookup; const ReferenceAttribute& _reference_attribute; const AttributeVector& _target_attribute; std::unique_ptr<AttributeVector::SearchContext> _target_search_context; @@ -40,6 +44,7 @@ class ImportedSearchContext : public ISearchContext { } void makeMergedPostings(bool isFilter); + void considerAddSearchCacheEntry(); public: ImportedSearchContext(std::unique_ptr<QueryTermSimple> term, const SearchContextParams& params, diff --git a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h index 8220b529aac..8568661dfdd 100644 --- a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h +++ b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h @@ -21,7 +21,7 @@ class PostingListMerger PostingVector _array; StartVector _startPos; - std::unique_ptr<BitVector> _bitVector; + std::shared_ptr<BitVector> _bitVector; uint32_t _docIdLimit; bool _arrayValid; @@ -39,6 +39,7 @@ public: bool emptyArray() const { return _array.empty(); } vespalib::ConstArrayRef<Posting> getArray() const { return _array; } const BitVector *getBitVector() const { return _bitVector.get(); } + const std::shared_ptr<BitVector> &getBitVectorSP() const { return _bitVector; } uint32_t getDocIdLimit() const { return _docIdLimit; } template <typename PostingListType> diff --git a/searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp b/searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp index 5195f03b8db..de2582dda6b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp @@ -20,7 +20,7 @@ SimpleResult::clear() tmp.swap(_hits); } -void +SimpleResult & SimpleResult::search(SearchIterator &sb) { clear(); @@ -30,9 +30,10 @@ SimpleResult::search(SearchIterator &sb) sb.unpack(sb.getDocId()); _hits.push_back(sb.getDocId()); } + return *this; } -void +SimpleResult & SimpleResult::searchStrict(SearchIterator &sb, uint32_t docIdLimit) { clear(); @@ -42,9 +43,10 @@ SimpleResult::searchStrict(SearchIterator &sb, uint32_t docIdLimit) sb.unpack(sb.getDocId()); _hits.push_back(sb.getDocId()); } + return *this; } -void +SimpleResult & SimpleResult::search(SearchIterator &sb, uint32_t docIdLimit) { clear(); @@ -57,6 +59,7 @@ SimpleResult::search(SearchIterator &sb, uint32_t docIdLimit) _hits.push_back(docId); } } + return *this; } std::ostream & diff --git a/searchlib/src/vespa/searchlib/queryeval/simpleresult.h b/searchlib/src/vespa/searchlib/queryeval/simpleresult.h index 30c88bb10c7..4a1fcb25e7a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simpleresult.h +++ b/searchlib/src/vespa/searchlib/queryeval/simpleresult.h @@ -23,6 +23,11 @@ public: SimpleResult() : _hits() {} /** + * Create a result with the given hits. + */ + SimpleResult(const std::vector<uint32_t> &hits) : _hits(hits) {} + + /** * Obtain the number of hits * * @return number of hits @@ -58,8 +63,8 @@ public: * * @param sb search object **/ - void search(SearchIterator &sb); - void searchStrict(SearchIterator &sb, uint32_t docIdLimit); + SimpleResult &search(SearchIterator &sb); + SimpleResult &searchStrict(SearchIterator &sb, uint32_t docIdLimit); /** * Fill this result with all the hits returned by the given search @@ -69,7 +74,7 @@ public: * @param sb search object * @param docIdLimit the end of the docId range for this search iterator **/ - void search(SearchIterator &sb, uint32_t docIdLimit); + SimpleResult &search(SearchIterator &sb, uint32_t docIdLimit); /** * Test of we contain the same hits as rhs. diff --git a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h index 85a2b140039..3620a1df725 100644 --- a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h +++ b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h @@ -44,34 +44,47 @@ enum class FastSearchConfig { Default }; +enum class FilterConfig { + ExplicitlyEnabled, + Default +}; + template<typename AttrVecType> std::shared_ptr<AttrVecType> create_typed_attribute(BasicType basic_type, CollectionType collection_type, FastSearchConfig fast_search = FastSearchConfig::Default, + FilterConfig filter = FilterConfig::Default, vespalib::stringref name = "parent") { Config cfg(basic_type, collection_type); if (fast_search == FastSearchConfig::ExplicitlyEnabled) { cfg.setFastSearch(true); } + if (filter == FilterConfig::ExplicitlyEnabled) { + cfg.setIsFilter(true); + } return std::dynamic_pointer_cast<AttrVecType>( AttributeFactory::createAttribute(name, std::move(cfg))); } template<typename AttrVecType> -std::shared_ptr<AttrVecType> create_single_attribute(BasicType type, vespalib::stringref name = "parent") { - return create_typed_attribute<AttrVecType>(type, CollectionType::SINGLE, FastSearchConfig::Default, name); +std::shared_ptr<AttrVecType> create_single_attribute(BasicType type, + FastSearchConfig fast_search = FastSearchConfig::Default, + FilterConfig filter = FilterConfig::Default, + vespalib::stringref name = "parent") { + return create_typed_attribute<AttrVecType>(type, CollectionType::SINGLE, fast_search, filter, name); } template<typename AttrVecType> std::shared_ptr<AttrVecType> create_array_attribute(BasicType type, vespalib::stringref name = "parent") { - return create_typed_attribute<AttrVecType>(type, CollectionType::ARRAY, FastSearchConfig::Default, name); + return create_typed_attribute<AttrVecType>(type, CollectionType::ARRAY, + FastSearchConfig::Default, FilterConfig::Default, name); } template<typename AttrVecType> std::shared_ptr<AttrVecType> create_wset_attribute(BasicType type, FastSearchConfig fast_search = FastSearchConfig::Default, vespalib::stringref name = "parent") { - return create_typed_attribute<AttrVecType>(type, CollectionType::WSET, fast_search, name); + return create_typed_attribute<AttrVecType>(type, CollectionType::WSET, fast_search, FilterConfig::Default, name); } template<typename VectorType> @@ -89,12 +102,13 @@ std::unique_ptr<QueryTermSimple> word_term(vespalib::stringref term) { } struct ImportedAttributeFixture { + bool use_search_cache; std::shared_ptr<AttributeVector> target_attr; std::shared_ptr<ReferenceAttribute> reference_attr; std::shared_ptr<ImportedAttributeVector> imported_attr; std::shared_ptr<MockGidToLidMapperFactory> mapper_factory; - ImportedAttributeFixture(); + ImportedAttributeFixture(bool use_search_cache_ = false); virtual ~ImportedAttributeFixture(); @@ -116,7 +130,7 @@ struct ImportedAttributeFixture { std::shared_ptr<ImportedAttributeVector> create_attribute_vector_from_members(vespalib::stringref name = default_imported_attr_name()) { - return std::make_shared<ImportedAttributeVector>(name, reference_attr, target_attr); + return std::make_shared<ImportedAttributeVector>(name, reference_attr, target_attr, use_search_cache); } template<typename AttrVecType> @@ -169,8 +183,10 @@ struct ImportedAttributeFixture { template<typename AttrVecType, typename ValueType> void reset_with_single_value_reference_mappings( BasicType type, - const std::vector<LidToLidMapping<ValueType>> &mappings) { - reset_with_new_target_attr(create_single_attribute<AttrVecType>(type)); + const std::vector<LidToLidMapping<ValueType>> &mappings, + FastSearchConfig fast_search = FastSearchConfig::Default, + FilterConfig filter = FilterConfig::Default) { + reset_with_new_target_attr(create_single_attribute<AttrVecType>(type, fast_search, filter)); // Fun experiment: rename `auto& mapping` to `auto& m` and watch GCC howl about // shadowing a variable... that exists in the set_up_and_map function! set_up_and_map<AttrVecType>(mappings, [this](auto &target_vec, auto &mapping) { @@ -205,8 +221,9 @@ struct ImportedAttributeFixture { } }; -ImportedAttributeFixture::ImportedAttributeFixture() - : target_attr(create_single_attribute<IntegerAttribute>(BasicType::INT32)), +ImportedAttributeFixture::ImportedAttributeFixture(bool use_search_cache_) + : use_search_cache(use_search_cache_), + target_attr(create_single_attribute<IntegerAttribute>(BasicType::INT32)), reference_attr(create_reference_attribute()), imported_attr(create_attribute_vector_from_members()), mapper_factory(std::make_shared<MockGidToLidMapperFactory>()) { @@ -240,8 +257,10 @@ template<typename AttrVecType, typename ValueType> void reset_with_single_value_reference_mappings( ImportedAttributeFixture &f, BasicType type, - const std::vector<ImportedAttributeFixture::LidToLidMapping<ValueType>> &mappings) { - f.reset_with_single_value_reference_mappings<AttrVecType, ValueType>(type, mappings); + const std::vector<ImportedAttributeFixture::LidToLidMapping<ValueType>> &mappings, + FastSearchConfig fast_search = FastSearchConfig::Default, + FilterConfig filter = FilterConfig::Default) { + f.reset_with_single_value_reference_mappings<AttrVecType, ValueType>(type, mappings, fast_search, filter); } template<typename AttrVecType, typename ValueType> |