summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGeir Storli <geirstorli@yahoo.no>2017-09-07 09:25:22 +0200
committerGitHub <noreply@github.com>2017-09-07 09:25:22 +0200
commita9b48c32108ed326d80ed0f6fac6ec4d6963df75 (patch)
treec83b9cb87fa06c5c9b945b885f3b9f7cb57809d2
parente42cd7c1d9b13835163ee0fbf4876f9c86fcf998 (diff)
parent4119e49f014b01395fe67b9d95e4fd3b1616ca3d (diff)
Merge pull request #3353 from vespa-engine/geirst/add-bitvector-search-cache-to-imported-attribute-vectors
Geirst/add bitvector search cache to imported attribute vectors
-rw-r--r--searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp2
-rw-r--r--searchcore/src/tests/proton/attribute/attribute_test.cpp83
-rw-r--r--searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp3
-rw-r--r--searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp3
-rw-r--r--searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp12
-rw-r--r--searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp4
-rw-r--r--searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp2
-rw-r--r--searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp3
-rw-r--r--searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp23
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp17
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h3
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp6
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h1
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp292
-rw-r--r--searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp12
-rw-r--r--searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h7
-rw-r--r--searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h5
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp5
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp3
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp3
-rw-r--r--searchlib/CMakeLists.txt1
-rw-r--r--searchlib/src/tests/attribute/bitvector_search_cache/CMakeLists.txt8
-rw-r--r--searchlib/src/tests/attribute/bitvector_search_cache/bitvector_search_cache_test.cpp62
-rw-r--r--searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp78
-rw-r--r--searchlib/src/vespa/searchlib/attribute/CMakeLists.txt1
-rw-r--r--searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.cpp53
-rw-r--r--searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.h50
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp28
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h20
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp3
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h3
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp26
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_search_context.h5
-rw-r--r--searchlib/src/vespa/searchlib/attribute/posting_list_merger.h3
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp9
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/simpleresult.h11
-rw-r--r--searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h43
43 files changed, 666 insertions, 239 deletions
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
index 834d7145ed0..5b64fa80d39 100644
--- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
@@ -133,7 +133,7 @@ struct ImportedAttributesRepoBuilder {
void add(const vespalib::string &name) {
auto refAttr = std::make_shared<ReferenceAttribute>(name + "_ref", AVConfig(BasicType::REFERENCE));
auto targetAttr = search::AttributeFactory::createAttribute(name + "_target", INT32_SINGLE);
- auto importedAttr = std::make_shared<ImportedAttributeVector>(name, refAttr, targetAttr);
+ auto importedAttr = std::make_shared<ImportedAttributeVector>(name, refAttr, targetAttr, false);
_repo->add(name, importedAttr);
}
ImportedAttributesRepo::UP build() {
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp
index e6052c02ee6..1112cf75570 100644
--- a/searchcore/src/tests/proton/attribute/attribute_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp
@@ -2,44 +2,45 @@
#include <vespa/log/log.h>
LOG_SETUP("attribute_test");
-#include <vespa/fastos/file.h>
-#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/config-attributes.h>
#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/predicate/predicate_slime_builder.h>
#include <vespa/document/update/arithmeticvalueupdate.h>
+#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/eval/tensor/default_tensor.h>
+#include <vespa/eval/tensor/tensor.h>
+#include <vespa/eval/tensor/tensor_factory.h>
+#include <vespa/eval/tensor/types.h>
+#include <vespa/fastos/file.h>
#include <vespa/searchcommon/attribute/attributecontent.h>
#include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h>
#include <vespa/searchcore/proton/attribute/attribute_writer.h>
#include <vespa/searchcore/proton/attribute/attributemanager.h>
#include <vespa/searchcore/proton/attribute/filter_attribute_manager.h>
+#include <vespa/searchcore/proton/attribute/imported_attributes_repo.h>
+#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/searchcore/proton/test/attribute_utils.h>
#include <vespa/searchcorespi/flush/iflushtarget.h>
#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/bitvector_search_cache.h>
+#include <vespa/searchlib/attribute/imported_attribute_vector.h>
#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/attribute/predicate_attribute.h>
+#include <vespa/searchlib/attribute/singlenumericattribute.hpp>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
#include <vespa/searchlib/common/idestructorcallback.h>
+#include <vespa/searchlib/common/sequencedtaskexecutorobserver.h>
#include <vespa/searchlib/index/docbuilder.h>
#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/predicate/predicate_hash.h>
+#include <vespa/searchlib/predicate/predicate_index.h>
+#include <vespa/searchlib/tensor/tensor_attribute.h>
+#include <vespa/searchlib/test/directory_handler.h>
#include <vespa/searchlib/util/filekit.h>
-#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/vespalib/io/fileutil.h>
#include <vespa/vespalib/test/insertion_operators.h>
-
-#include <vespa/document/predicate/predicate_slime_builder.h>
-#include <vespa/document/update/assignvalueupdate.h>
-#include <vespa/searchlib/attribute/attributevector.hpp>
-#include <vespa/searchlib/attribute/predicate_attribute.h>
-#include <vespa/searchlib/predicate/predicate_index.h>
-#include <vespa/searchlib/attribute/singlenumericattribute.hpp>
-#include <vespa/searchlib/predicate/predicate_hash.h>
-#include <vespa/searchlib/common/foregroundtaskexecutor.h>
-#include <vespa/searchlib/common/sequencedtaskexecutorobserver.h>
-#include <vespa/searchlib/test/directory_handler.h>
-#include <vespa/eval/tensor/tensor.h>
-#include <vespa/eval/tensor/types.h>
-#include <vespa/eval/tensor/default_tensor.h>
-#include <vespa/eval/tensor/tensor_factory.h>
-#include <vespa/searchlib/tensor/tensor_attribute.h>
-#include <vespa/config-attributes.h>
-
+#include <vespa/vespalib/testkit/testapp.h>
namespace vespa { namespace config { namespace search {}}}
@@ -50,8 +51,12 @@ using namespace search::index;
using namespace search;
using namespace vespa::config::search;
+using proton::ImportedAttributesRepo;
using proton::test::AttributeUtils;
using search::TuneFileAttributes;
+using search::attribute::BitVectorSearchCache;
+using search::attribute::ImportedAttributeVector;
+using search::attribute::ReferenceAttribute;
using search::index::DummyFileHeaderContext;
using search::index::schema::CollectionType;
using search::predicate::PredicateHash;
@@ -152,7 +157,7 @@ struct Fixture
_aw->remove(serialNum, lid, immediateCommit, emptyCallback);
}
void commit(SerialNum serialNum) {
- _aw->commit(serialNum, emptyCallback);
+ _aw->forceCommit(serialNum, emptyCallback);
}
void assertExecuteHistory(std::vector<uint32_t> expExecuteHistory) {
EXPECT_EQUAL(expExecuteHistory, _attributeFieldWriter.getExecuteHistory());
@@ -345,14 +350,14 @@ TEST_F("require that visibilitydelay is honoured", Fixture)
awDelayed.put(5, *doc, 4, false, emptyCallback);
EXPECT_EQUAL(5u, a1->getNumDocs());
EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken());
- awDelayed.commit(6, emptyCallback);
+ awDelayed.forceCommit(6, emptyCallback);
EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken());
AttributeWriter awDelayedShort(f._m);
awDelayedShort.put(7, *doc, 2, false, emptyCallback);
EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken());
awDelayedShort.put(8, *doc, 2, false, emptyCallback);
- awDelayedShort.commit(8, emptyCallback);
+ awDelayedShort.forceCommit(8, emptyCallback);
EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken());
verifyAttributeContent(*a1, 2, "10");
@@ -364,7 +369,7 @@ TEST_F("require that visibilitydelay is honoured", Fixture)
2, false, emptyCallback);
EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken());
verifyAttributeContent(*a1, 2, "10");
- awDelayed.commit(12, emptyCallback);
+ awDelayed.forceCommit(12, emptyCallback);
EXPECT_EQUAL(12u, a1->getStatus().getLastSyncToken());
verifyAttributeContent(*a1, 2, "30");
@@ -712,6 +717,34 @@ TEST_F("require that attribute writer spreads write over 3 write contexts", Fixt
TEST_DO(putAttributes(f, {0, 1, 2}));
}
+ImportedAttributeVector::SP
+createImportedAttribute(const vespalib::string &name)
+{
+ auto result = std::make_shared<ImportedAttributeVector>(name,
+ ReferenceAttribute::SP(),
+ AttributeVector::SP(),
+ true);
+ result->getSearchCache()->insert("foo", BitVectorSearchCache::Entry::SP());
+ return result;
+}
+
+ImportedAttributesRepo::UP
+createImportedAttributesRepo()
+{
+ auto result = std::make_unique<ImportedAttributesRepo>();
+ result->add("imported_a", createImportedAttribute("imported_a"));
+ result->add("imported_b", createImportedAttribute("imported_b"));
+ return result;
+}
+
+TEST_F("require that AttributeWriter::forceCommit() clears search cache in imported attribute vectors", Fixture)
+{
+ f._m->setImportedAttributes(createImportedAttributesRepo());
+ f.commit(10);
+ EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_a")->getSearchCache()->size());
+ EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_b")->getSearchCache()->size());
+}
+
TEST_MAIN()
{
vespalib::rmdir(test_dir, true);
diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp
index 3b2c87b3840..d95d79fbd11 100644
--- a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp
+++ b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp
@@ -70,7 +70,8 @@ struct Fixture {
Fixture &addAttribute(const vespalib::string &name) {
auto attr = std::make_shared<ImportedAttributeVector>(name,
createReferenceAttribute(name + "_ref"),
- createTargetAttribute(name + "_target"));
+ createTargetAttribute(name + "_target"),
+ false);
repo.add(name, attr);
return *this;
}
diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp
index 99b1d11e862..c41b88be2b4 100644
--- a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp
+++ b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp
@@ -21,7 +21,8 @@ createAttr(const vespalib::string &name)
{
return std::make_shared<ImportedAttributeVector>(name,
ReferenceAttribute::SP(),
- AttributeVector::SP());
+ AttributeVector::SP(),
+ false);
}
struct Fixture {
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
index d76117df4c2..45bda8c69cc 100644
--- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
@@ -141,7 +141,9 @@ struct EmptyConstantValueFactory : public vespalib::eval::ConstantValueFactory {
};
struct MyDocumentDBReferenceResolver : public IDocumentDBReferenceResolver {
- std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &, const search::IAttributeManager &) override {
+ std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &,
+ const search::IAttributeManager &,
+ fastos::TimeStamp) override {
return std::make_unique<ImportedAttributesRepo>();
}
void teardown(const search::IAttributeManager &) override { }
@@ -657,12 +659,18 @@ TEST_F("require that we can reconfigure matchers", Fixture)
}
}
-TEST("require that attribute manager should change when imported fields has changed")
+TEST("require that attribute manager (imported attributes) should change when imported fields has changed")
{
ReconfigParams params(CCR().setImportedFieldsChanged(true));
EXPECT_TRUE(params.shouldAttributeManagerChange());
}
+TEST("require that attribute manager (imported attributes) should change when visibility delay has changed")
+{
+ ReconfigParams params(CCR().setVisibilityDelayChanged(true));
+ EXPECT_TRUE(params.shouldAttributeManagerChange());
+}
+
void
assertMaintenanceControllerShouldNotChange(DocumentDBConfig::ComparisonResult result)
{
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index c3928e193e8..8cd57b1b0cb 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -108,7 +108,9 @@ struct MyMetricsWireService : public DummyWireService
};
struct MyDocumentDBReferenceResolver : public IDocumentDBReferenceResolver {
- std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &, const search::IAttributeManager &) override {
+ std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &,
+ const search::IAttributeManager &,
+ fastos::TimeStamp) override {
return std::make_unique<ImportedAttributesRepo>();
}
void teardown(const search::IAttributeManager &) override { }
diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
index 8af365d6327..c02aa033bb9 100644
--- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
@@ -385,7 +385,7 @@ struct MyAttributeWriter : public IAttributeWriter
virtual const proton::IAttributeManager::SP &getAttributeManager() const override {
return _mgr;
}
- void commit(SerialNum serialNum, OnWriteDoneType) override {
+ void forceCommit(SerialNum serialNum, OnWriteDoneType) override {
(void) serialNum; ++_commitCount;
_tracer.traceCommit(attributeAdapterTypeName, serialNum);
}
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
index 6eb447ac968..4b4e5b955a1 100644
--- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
@@ -384,6 +384,9 @@ struct MyAttributeManager : public proton::IAttributeManager
virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo>) override {
abort();
}
+ virtual const ImportedAttributesRepo *getImportedAttributes() const override {
+ abort();
+ }
};
struct MockLidSpaceCompactionHandler : public ILidSpaceCompactionHandler
diff --git a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp
index 65e15a3efaa..a6576d11279 100644
--- a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp
+++ b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp
@@ -224,9 +224,12 @@ struct Fixture {
oldAttrMgr.addReferenceAttribute("parent2_ref");
oldAttrMgr.addReferenceAttribute("parent3_ref");
}
- ImportedAttributesRepo::UP resolve() {
+ ImportedAttributesRepo::UP resolve(fastos::TimeStamp visibilityDelay) {
DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter);
- return resolver.resolve(attrMgr, oldAttrMgr);
+ return resolver.resolve(attrMgr, oldAttrMgr, visibilityDelay);
+ }
+ ImportedAttributesRepo::UP resolve() {
+ return resolve(fastos::TimeStamp(0));
}
void teardown() {
DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter);
@@ -238,11 +241,13 @@ struct Fixture {
void assertImportedAttribute(const vespalib::string &name,
const vespalib::string &referenceField,
const vespalib::string &targetField,
+ bool useSearchCache,
ImportedAttributeVector::SP attr) {
ASSERT_TRUE(attr.get());
EXPECT_EQUAL(name, attr->getName());
EXPECT_EQUAL(attrMgr.getReferenceAttribute(referenceField), attr->getReferenceAttribute().get());
EXPECT_EQUAL(parentReference->getAttribute(targetField).get(), attr->getTargetAttribute().get());
+ EXPECT_EQUAL(useSearchCache, attr->getSearchCache().get() != nullptr);
}
MockGidToLidChangeHandler &getGidToLidChangeHandler(const vespalib::string &referencedDocTypeName) {
@@ -272,12 +277,20 @@ TEST_F("require that reference attributes are connected to gid mapper", Fixture)
EXPECT_EQUAL(f.factory.get(), f.getMapperFactoryPtr("other_ref"));
}
-TEST_F("require that imported attributes are instantiated", Fixture)
+TEST_F("require that imported attributes are instantiated without search cache as default", Fixture)
{
auto repo = f.resolve();
EXPECT_EQUAL(2u, repo->size());
- f.assertImportedAttribute("imported_a", "ref", "target_a", repo->get("imported_a"));
- f.assertImportedAttribute("imported_b", "other_ref", "target_b", repo->get("imported_b"));
+ f.assertImportedAttribute("imported_a", "ref", "target_a", false, repo->get("imported_a"));
+ f.assertImportedAttribute("imported_b", "other_ref", "target_b", false, repo->get("imported_b"));
+}
+
+TEST_F("require that imported attributes are instantiated with search cache if visibility delay > 0", Fixture)
+{
+ auto repo = f.resolve(fastos::TimeStamp::Seconds(1.0));
+ EXPECT_EQUAL(2u, repo->size());
+ f.assertImportedAttribute("imported_a", "ref", "target_a", true, repo->get("imported_a"));
+ f.assertImportedAttribute("imported_b", "other_ref", "target_b", true, repo->get("imported_b"));
}
TEST_F("require that listeners are added", Fixture)
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
index 6d4915d800d..906f71fecb4 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
@@ -2,18 +2,20 @@
#include "attribute_writer.h"
#include "attributemanager.h"
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/searchcore/proton/attribute/imported_attributes_repo.h>
#include <vespa/searchcore/proton/common/attrupdate.h>
#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/imported_attribute_vector.h>
#include <vespa/searchlib/common/isequencedtaskexecutor.h>
-#include <vespa/document/datatype/documenttype.h>
#include <vespa/log/log.h>
-#include <vespa/document/base/exceptions.h>
-
LOG_SETUP(".proton.server.attributeadapter");
using namespace document;
using namespace search;
+using search::attribute::ImportedAttributeVector;
namespace proton {
@@ -459,8 +461,15 @@ AttributeWriter::heartBeat(SerialNum serialNum)
void
-AttributeWriter::commit(SerialNum serialNum, OnWriteDoneType onWriteDone)
+AttributeWriter::forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone)
{
+ if (_mgr->getImportedAttributes() != nullptr) {
+ std::vector<std::shared_ptr<ImportedAttributeVector>> importedAttrs;
+ _mgr->getImportedAttributes()->getAll(importedAttrs);
+ for (const auto &attr : importedAttrs) {
+ attr->clearSearchCache();
+ }
+ }
for (const auto &wc : _writeContexts) {
auto commitTask = std::make_unique<CommitTask>(wc, serialNum, onWriteDone);
_attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(commitTask));
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
index 58b92459358..cbda11180c0 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
@@ -74,7 +74,7 @@ public:
const proton::IAttributeManager::SP &getAttributeManager() const override {
return _mgr;
}
- void commit(SerialNum serialNum, OnWriteDoneType onWriteDone) override;
+ void forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) override;
virtual void onReplayDone(uint32_t docIdLimit) override;
};
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
index a7de9e2efa4..7148815ca9e 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
@@ -155,7 +155,6 @@ public:
static void padAttribute(search::AttributeVector &v, uint32_t docIdLimit);
- const ImportedAttributesRepo *getImportedAttributes() const { return _importedAttributes.get(); }
// Implements search::IAttributeManager
virtual search::AttributeGuard::UP getAttribute(const vespalib::string &name) const override;
@@ -200,6 +199,8 @@ public:
virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override;
virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override;
+
+ virtual const ImportedAttributesRepo *getImportedAttributes() const override { return _importedAttributes.get(); }
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
index 9f78a431430..af64f47fc0a 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
@@ -213,4 +213,10 @@ FilterAttributeManager::setImportedAttributes(std::unique_ptr<ImportedAttributes
throw vespalib::IllegalArgumentException("Not implemented");
}
+const ImportedAttributesRepo *
+FilterAttributeManager::getImportedAttributes() const
+{
+ throw vespalib::IllegalArgumentException("Not implemented");
+}
+
}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
index 958bf78eeb6..81fafa477ee 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
@@ -53,6 +53,7 @@ public:
virtual void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const override;
virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override;
virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override;
+ virtual const ImportedAttributesRepo *getImportedAttributes() const override;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
index eb9bde1048f..31bd7c27c85 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
@@ -101,6 +101,8 @@ struct IAttributeManager : public search::IAttributeManager
virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const = 0;
virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) = 0;
+
+ virtual const ImportedAttributesRepo *getImportedAttributes() const = 0;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h
index 5903b4d76e4..7455ccda5a3 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h
@@ -61,7 +61,7 @@ public:
/**
* Commit all underlying attribute vectors with the given serial number.
*/
- virtual void commit(SerialNum serialNum, OnWriteDoneType onWriteDone) = 0;
+ virtual void forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) = 0;
virtual void onReplayDone(uint32_t docIdLimit) = 0;
};
diff --git a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp
index 5ac5cf3488e..b6c6c798506 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp
@@ -13,159 +13,159 @@ using search::grouping::GroupingSession;
namespace proton::matching {
- namespace {
- using Stats = SessionManager::Stats;
- struct SessionCacheBase {
- protected:
- Stats _stats;
- vespalib::Lock _lock;
-
- void entryDropped(const SessionId &id);
- ~SessionCacheBase() {}
- };
-
- template <typename T>
- struct SessionCache : SessionCacheBase {
- typedef typename T::UP EntryUP;
- vespalib::lrucache_map<vespalib::LruParam<SessionId, EntryUP> > _cache;
-
- SessionCache(uint32_t max_size) : _cache(max_size) {}
-
- void insert(EntryUP session) {
- vespalib::LockGuard guard(_lock);
- const SessionId &id(session->getSessionId());
- if (_cache.size() >= _cache.capacity()) {
- entryDropped(id);
- }
- _cache.insert(id, std::move(session));
- _stats.numInsert++;
- }
- EntryUP pick(const SessionId & id) {
- vespalib::LockGuard guard(_lock);
- EntryUP ret;
- if (_cache.hasKey(id)) {
- _stats.numPick++;
- ret = std::move(_cache[id]);
- _cache.erase(id);
- }
- return ret;
- }
- void pruneTimedOutSessions(fastos::TimeStamp currentTime) {
- std::vector<EntryUP> toDestruct = stealTimedOutSessions(currentTime);
- toDestruct.clear();
- }
- std::vector<EntryUP> stealTimedOutSessions(fastos::TimeStamp currentTime) {
- std::vector<EntryUP> toDestruct;
- vespalib::LockGuard guard(_lock);
- toDestruct.reserve(_cache.size());
- for (auto it(_cache.begin()), mt(_cache.end()); it != mt;) {
- auto &session = *it;
- if (session->getTimeOfDoom() < currentTime) {
- toDestruct.push_back(std::move(session));
- it = _cache.erase(it);
- _stats.numTimedout++;
- } else {
- it++;
- }
- }
- return toDestruct;
- }
- Stats getStats() {
- vespalib::LockGuard guard(_lock);
- Stats stats = _stats;
- stats.numCached = _cache.size();
- _stats = Stats();
- return stats;
- }
- bool empty() const {
- vespalib::LockGuard guard(_lock);
- return _cache.empty();
- }
- };
-
- template <typename T>
- struct SessionMap : SessionCacheBase {
- typedef typename T::SP EntrySP;
- vespalib::hash_map<SessionId, EntrySP> _map;
-
- void insert(EntrySP session) {
- vespalib::LockGuard guard(_lock);
- const SessionId &id(session->getSessionId());
- _map.insert(std::make_pair(id, session));
- _stats.numInsert++;
- }
- EntrySP pick(const SessionId & id) {
- vespalib::LockGuard guard(_lock);
- auto it = _map.find(id);
- if (it != _map.end()) {
- _stats.numPick++;
- return it->second;
- }
- return EntrySP();
- }
- void pruneTimedOutSessions(fastos::TimeStamp currentTime) {
- std::vector<EntrySP> toDestruct = stealTimedOutSessions(currentTime);
- toDestruct.clear();
- }
- std::vector<EntrySP> stealTimedOutSessions(fastos::TimeStamp currentTime) {
- std::vector<EntrySP> toDestruct;
- std::vector<SessionId> keys;
- vespalib::LockGuard guard(_lock);
- keys.reserve(_map.size());
- toDestruct.reserve(_map.size());
- for (auto & it : _map) {
- EntrySP &session = it.second;
- if (session->getTimeOfDoom() < currentTime) {
- keys.push_back(it.first);
- toDestruct.push_back(EntrySP());
- toDestruct.back().swap(session);
- }
- }
- for (auto key : keys) {
- _map.erase(key);
- _stats.numTimedout++;
- }
- return toDestruct;
- }
- Stats getStats() {
- vespalib::LockGuard guard(_lock);
- Stats stats = _stats;
- stats.numCached = _map.size();
- _stats = Stats();
- return stats;
- }
- size_t size() const {
- vespalib::LockGuard guard(_lock);
- return _map.size();
- }
- bool empty() const {
- vespalib::LockGuard guard(_lock);
- return _map.empty();
+namespace {
+using Stats = SessionManager::Stats;
+struct SessionCacheBase {
+protected:
+ Stats _stats;
+ vespalib::Lock _lock;
+
+ void entryDropped(const SessionId &id);
+ ~SessionCacheBase() {}
+};
+
+template <typename T>
+struct SessionCache : SessionCacheBase {
+ typedef typename T::UP EntryUP;
+ vespalib::lrucache_map<vespalib::LruParam<SessionId, EntryUP> > _cache;
+
+ SessionCache(uint32_t max_size) : _cache(max_size) {}
+
+ void insert(EntryUP session) {
+ vespalib::LockGuard guard(_lock);
+ const SessionId &id(session->getSessionId());
+ if (_cache.size() >= _cache.capacity()) {
+ entryDropped(id);
+ }
+ _cache.insert(id, std::move(session));
+ _stats.numInsert++;
+ }
+ EntryUP pick(const SessionId & id) {
+ vespalib::LockGuard guard(_lock);
+ EntryUP ret;
+ if (_cache.hasKey(id)) {
+ _stats.numPick++;
+ ret = std::move(_cache[id]);
+ _cache.erase(id);
+ }
+ return ret;
+ }
+ void pruneTimedOutSessions(fastos::TimeStamp currentTime) {
+ std::vector<EntryUP> toDestruct = stealTimedOutSessions(currentTime);
+ toDestruct.clear();
+ }
+ std::vector<EntryUP> stealTimedOutSessions(fastos::TimeStamp currentTime) {
+ std::vector<EntryUP> toDestruct;
+ vespalib::LockGuard guard(_lock);
+ toDestruct.reserve(_cache.size());
+ for (auto it(_cache.begin()), mt(_cache.end()); it != mt;) {
+ auto &session = *it;
+ if (session->getTimeOfDoom() < currentTime) {
+ toDestruct.push_back(std::move(session));
+ it = _cache.erase(it);
+ _stats.numTimedout++;
+ } else {
+ it++;
}
- template <typename F>
- void each(F f) const {
- vespalib::LockGuard guard(_lock);
- for (const auto &entry: _map) {
- f(*entry.second);
- }
+ }
+ return toDestruct;
+ }
+ Stats getStats() {
+ vespalib::LockGuard guard(_lock);
+ Stats stats = _stats;
+ stats.numCached = _cache.size();
+ _stats = Stats();
+ return stats;
+ }
+ bool empty() const {
+ vespalib::LockGuard guard(_lock);
+ return _cache.empty();
+ }
+};
+
+template <typename T>
+struct SessionMap : SessionCacheBase {
+ typedef typename T::SP EntrySP;
+ vespalib::hash_map<SessionId, EntrySP> _map;
+
+ void insert(EntrySP session) {
+ vespalib::LockGuard guard(_lock);
+ const SessionId &id(session->getSessionId());
+ _map.insert(std::make_pair(id, session));
+ _stats.numInsert++;
+ }
+ EntrySP pick(const SessionId & id) {
+ vespalib::LockGuard guard(_lock);
+ auto it = _map.find(id);
+ if (it != _map.end()) {
+ _stats.numPick++;
+ return it->second;
+ }
+ return EntrySP();
+ }
+ void pruneTimedOutSessions(fastos::TimeStamp currentTime) {
+ std::vector<EntrySP> toDestruct = stealTimedOutSessions(currentTime);
+ toDestruct.clear();
+ }
+ std::vector<EntrySP> stealTimedOutSessions(fastos::TimeStamp currentTime) {
+ std::vector<EntrySP> toDestruct;
+ std::vector<SessionId> keys;
+ vespalib::LockGuard guard(_lock);
+ keys.reserve(_map.size());
+ toDestruct.reserve(_map.size());
+ for (auto & it : _map) {
+ EntrySP &session = it.second;
+ if (session->getTimeOfDoom() < currentTime) {
+ keys.push_back(it.first);
+ toDestruct.push_back(EntrySP());
+ toDestruct.back().swap(session);
}
- };
-
- void SessionCacheBase::entryDropped(const SessionId &id) {
- LOG(debug, "Session cache is full, dropping entry to fit session '%s'", id.c_str());
- _stats.numDropped++;
}
-
+ for (auto key : keys) {
+ _map.erase(key);
+ _stats.numTimedout++;
+ }
+ return toDestruct;
}
+ Stats getStats() {
+ vespalib::LockGuard guard(_lock);
+ Stats stats = _stats;
+ stats.numCached = _map.size();
+ _stats = Stats();
+ return stats;
+ }
+ size_t size() const {
+ vespalib::LockGuard guard(_lock);
+ return _map.size();
+ }
+ bool empty() const {
+ vespalib::LockGuard guard(_lock);
+ return _map.empty();
+ }
+ template <typename F>
+ void each(F f) const {
+ vespalib::LockGuard guard(_lock);
+ for (const auto &entry: _map) {
+ f(*entry.second);
+ }
+ }
+};
+
+void SessionCacheBase::entryDropped(const SessionId &id) {
+ LOG(debug, "Session cache is full, dropping entry to fit session '%s'", id.c_str());
+ _stats.numDropped++;
+}
+
+}
- struct GroupingSessionCache : public SessionCache<search::grouping::GroupingSession> {
- using Parent = SessionCache<search::grouping::GroupingSession>;
- using Parent::Parent;
- };
+struct GroupingSessionCache : public SessionCache<search::grouping::GroupingSession> {
+ using Parent = SessionCache<search::grouping::GroupingSession>;
+ using Parent::Parent;
+};
- struct SearchSessionCache : public SessionMap<SearchSession> {
+struct SearchSessionCache : public SessionMap<SearchSession> {
- };
+};
SessionManager::SessionManager(uint32_t maxSize)
diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp
index 5147840d5a3..c56aa18c1f7 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp
+++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp
@@ -135,13 +135,15 @@ DocumentDBReferenceResolver::listenToGidToLidChanges(const IAttributeManager &at
}
ImportedAttributesRepo::UP
-DocumentDBReferenceResolver::createImportedAttributesRepo(const IAttributeManager &attrMgr)
+DocumentDBReferenceResolver::createImportedAttributesRepo(const IAttributeManager &attrMgr,
+ bool useSearchCache)
{
auto result = std::make_unique<ImportedAttributesRepo>();
for (const auto &attr : _importedFieldsCfg.attribute) {
ReferenceAttribute::SP refAttr = getReferenceAttribute(attr.referencefield, attrMgr);
AttributeVector::SP targetAttr = getTargetDocumentDB(refAttr->getName())->getAttribute(attr.targetfield);
- ImportedAttributeVector::SP importedAttr = std::make_shared<ImportedAttributeVector>(attr.name, refAttr, targetAttr);
+ ImportedAttributeVector::SP importedAttr =
+ std::make_shared<ImportedAttributeVector>(attr.name, refAttr, targetAttr, useSearchCache);
result->add(importedAttr->getName(), importedAttr);
}
return result;
@@ -169,12 +171,14 @@ DocumentDBReferenceResolver::~DocumentDBReferenceResolver()
}
ImportedAttributesRepo::UP
-DocumentDBReferenceResolver::resolve(const IAttributeManager &newAttrMgr, const IAttributeManager &oldAttrMgr)
+DocumentDBReferenceResolver::resolve(const IAttributeManager &newAttrMgr,
+ const IAttributeManager &oldAttrMgr,
+ fastos::TimeStamp visibilityDelay)
{
connectReferenceAttributesToGidMapper(newAttrMgr);
detectOldListeners(oldAttrMgr);
listenToGidToLidChanges(newAttrMgr);
- return createImportedAttributesRepo(newAttrMgr);
+ return createImportedAttributesRepo(newAttrMgr, (visibilityDelay > 0));
}
void
diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h
index cd58d4b5365..8e0a4178622 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h
+++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h
@@ -36,7 +36,8 @@ private:
GidToLidChangeRegistrator &getRegistrator(const vespalib::string &docTypeName);
std::shared_ptr<IDocumentDBReference> getTargetDocumentDB(const vespalib::string &refAttrName) const;
void connectReferenceAttributesToGidMapper(const search::IAttributeManager &attrMgr);
- std::unique_ptr<ImportedAttributesRepo> createImportedAttributesRepo(const search::IAttributeManager &attrMgr);
+ std::unique_ptr<ImportedAttributesRepo> createImportedAttributesRepo(const search::IAttributeManager &attrMgr,
+ bool useSearchCache);
void detectOldListeners(const search::IAttributeManager &attrMgr);
void listenToGidToLidChanges(const search::IAttributeManager &attrMgr);
@@ -49,7 +50,9 @@ public:
search::ISequencedTaskExecutor &attributeFieldWriter);
~DocumentDBReferenceResolver();
- virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr, const search::IAttributeManager &oldAttrMgr) override;
+ virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr,
+ const search::IAttributeManager &oldAttrMgr,
+ fastos::TimeStamp visibilityDelay) override;
virtual void teardown(const search::IAttributeManager &oldAttrMgr) override;
};
diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h
index c0d1ab6f68c..7e5b26ac34c 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h
+++ b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once
+#include <vespa/fastos/timestamp.h>
#include <memory>
namespace search { class IAttributeManager; }
@@ -14,7 +15,9 @@ class ImportedAttributesRepo;
*/
struct IDocumentDBReferenceResolver {
virtual ~IDocumentDBReferenceResolver() {}
- virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr, const search::IAttributeManager &oldAttrMgr) = 0;
+ virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr,
+ const search::IAttributeManager &oldAttrMgr,
+ fastos::TimeStamp visibilityDelay) = 0;
virtual void teardown(const search::IAttributeManager &oldAttrMgr) = 0;
};
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
index d4e10b677a5..86d43517d00 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
@@ -40,7 +40,8 @@ DocumentDBConfig::ComparisonResult::ComparisonResult()
importedFieldsChanged(false),
tuneFileDocumentDBChanged(false),
schemaChanged(false),
- maintenanceChanged(false)
+ maintenanceChanged(false),
+ visibilityDelayChanged(false)
{ }
DocumentDBConfig::DocumentDBConfig(
@@ -171,6 +172,8 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const
retval.maintenanceChanged =
!equals<DocumentDBMaintenanceConfig>(_maintenance.get(),
rhs._maintenance.get());
+ retval.visibilityDelayChanged =
+ (_maintenance->getVisibilityDelay() != rhs._maintenance->getVisibilityDelay());
return retval;
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
index 69e9354b741..773450af620 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
@@ -49,6 +49,7 @@ public:
bool tuneFileDocumentDBChanged;
bool schemaChanged;
bool maintenanceChanged;
+ bool visibilityDelayChanged;
ComparisonResult();
ComparisonResult &setRankProfilesChanged(bool val) { rankProfilesChanged = val; return *this; }
@@ -64,6 +65,7 @@ public:
ComparisonResult &setTuneFileDocumentDBChanged(bool val) { tuneFileDocumentDBChanged = val; return *this; }
ComparisonResult &setSchemaChanged(bool val) { schemaChanged = val; return *this; }
ComparisonResult &setMaintenanceChanged(bool val) { maintenanceChanged = val; return *this; }
+ ComparisonResult &setVisibilityDelayChanged(bool val) { visibilityDelayChanged = val; return *this; }
};
using SP = std::shared_ptr<DocumentDBConfig>;
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
index 8f2e6081b95..692bb12213e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
@@ -328,8 +328,7 @@ FastAccessDocSubDB::onReprocessDone(SerialNum serialNum)
IFeedView::SP feedView = _iFeedView.get();
IAttributeWriter::SP attrWriter =
static_cast<FastAccessFeedView &>(*feedView).getAttributeWriter();
- attrWriter->commit(serialNum,
- std::shared_ptr<search::IDestructorCallback>());
+ attrWriter->forceCommit(serialNum, std::shared_ptr<search::IDestructorCallback>());
_writeService.attributeFieldWriter().sync();
_writeService.summary().sync();
Parent::onReprocessDone(serialNum);
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
index a48c6373c00..2444223c96f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
@@ -106,7 +106,7 @@ void
FastAccessFeedView::forceCommit(SerialNum serialNum,
OnForceCommitDoneType onCommitDone)
{
- _attributeWriter->commit(serialNum, onCommitDone);
+ _attributeWriter->forceCommit(serialNum, onCommitDone);
onCommitDone->registerCommittedDocIdLimit(_metaStore.getCommittedDocIdLimit(), &_docIdLimit);
Parent::forceCommit(serialNum, onCommitDone);
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp
index afe19a4ecc3..c1d139c7486 100644
--- a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp
@@ -49,7 +49,7 @@ ReconfigParams::shouldIndexManagerChange() const
bool
ReconfigParams::shouldAttributeManagerChange() const
{
- return _res.attributesChanged || _res.importedFieldsChanged;
+ return _res.attributesChanged || _res.importedFieldsChanged || _res.visibilityDelayChanged;
}
bool
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
index 571e64e104c..03ada12c4f2 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
@@ -210,7 +210,8 @@ SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig,
IAttributeWriter::SP attrWriter = _feedView.get()->getAttributeWriter();
if (params.shouldAttributeManagerChange()) {
IAttributeManager::SP newAttrMgr = attrMgr->create(attrSpec);
- newAttrMgr->setImportedAttributes(resolver.resolve(*newAttrMgr, *attrMgr));
+ newAttrMgr->setImportedAttributes(resolver.resolve(*newAttrMgr, *attrMgr,
+ newConfig.getMaintenanceConfigSP()->getVisibilityDelay()));
IAttributeManager::SP oldAttrMgr = attrMgr;
attrMgr = newAttrMgr;
shouldMatchViewChange = true;
diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt
index d079ef167b8..787ca6ed008 100644
--- a/searchlib/CMakeLists.txt
+++ b/searchlib/CMakeLists.txt
@@ -73,6 +73,7 @@ vespa_define_module(
src/tests/attribute/attributemanager
src/tests/attribute/benchmark
src/tests/attribute/bitvector
+ src/tests/attribute/bitvector_search_cache
src/tests/attribute/changevector
src/tests/attribute/compaction
src/tests/attribute/comparator
diff --git a/searchlib/src/tests/attribute/bitvector_search_cache/CMakeLists.txt b/searchlib/src/tests/attribute/bitvector_search_cache/CMakeLists.txt
new file mode 100644
index 00000000000..2442474bcea
--- /dev/null
+++ b/searchlib/src/tests/attribute/bitvector_search_cache/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_bitvector_search_cache_test_app TEST
+ SOURCES
+ bitvector_search_cache_test.cpp
+ DEPENDS
+ searchlib
+)
+vespa_add_test(NAME searchlib_bitvector_search_cache_test_app COMMAND searchlib_bitvector_search_cache_test_app)
diff --git a/searchlib/src/tests/attribute/bitvector_search_cache/bitvector_search_cache_test.cpp b/searchlib/src/tests/attribute/bitvector_search_cache/bitvector_search_cache_test.cpp
new file mode 100644
index 00000000000..1f738a209df
--- /dev/null
+++ b/searchlib/src/tests/attribute/bitvector_search_cache/bitvector_search_cache_test.cpp
@@ -0,0 +1,62 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchlib/attribute/bitvector_search_cache.h>
+#include <vespa/searchlib/common/bitvector.h>
+
+using namespace search;
+using namespace search::attribute;
+
+using BitVectorSP = BitVectorSearchCache::BitVectorSP;
+using Entry = BitVectorSearchCache::Entry;
+
+Entry::SP
+makeEntry()
+{
+ return std::make_shared<Entry>(BitVector::create(5), 10);
+}
+
+struct Fixture {
+ BitVectorSearchCache cache;
+ Entry::SP entry1;
+ Entry::SP entry2;
+ Fixture()
+ : cache(),
+ entry1(makeEntry()),
+ entry2(makeEntry())
+ {}
+};
+
+TEST_F("require that bit vectors can be inserted and retrieved", Fixture)
+{
+ EXPECT_EQUAL(0u, f.cache.size());
+ f.cache.insert("foo", f.entry1);
+ f.cache.insert("bar", f.entry2);
+ EXPECT_EQUAL(2u, f.cache.size());
+
+ EXPECT_EQUAL(f.entry1, f.cache.find("foo"));
+ EXPECT_EQUAL(f.entry2, f.cache.find("bar"));
+ EXPECT_TRUE(f.cache.find("baz").get() == nullptr);
+}
+
+TEST_F("require that insert() doesn't replace existing bit vector", Fixture)
+{
+ f.cache.insert("foo", f.entry1);
+ f.cache.insert("foo", f.entry2);
+ EXPECT_EQUAL(1u, f.cache.size());
+ EXPECT_EQUAL(f.entry1, f.cache.find("foo"));
+}
+
+TEST_F("require that cache can be cleared", Fixture)
+{
+ f.cache.insert("foo", f.entry1);
+ f.cache.insert("bar", f.entry2);
+ EXPECT_EQUAL(2u, f.cache.size());
+ f.cache.clear();
+
+ EXPECT_EQUAL(0u, f.cache.size());
+ EXPECT_TRUE(f.cache.find("foo").get() == nullptr);
+ EXPECT_TRUE(f.cache.find("bar").get() == nullptr);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp
index c9e0757ee16..f6ab6bb50bf 100644
--- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp
+++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp
@@ -1,21 +1,28 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/searchlib/test/imported_attribute_fixture.h>
+#include <vespa/searchcommon/attribute/search_context_params.h>
#include <vespa/searchlib/attribute/imported_search_context.h>
#include <vespa/searchlib/fef/termfieldmatchdata.h>
-#include <vespa/searchcommon/attribute/search_context_params.h>
+#include <vespa/searchlib/queryeval/simpleresult.h>
+#include <vespa/searchlib/test/imported_attribute_fixture.h>
+#include <vespa/vespalib/test/insertion_operators.h>
namespace search::attribute {
using fef::TermFieldMatchData;
+using queryeval::SearchIterator;
+using queryeval::SimpleResult;
using vespalib::Trinary;
struct Fixture : ImportedAttributeFixture {
+
+ Fixture(bool useSearchCache = false) : ImportedAttributeFixture(useSearchCache) {}
+
std::unique_ptr<ImportedSearchContext> create_context(std::unique_ptr<QueryTermSimple> term) {
return std::make_unique<ImportedSearchContext>(std::move(term), SearchContextParams(), *imported_attr);
}
- std::unique_ptr<queryeval::SearchIterator> create_iterator(
+ std::unique_ptr<SearchIterator> create_iterator(
ImportedSearchContext& ctx,
TermFieldMatchData& match,
bool strict) {
@@ -25,17 +32,21 @@ struct Fixture : ImportedAttributeFixture {
return iter;
}
- std::unique_ptr<queryeval::SearchIterator> create_non_strict_iterator(
+ std::unique_ptr<SearchIterator> create_non_strict_iterator(
ImportedSearchContext& ctx,
TermFieldMatchData& match) {
return create_iterator(ctx, match, false);
}
- std::unique_ptr<queryeval::SearchIterator> create_strict_iterator(
+ std::unique_ptr<SearchIterator> create_strict_iterator(
ImportedSearchContext& ctx,
TermFieldMatchData& match) {
return create_iterator(ctx, match, true);
}
+
+ void assertSearch(const std::vector<uint32_t> &expDocIds, SearchIterator &iter) {
+ EXPECT_EQUAL(SimpleResult(expDocIds), SimpleResult().searchStrict(iter, imported_attr->getNumDocs()));
+ }
};
template <typename Iterator>
@@ -339,6 +350,63 @@ TEST_F("queryTerm() returns term context was created with", WsetValueFixture) {
EXPECT_EQUAL(std::string("helloworld"), std::string(ctx->queryTerm().getTerm()));
}
+struct SearchCacheFixture : Fixture {
+ SearchCacheFixture() : Fixture(true) {
+ reset_with_single_value_reference_mappings<IntegerAttribute, int32_t>(
+ BasicType::INT32,
+ {{DocId(3), dummy_gid(5), DocId(5), 5678},
+ {DocId(4), dummy_gid(6), DocId(6), 1234},
+ {DocId(5), dummy_gid(8), DocId(8), 5678},
+ {DocId(7), dummy_gid(9), DocId(9), 4321}},
+ FastSearchConfig::ExplicitlyEnabled,
+ FilterConfig::ExplicitlyEnabled);
+ }
+};
+
+BitVectorSearchCache::Entry::SP
+makeSearchCacheEntry(const std::vector<uint32_t> docIds, uint32_t docIdLimit)
+{
+ std::shared_ptr<BitVector> bitVector = BitVector::create(docIdLimit);
+ for (uint32_t docId : docIds) {
+ bitVector->setBit(docId);
+ }
+ return std::make_shared<BitVectorSearchCache::Entry>(bitVector, docIdLimit);
+}
+
+TEST_F("Bit vector from search cache is used if found", SearchCacheFixture)
+{
+ f.imported_attr->getSearchCache()->insert("5678",
+ makeSearchCacheEntry({2, 6}, f.imported_attr->getNumDocs()));
+ auto ctx = f.create_context(word_term("5678"));
+ ctx->fetchPostings(true);
+ TermFieldMatchData match;
+ auto iter = f.create_strict_iterator(*ctx, match);
+ TEST_DO(f.assertSearch({2, 6}, *iter)); // Note: would be {3, 5} if cache was not used
+}
+
+void
+assertBitVector(const std::vector<uint32_t> &expDocIds, const BitVector &bitVector)
+{
+ std::vector<uint32_t> actDocsIds;
+ bitVector.foreach_truebit([&](uint32_t docId){ actDocsIds.push_back(docId); });
+ EXPECT_EQUAL(expDocIds, actDocsIds);
+}
+
+TEST_F("Entry is inserted into search cache if bit vector posting list is used", SearchCacheFixture)
+{
+ EXPECT_EQUAL(0u, f.imported_attr->getSearchCache()->size());
+ auto ctx = f.create_context(word_term("5678"));
+ ctx->fetchPostings(true);
+ TermFieldMatchData match;
+ auto iter = f.create_strict_iterator(*ctx, match);
+ TEST_DO(f.assertSearch({3, 5}, *iter));
+
+ EXPECT_EQUAL(1u, f.imported_attr->getSearchCache()->size());
+ auto cacheEntry = f.imported_attr->getSearchCache()->find("5678");
+ EXPECT_EQUAL(cacheEntry->docIdLimit, f.imported_attr->getNumDocs());
+ TEST_DO(assertBitVector({3, 5}, *cacheEntry->bitVector));
+}
+
}
TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt
index aa0b63894c0..e99092f0206 100644
--- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt
@@ -21,6 +21,7 @@ vespa_add_library(searchlib_attribute OBJECT
attributesaver.cpp
attributevector.cpp
attrvector.cpp
+ bitvector_search_cache.cpp
changevector.cpp
configconverter.cpp
createarrayfastsearch.cpp
diff --git a/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.cpp b/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.cpp
new file mode 100644
index 00000000000..cf49856234a
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.cpp
@@ -0,0 +1,53 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "bitvector_search_cache.h"
+#include <vespa/searchlib/common/bitvector.h>
+#include <vespa/vespalib/stllike/hash_map.hpp>
+
+namespace search::attribute {
+
+using BitVectorSP = BitVectorSearchCache::BitVectorSP;
+
+BitVectorSearchCache::BitVectorSearchCache()
+ : _mutex(),
+ _cache()
+{
+}
+
+BitVectorSearchCache::~BitVectorSearchCache()
+{
+}
+
+void
+BitVectorSearchCache::insert(const vespalib::string &term, Entry::SP entry)
+{
+ LockGuard guard(_mutex);
+ _cache.insert(std::make_pair(term, std::move(entry)));
+}
+
+BitVectorSearchCache::Entry::SP
+BitVectorSearchCache::find(const vespalib::string &term) const
+{
+ LockGuard guard(_mutex);
+ auto itr = _cache.find(term);
+ if (itr != _cache.end()) {
+ return itr->second;
+ }
+ return Entry::SP();
+}
+
+size_t
+BitVectorSearchCache::size() const
+{
+ LockGuard guard(_mutex);
+ return _cache.size();
+}
+
+void
+BitVectorSearchCache::clear()
+{
+ LockGuard guard(_mutex);
+ _cache.clear();
+}
+
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.h b/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.h
new file mode 100644
index 00000000000..73fb95e1752
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.h
@@ -0,0 +1,50 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/hash_map.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <memory>
+#include <mutex>
+
+namespace search {
+
+class BitVector;
+
+namespace attribute {
+
+/**
+ * Class that caches posting lists (as bit vectors) for a set of search terms.
+ *
+ * Lifetime of cached bit vectors is controlled by calling clear() at regular intervals.
+ */
+class BitVectorSearchCache {
+public:
+ using BitVectorSP = std::shared_ptr<BitVector>;
+
+ struct Entry {
+ using SP = std::shared_ptr<Entry>;
+ BitVectorSP bitVector;
+ uint32_t docIdLimit;
+ Entry(BitVectorSP bitVector_, uint32_t docIdLimit_)
+ : bitVector(std::move(bitVector_)), docIdLimit(docIdLimit_) {}
+ };
+
+private:
+ using LockGuard = std::lock_guard<std::mutex>;
+ using Cache = vespalib::hash_map<vespalib::string, Entry::SP>;
+
+ mutable std::mutex _mutex;
+ Cache _cache;
+
+public:
+ BitVectorSearchCache();
+ ~BitVectorSearchCache();
+ void insert(const vespalib::string &term, Entry::SP entry);
+ Entry::SP find(const vespalib::string &term) const;
+ size_t size() const;
+ void clear();
+};
+
+}
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp
index 036f482b2e8..9efac9ce541 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp
@@ -3,7 +3,7 @@
#include "imported_attribute_vector.h"
#include "imported_attribute_vector_read_guard.h"
#include "imported_search_context.h"
-#include "attributeguard.h"
+#include "bitvector_search_cache.h"
#include <vespa/searchlib/query/queryterm.h>
#include <vespa/vespalib/util/exceptions.h>
@@ -13,10 +13,24 @@ namespace attribute {
ImportedAttributeVector::ImportedAttributeVector(
vespalib::stringref name,
std::shared_ptr<ReferenceAttribute> reference_attribute,
- std::shared_ptr<AttributeVector> target_attribute)
+ std::shared_ptr<AttributeVector> target_attribute,
+ bool use_search_cache)
: _name(name),
_reference_attribute(std::move(reference_attribute)),
- _target_attribute(std::move(target_attribute))
+ _target_attribute(std::move(target_attribute)),
+ _search_cache(use_search_cache ? std::make_shared<BitVectorSearchCache>() :
+ std::shared_ptr<BitVectorSearchCache>())
+{
+}
+
+ImportedAttributeVector::ImportedAttributeVector(vespalib::stringref name,
+ std::shared_ptr<ReferenceAttribute> reference_attribute,
+ std::shared_ptr<AttributeVector> target_attribute,
+ std::shared_ptr<BitVectorSearchCache> search_cache)
+ : _name(name),
+ _reference_attribute(std::move(reference_attribute)),
+ _target_attribute(std::move(target_attribute)),
+ _search_cache(std::move(search_cache))
{
}
@@ -27,7 +41,7 @@ std::unique_ptr<ImportedAttributeVector>
ImportedAttributeVector::makeReadGuard(bool stableEnumGuard) const
{
return std::make_unique<ImportedAttributeVectorReadGuard>
- (getName(), getReferenceAttribute(), getTargetAttribute(), stableEnumGuard);
+ (getName(), getReferenceAttribute(), getTargetAttribute(), getSearchCache(), stableEnumGuard);
}
const vespalib::string& search::attribute::ImportedAttributeVector::getName() const {
@@ -131,6 +145,12 @@ bool ImportedAttributeVector::hasEnum() const {
return _target_attribute->hasEnum();
}
+void ImportedAttributeVector::clearSearchCache() {
+ if (_search_cache) {
+ _search_cache->clear();
+ }
+}
+
long ImportedAttributeVector::onSerializeForAscendingSort(DocId doc,
void *serTo,
long available,
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h
index bddd0fa9093..c94eda3d4db 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h
@@ -14,6 +14,8 @@ class AttributeEnumGuard;
namespace attribute {
+class BitVectorSearchCache;
+
/**
* Attribute vector which does not store values of its own, but rather serves as a
* convenient indirection wrapper towards a target vector, usually in another
@@ -29,7 +31,12 @@ public:
using SP = std::shared_ptr<ImportedAttributeVector>;
ImportedAttributeVector(vespalib::stringref name,
std::shared_ptr<ReferenceAttribute> reference_attribute,
- std::shared_ptr<AttributeVector> target_attribute);
+ std::shared_ptr<AttributeVector> target_attribute,
+ bool use_search_cache);
+ ImportedAttributeVector(vespalib::stringref name,
+ std::shared_ptr<ReferenceAttribute> reference_attribute,
+ std::shared_ptr<AttributeVector> target_attribute,
+ std::shared_ptr<BitVectorSearchCache> search_cache);
~ImportedAttributeVector();
const vespalib::string & getName() const override;
@@ -65,6 +72,10 @@ public:
const std::shared_ptr<AttributeVector>& getTargetAttribute() const noexcept {
return _target_attribute;
}
+ const std::shared_ptr<BitVectorSearchCache> &getSearchCache() const {
+ return _search_cache;
+ }
+ void clearSearchCache();
/*
* Create an imported attribute with a snapshot of lid to lid mapping.
@@ -77,9 +88,10 @@ protected:
const common::BlobConverter * bc) const override;
- vespalib::string _name;
- std::shared_ptr<ReferenceAttribute> _reference_attribute;
- std::shared_ptr<AttributeVector> _target_attribute;
+ vespalib::string _name;
+ std::shared_ptr<ReferenceAttribute> _reference_attribute;
+ std::shared_ptr<AttributeVector> _target_attribute;
+ std::shared_ptr<BitVectorSearchCache> _search_cache;
};
} // attribute
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
index 563834e6cdb..9586e2be79b 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
@@ -9,8 +9,9 @@ ImportedAttributeVectorReadGuard::ImportedAttributeVectorReadGuard(
vespalib::stringref name,
std::shared_ptr<ReferenceAttribute> reference_attribute,
std::shared_ptr<AttributeVector> target_attribute,
+ std::shared_ptr<BitVectorSearchCache> search_cache,
bool stableEnumGuard)
- : ImportedAttributeVector(name, std::move(reference_attribute), std::move(target_attribute)),
+ : ImportedAttributeVector(name, std::move(reference_attribute), std::move(target_attribute), std::move(search_cache)),
_referencedLids(),
_referencedLidLimit(0u),
_reference_attribute_guard(_reference_attribute),
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
index f4db2b538d5..77554fdb849 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
@@ -7,6 +7,8 @@
namespace search::attribute {
+class BitVectorSearchCache;
+
/*
* Short lived attribute vector that does not store values on its own.
*
@@ -31,6 +33,7 @@ public:
ImportedAttributeVectorReadGuard(vespalib::stringref name,
std::shared_ptr<ReferenceAttribute> reference_attribute,
std::shared_ptr<AttributeVector> target_attribute,
+ std::shared_ptr<BitVectorSearchCache> search_cache,
bool stableEnumGuard);
~ImportedAttributeVectorReadGuard();
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp
index 5a040941e67..26e51f77ccf 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp
@@ -1,5 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "bitvector_search_cache.h"
#include "imported_search_context.h"
#include "imported_attribute_vector.h"
#include <vespa/searchlib/common/bitvectoriterator.h>
@@ -24,6 +25,10 @@ ImportedSearchContext::ImportedSearchContext(
const SearchContextParams& params,
const ImportedAttributeVector& imported_attribute)
: _imported_attribute(imported_attribute),
+ _queryTerm(term->getTerm()),
+ _useSearchCache(_imported_attribute.getSearchCache().get() != nullptr),
+ _searchCacheLookup((_useSearchCache ? _imported_attribute.getSearchCache()->find(_queryTerm) :
+ std::shared_ptr<BitVectorSearchCache::Entry>())),
_reference_attribute(*_imported_attribute.getReferenceAttribute()),
_target_attribute(*_imported_attribute.getTargetAttribute()),
_target_search_context(_target_attribute.getSearch(std::move(term), params)),
@@ -43,6 +48,9 @@ unsigned int ImportedSearchContext::approximateHits() const {
std::unique_ptr<queryeval::SearchIterator>
ImportedSearchContext::createIterator(fef::TermFieldMatchData* matchData, bool strict) {
+ if (_searchCacheLookup) {
+ return BitVectorIterator::create(_searchCacheLookup->bitVector.get(), _searchCacheLookup->docIdLimit, *matchData, strict);
+ }
if (_merger.hasArray()) {
if (_merger.emptyArray()) {
return SearchIterator::UP(new EmptySearch());
@@ -204,12 +212,24 @@ void ImportedSearchContext::makeMergedPostings(bool isFilter)
_merger.merge();
}
+void
+ImportedSearchContext::considerAddSearchCacheEntry()
+{
+ if (_useSearchCache && _merger.hasBitVector()) {
+ auto cacheEntry = std::make_shared<BitVectorSearchCache::Entry>(_merger.getBitVectorSP(), _merger.getDocIdLimit());
+ _imported_attribute.getSearchCache()->insert(_queryTerm, std::move(cacheEntry));
+ }
+}
+
void ImportedSearchContext::fetchPostings(bool strict) {
assert(!_fetchPostingsDone);
_fetchPostingsDone = true;
- _target_search_context->fetchPostings(strict);
- if (strict || _target_attribute.getConfig().fastSearch()) {
- makeMergedPostings(_target_attribute.getConfig().getIsFilter());
+ if (!_searchCacheLookup) {
+ _target_search_context->fetchPostings(strict);
+ if (strict || _target_attribute.getConfig().fastSearch()) {
+ makeMergedPostings(_target_attribute.getConfig().getIsFilter());
+ considerAddSearchCacheEntry();
+ }
}
}
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h
index c4a6101ac0a..f3f44332a2e 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h
+++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h
@@ -3,6 +3,7 @@
#pragma once
#include "attributevector.h"
+#include "bitvector_search_cache.h"
#include <vespa/searchcommon/attribute/i_search_context.h>
#include <vespa/searchlib/attribute/posting_list_merger.h>
#include <vespa/vespalib/util/arrayref.h>
@@ -26,6 +27,9 @@ class SearchContextParams;
class ImportedSearchContext : public ISearchContext {
using ReferencedLids = vespalib::ConstArrayRef<uint32_t>;
const ImportedAttributeVector& _imported_attribute;
+ vespalib::string _queryTerm;
+ bool _useSearchCache;
+ BitVectorSearchCache::Entry::SP _searchCacheLookup;
const ReferenceAttribute& _reference_attribute;
const AttributeVector& _target_attribute;
std::unique_ptr<AttributeVector::SearchContext> _target_search_context;
@@ -40,6 +44,7 @@ class ImportedSearchContext : public ISearchContext {
}
void makeMergedPostings(bool isFilter);
+ void considerAddSearchCacheEntry();
public:
ImportedSearchContext(std::unique_ptr<QueryTermSimple> term,
const SearchContextParams& params,
diff --git a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h
index 8220b529aac..8568661dfdd 100644
--- a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h
+++ b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h
@@ -21,7 +21,7 @@ class PostingListMerger
PostingVector _array;
StartVector _startPos;
- std::unique_ptr<BitVector> _bitVector;
+ std::shared_ptr<BitVector> _bitVector;
uint32_t _docIdLimit;
bool _arrayValid;
@@ -39,6 +39,7 @@ public:
bool emptyArray() const { return _array.empty(); }
vespalib::ConstArrayRef<Posting> getArray() const { return _array; }
const BitVector *getBitVector() const { return _bitVector.get(); }
+ const std::shared_ptr<BitVector> &getBitVectorSP() const { return _bitVector; }
uint32_t getDocIdLimit() const { return _docIdLimit; }
template <typename PostingListType>
diff --git a/searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp b/searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp
index 5195f03b8db..de2582dda6b 100644
--- a/searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp
@@ -20,7 +20,7 @@ SimpleResult::clear()
tmp.swap(_hits);
}
-void
+SimpleResult &
SimpleResult::search(SearchIterator &sb)
{
clear();
@@ -30,9 +30,10 @@ SimpleResult::search(SearchIterator &sb)
sb.unpack(sb.getDocId());
_hits.push_back(sb.getDocId());
}
+ return *this;
}
-void
+SimpleResult &
SimpleResult::searchStrict(SearchIterator &sb, uint32_t docIdLimit)
{
clear();
@@ -42,9 +43,10 @@ SimpleResult::searchStrict(SearchIterator &sb, uint32_t docIdLimit)
sb.unpack(sb.getDocId());
_hits.push_back(sb.getDocId());
}
+ return *this;
}
-void
+SimpleResult &
SimpleResult::search(SearchIterator &sb, uint32_t docIdLimit)
{
clear();
@@ -57,6 +59,7 @@ SimpleResult::search(SearchIterator &sb, uint32_t docIdLimit)
_hits.push_back(docId);
}
}
+ return *this;
}
std::ostream &
diff --git a/searchlib/src/vespa/searchlib/queryeval/simpleresult.h b/searchlib/src/vespa/searchlib/queryeval/simpleresult.h
index 30c88bb10c7..4a1fcb25e7a 100644
--- a/searchlib/src/vespa/searchlib/queryeval/simpleresult.h
+++ b/searchlib/src/vespa/searchlib/queryeval/simpleresult.h
@@ -23,6 +23,11 @@ public:
SimpleResult() : _hits() {}
/**
+ * Create a result with the given hits.
+ */
+ SimpleResult(const std::vector<uint32_t> &hits) : _hits(hits) {}
+
+ /**
* Obtain the number of hits
*
* @return number of hits
@@ -58,8 +63,8 @@ public:
*
* @param sb search object
**/
- void search(SearchIterator &sb);
- void searchStrict(SearchIterator &sb, uint32_t docIdLimit);
+ SimpleResult &search(SearchIterator &sb);
+ SimpleResult &searchStrict(SearchIterator &sb, uint32_t docIdLimit);
/**
* Fill this result with all the hits returned by the given search
@@ -69,7 +74,7 @@ public:
* @param sb search object
* @param docIdLimit the end of the docId range for this search iterator
**/
- void search(SearchIterator &sb, uint32_t docIdLimit);
+ SimpleResult &search(SearchIterator &sb, uint32_t docIdLimit);
/**
* Test of we contain the same hits as rhs.
diff --git a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h
index 85a2b140039..3620a1df725 100644
--- a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h
+++ b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h
@@ -44,34 +44,47 @@ enum class FastSearchConfig {
Default
};
+enum class FilterConfig {
+ ExplicitlyEnabled,
+ Default
+};
+
template<typename AttrVecType>
std::shared_ptr<AttrVecType> create_typed_attribute(BasicType basic_type,
CollectionType collection_type,
FastSearchConfig fast_search = FastSearchConfig::Default,
+ FilterConfig filter = FilterConfig::Default,
vespalib::stringref name = "parent") {
Config cfg(basic_type, collection_type);
if (fast_search == FastSearchConfig::ExplicitlyEnabled) {
cfg.setFastSearch(true);
}
+ if (filter == FilterConfig::ExplicitlyEnabled) {
+ cfg.setIsFilter(true);
+ }
return std::dynamic_pointer_cast<AttrVecType>(
AttributeFactory::createAttribute(name, std::move(cfg)));
}
template<typename AttrVecType>
-std::shared_ptr<AttrVecType> create_single_attribute(BasicType type, vespalib::stringref name = "parent") {
- return create_typed_attribute<AttrVecType>(type, CollectionType::SINGLE, FastSearchConfig::Default, name);
+std::shared_ptr<AttrVecType> create_single_attribute(BasicType type,
+ FastSearchConfig fast_search = FastSearchConfig::Default,
+ FilterConfig filter = FilterConfig::Default,
+ vespalib::stringref name = "parent") {
+ return create_typed_attribute<AttrVecType>(type, CollectionType::SINGLE, fast_search, filter, name);
}
template<typename AttrVecType>
std::shared_ptr<AttrVecType> create_array_attribute(BasicType type, vespalib::stringref name = "parent") {
- return create_typed_attribute<AttrVecType>(type, CollectionType::ARRAY, FastSearchConfig::Default, name);
+ return create_typed_attribute<AttrVecType>(type, CollectionType::ARRAY,
+ FastSearchConfig::Default, FilterConfig::Default, name);
}
template<typename AttrVecType>
std::shared_ptr<AttrVecType> create_wset_attribute(BasicType type,
FastSearchConfig fast_search = FastSearchConfig::Default,
vespalib::stringref name = "parent") {
- return create_typed_attribute<AttrVecType>(type, CollectionType::WSET, fast_search, name);
+ return create_typed_attribute<AttrVecType>(type, CollectionType::WSET, fast_search, FilterConfig::Default, name);
}
template<typename VectorType>
@@ -89,12 +102,13 @@ std::unique_ptr<QueryTermSimple> word_term(vespalib::stringref term) {
}
struct ImportedAttributeFixture {
+ bool use_search_cache;
std::shared_ptr<AttributeVector> target_attr;
std::shared_ptr<ReferenceAttribute> reference_attr;
std::shared_ptr<ImportedAttributeVector> imported_attr;
std::shared_ptr<MockGidToLidMapperFactory> mapper_factory;
- ImportedAttributeFixture();
+ ImportedAttributeFixture(bool use_search_cache_ = false);
virtual ~ImportedAttributeFixture();
@@ -116,7 +130,7 @@ struct ImportedAttributeFixture {
std::shared_ptr<ImportedAttributeVector>
create_attribute_vector_from_members(vespalib::stringref name = default_imported_attr_name()) {
- return std::make_shared<ImportedAttributeVector>(name, reference_attr, target_attr);
+ return std::make_shared<ImportedAttributeVector>(name, reference_attr, target_attr, use_search_cache);
}
template<typename AttrVecType>
@@ -169,8 +183,10 @@ struct ImportedAttributeFixture {
template<typename AttrVecType, typename ValueType>
void reset_with_single_value_reference_mappings(
BasicType type,
- const std::vector<LidToLidMapping<ValueType>> &mappings) {
- reset_with_new_target_attr(create_single_attribute<AttrVecType>(type));
+ const std::vector<LidToLidMapping<ValueType>> &mappings,
+ FastSearchConfig fast_search = FastSearchConfig::Default,
+ FilterConfig filter = FilterConfig::Default) {
+ reset_with_new_target_attr(create_single_attribute<AttrVecType>(type, fast_search, filter));
// Fun experiment: rename `auto& mapping` to `auto& m` and watch GCC howl about
// shadowing a variable... that exists in the set_up_and_map function!
set_up_and_map<AttrVecType>(mappings, [this](auto &target_vec, auto &mapping) {
@@ -205,8 +221,9 @@ struct ImportedAttributeFixture {
}
};
-ImportedAttributeFixture::ImportedAttributeFixture()
- : target_attr(create_single_attribute<IntegerAttribute>(BasicType::INT32)),
+ImportedAttributeFixture::ImportedAttributeFixture(bool use_search_cache_)
+ : use_search_cache(use_search_cache_),
+ target_attr(create_single_attribute<IntegerAttribute>(BasicType::INT32)),
reference_attr(create_reference_attribute()),
imported_attr(create_attribute_vector_from_members()),
mapper_factory(std::make_shared<MockGidToLidMapperFactory>()) {
@@ -240,8 +257,10 @@ template<typename AttrVecType, typename ValueType>
void reset_with_single_value_reference_mappings(
ImportedAttributeFixture &f,
BasicType type,
- const std::vector<ImportedAttributeFixture::LidToLidMapping<ValueType>> &mappings) {
- f.reset_with_single_value_reference_mappings<AttrVecType, ValueType>(type, mappings);
+ const std::vector<ImportedAttributeFixture::LidToLidMapping<ValueType>> &mappings,
+ FastSearchConfig fast_search = FastSearchConfig::Default,
+ FilterConfig filter = FilterConfig::Default) {
+ f.reset_with_single_value_reference_mappings<AttrVecType, ValueType>(type, mappings, fast_search, filter);
}
template<typename AttrVecType, typename ValueType>