Instantiate bit vector search cache in imported attribute vectors if visibility delay > 0.

13 files changed, 62 insertions, 21 deletions

diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
index 834d7145ed0..5b64fa80d39 100644
--- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
@@ -133,7 +133,7 @@ struct ImportedAttributesRepoBuilder {
     void add(const vespalib::string &name) {
         auto refAttr = std::make_shared<ReferenceAttribute>(name + "_ref", AVConfig(BasicType::REFERENCE));
         auto targetAttr = search::AttributeFactory::createAttribute(name + "_target", INT32_SINGLE);
-        auto importedAttr = std::make_shared<ImportedAttributeVector>(name, refAttr, targetAttr);
+        auto importedAttr = std::make_shared<ImportedAttributeVector>(name, refAttr, targetAttr, false);
         _repo->add(name, importedAttr);
     }
     ImportedAttributesRepo::UP build() {
diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp
index 3b2c87b3840..d95d79fbd11 100644
--- a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp
+++ b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp
@@ -70,7 +70,8 @@ struct Fixture {
     Fixture &addAttribute(const vespalib::string &name) {
         auto attr = std::make_shared<ImportedAttributeVector>(name,
                                                               createReferenceAttribute(name + "_ref"),
-                                                              createTargetAttribute(name + "_target"));
+                                                              createTargetAttribute(name + "_target"),
+                                                              false);
         repo.add(name, attr);
         return *this;
     }
diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp
index 99b1d11e862..c41b88be2b4 100644
--- a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp
+++ b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp
@@ -21,7 +21,8 @@ createAttr(const vespalib::string &name)
 {
     return std::make_shared<ImportedAttributeVector>(name,
                                                      ReferenceAttribute::SP(),
-                                                     AttributeVector::SP());
+                                                     AttributeVector::SP(),
+                                                     false);
 }
 
 struct Fixture {
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
index d76117df4c2..45bda8c69cc 100644
--- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
@@ -141,7 +141,9 @@ struct EmptyConstantValueFactory : public vespalib::eval::ConstantValueFactory {
 };
 
 struct MyDocumentDBReferenceResolver : public IDocumentDBReferenceResolver {
-    std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &, const search::IAttributeManager &) override {
+    std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &,
+                                                    const search::IAttributeManager &,
+                                                    fastos::TimeStamp) override {
         return std::make_unique<ImportedAttributesRepo>();
     }
     void teardown(const search::IAttributeManager &) override { }
@@ -657,12 +659,18 @@ TEST_F("require that we can reconfigure matchers", Fixture)
     }
 }
 
-TEST("require that attribute manager should change when imported fields has changed")
+TEST("require that attribute manager (imported attributes) should change when imported fields has changed")
 {
     ReconfigParams params(CCR().setImportedFieldsChanged(true));
     EXPECT_TRUE(params.shouldAttributeManagerChange());
 }
 
+TEST("require that attribute manager (imported attributes) should change when visibility delay has changed")
+{
+    ReconfigParams params(CCR().setVisibilityDelayChanged(true));
+    EXPECT_TRUE(params.shouldAttributeManagerChange());
+}
+
 void
 assertMaintenanceControllerShouldNotChange(DocumentDBConfig::ComparisonResult result)
 {
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index c3928e193e8..8cd57b1b0cb 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -108,7 +108,9 @@ struct MyMetricsWireService : public DummyWireService
 };
 
 struct MyDocumentDBReferenceResolver : public IDocumentDBReferenceResolver {
-    std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &, const search::IAttributeManager &) override {
+    std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &,
+													const search::IAttributeManager &,
+                                                    fastos::TimeStamp) override {
         return std::make_unique<ImportedAttributesRepo>();
 	}
     void teardown(const search::IAttributeManager &) override { }
diff --git a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp
index 65e15a3efaa..a6576d11279 100644
--- a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp
+++ b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp
@@ -224,9 +224,12 @@ struct Fixture {
         oldAttrMgr.addReferenceAttribute("parent2_ref");
         oldAttrMgr.addReferenceAttribute("parent3_ref");
     }
-    ImportedAttributesRepo::UP resolve() {
+    ImportedAttributesRepo::UP resolve(fastos::TimeStamp visibilityDelay) {
         DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter);
-        return resolver.resolve(attrMgr, oldAttrMgr);
+        return resolver.resolve(attrMgr, oldAttrMgr, visibilityDelay);
+    }
+    ImportedAttributesRepo::UP resolve() {
+        return resolve(fastos::TimeStamp(0));
     }
     void teardown() {
         DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter);
@@ -238,11 +241,13 @@ struct Fixture {
     void assertImportedAttribute(const vespalib::string &name,
                                  const vespalib::string &referenceField,
                                  const vespalib::string &targetField,
+                                 bool useSearchCache,
                                  ImportedAttributeVector::SP attr) {
         ASSERT_TRUE(attr.get());
         EXPECT_EQUAL(name, attr->getName());
         EXPECT_EQUAL(attrMgr.getReferenceAttribute(referenceField), attr->getReferenceAttribute().get());
         EXPECT_EQUAL(parentReference->getAttribute(targetField).get(), attr->getTargetAttribute().get());
+        EXPECT_EQUAL(useSearchCache, attr->getSearchCache().get() != nullptr);
     }
 
     MockGidToLidChangeHandler &getGidToLidChangeHandler(const vespalib::string &referencedDocTypeName) {
@@ -272,12 +277,20 @@ TEST_F("require that reference attributes are connected to gid mapper", Fixture)
     EXPECT_EQUAL(f.factory.get(), f.getMapperFactoryPtr("other_ref"));
 }
 
-TEST_F("require that imported attributes are instantiated", Fixture)
+TEST_F("require that imported attributes are instantiated without search cache as default", Fixture)
 {
     auto repo = f.resolve();
     EXPECT_EQUAL(2u, repo->size());
-    f.assertImportedAttribute("imported_a", "ref", "target_a", repo->get("imported_a"));
-    f.assertImportedAttribute("imported_b", "other_ref", "target_b", repo->get("imported_b"));
+    f.assertImportedAttribute("imported_a", "ref", "target_a", false, repo->get("imported_a"));
+    f.assertImportedAttribute("imported_b", "other_ref", "target_b", false, repo->get("imported_b"));
+}
+
+TEST_F("require that imported attributes are instantiated with search cache if visibility delay > 0", Fixture)
+{
+    auto repo = f.resolve(fastos::TimeStamp::Seconds(1.0));
+    EXPECT_EQUAL(2u, repo->size());
+    f.assertImportedAttribute("imported_a", "ref", "target_a", true, repo->get("imported_a"));
+    f.assertImportedAttribute("imported_b", "other_ref", "target_b", true, repo->get("imported_b"));
 }
 
 TEST_F("require that listeners are added", Fixture)
diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp
index 5147840d5a3..c56aa18c1f7 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp
+++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp
@@ -135,13 +135,15 @@ DocumentDBReferenceResolver::listenToGidToLidChanges(const IAttributeManager &at
 }
 
 ImportedAttributesRepo::UP
-DocumentDBReferenceResolver::createImportedAttributesRepo(const IAttributeManager &attrMgr)
+DocumentDBReferenceResolver::createImportedAttributesRepo(const IAttributeManager &attrMgr,
+                                                          bool useSearchCache)
 {
     auto result = std::make_unique<ImportedAttributesRepo>();
     for (const auto &attr : _importedFieldsCfg.attribute) {
         ReferenceAttribute::SP refAttr = getReferenceAttribute(attr.referencefield, attrMgr);
         AttributeVector::SP targetAttr = getTargetDocumentDB(refAttr->getName())->getAttribute(attr.targetfield);
-        ImportedAttributeVector::SP importedAttr = std::make_shared<ImportedAttributeVector>(attr.name, refAttr, targetAttr);
+        ImportedAttributeVector::SP importedAttr =
+                std::make_shared<ImportedAttributeVector>(attr.name, refAttr, targetAttr, useSearchCache);
         result->add(importedAttr->getName(), importedAttr);
     }
     return result;
@@ -169,12 +171,14 @@ DocumentDBReferenceResolver::~DocumentDBReferenceResolver()
 }
 
 ImportedAttributesRepo::UP
-DocumentDBReferenceResolver::resolve(const IAttributeManager &newAttrMgr, const IAttributeManager &oldAttrMgr)
+DocumentDBReferenceResolver::resolve(const IAttributeManager &newAttrMgr,
+                                     const IAttributeManager &oldAttrMgr,
+                                     fastos::TimeStamp visibilityDelay)
 {
     connectReferenceAttributesToGidMapper(newAttrMgr);
     detectOldListeners(oldAttrMgr);
     listenToGidToLidChanges(newAttrMgr);
-    return createImportedAttributesRepo(newAttrMgr);
+    return createImportedAttributesRepo(newAttrMgr, (visibilityDelay > 0));
 }
 
 void
diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h
index cd58d4b5365..8e0a4178622 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h
+++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h
@@ -36,7 +36,8 @@ private:
     GidToLidChangeRegistrator &getRegistrator(const vespalib::string &docTypeName);
     std::shared_ptr<IDocumentDBReference> getTargetDocumentDB(const vespalib::string &refAttrName) const;
     void connectReferenceAttributesToGidMapper(const search::IAttributeManager &attrMgr);
-    std::unique_ptr<ImportedAttributesRepo> createImportedAttributesRepo(const search::IAttributeManager &attrMgr);
+    std::unique_ptr<ImportedAttributesRepo> createImportedAttributesRepo(const search::IAttributeManager &attrMgr,
+                                                                         bool useSearchCache);
     void detectOldListeners(const search::IAttributeManager &attrMgr);
     void listenToGidToLidChanges(const search::IAttributeManager &attrMgr);
 
@@ -49,7 +50,9 @@ public:
                                 search::ISequencedTaskExecutor &attributeFieldWriter);
     ~DocumentDBReferenceResolver();
 
-    virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr, const search::IAttributeManager &oldAttrMgr) override;
+    virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr,
+                                                            const search::IAttributeManager &oldAttrMgr,
+                                                            fastos::TimeStamp visibilityDelay) override;
     virtual void teardown(const search::IAttributeManager &oldAttrMgr) override;
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h
index c0d1ab6f68c..7e5b26ac34c 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h
+++ b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
+#include <vespa/fastos/timestamp.h>
 #include <memory>
 
 namespace search { class IAttributeManager; }
@@ -14,7 +15,9 @@ class ImportedAttributesRepo;
  */
 struct IDocumentDBReferenceResolver {
     virtual ~IDocumentDBReferenceResolver() {}
-    virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr, const search::IAttributeManager &oldAttrMgr) = 0;
+    virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr,
+                                                            const search::IAttributeManager &oldAttrMgr,
+                                                            fastos::TimeStamp visibilityDelay) = 0;
     virtual void teardown(const search::IAttributeManager &oldAttrMgr) = 0;
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
index d4e10b677a5..86d43517d00 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
@@ -40,7 +40,8 @@ DocumentDBConfig::ComparisonResult::ComparisonResult()
       importedFieldsChanged(false),
       tuneFileDocumentDBChanged(false),
       schemaChanged(false),
-      maintenanceChanged(false)
+      maintenanceChanged(false),
+      visibilityDelayChanged(false)
 { }
 
 DocumentDBConfig::DocumentDBConfig(
@@ -171,6 +172,8 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const
     retval.maintenanceChanged =
         !equals<DocumentDBMaintenanceConfig>(_maintenance.get(),
                 rhs._maintenance.get());
+    retval.visibilityDelayChanged =
+            (_maintenance->getVisibilityDelay() != rhs._maintenance->getVisibilityDelay());
     return retval;
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
index 69e9354b741..773450af620 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
@@ -49,6 +49,7 @@ public:
         bool tuneFileDocumentDBChanged;
         bool schemaChanged;
         bool maintenanceChanged;
+        bool visibilityDelayChanged;
 
         ComparisonResult();
         ComparisonResult &setRankProfilesChanged(bool val) { rankProfilesChanged = val; return *this; }
@@ -64,6 +65,7 @@ public:
         ComparisonResult &setTuneFileDocumentDBChanged(bool val) { tuneFileDocumentDBChanged = val; return *this; }
         ComparisonResult &setSchemaChanged(bool val) { schemaChanged = val; return *this; }
         ComparisonResult &setMaintenanceChanged(bool val) { maintenanceChanged = val; return *this; }
+        ComparisonResult &setVisibilityDelayChanged(bool val) { visibilityDelayChanged = val; return *this; }
     };
 
     using SP = std::shared_ptr<DocumentDBConfig>;
diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp
index afe19a4ecc3..c1d139c7486 100644
--- a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp
@@ -49,7 +49,7 @@ ReconfigParams::shouldIndexManagerChange() const
 bool
 ReconfigParams::shouldAttributeManagerChange() const
 {
-    return _res.attributesChanged || _res.importedFieldsChanged;
+    return _res.attributesChanged || _res.importedFieldsChanged || _res.visibilityDelayChanged;
 }
 
 bool
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
index 571e64e104c..03ada12c4f2 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
@@ -210,7 +210,8 @@ SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig,
     IAttributeWriter::SP attrWriter = _feedView.get()->getAttributeWriter();
     if (params.shouldAttributeManagerChange()) {
         IAttributeManager::SP newAttrMgr = attrMgr->create(attrSpec);
-        newAttrMgr->setImportedAttributes(resolver.resolve(*newAttrMgr, *attrMgr));
+        newAttrMgr->setImportedAttributes(resolver.resolve(*newAttrMgr, *attrMgr,
+                                                           newConfig.getMaintenanceConfigSP()->getVisibilityDelay()));
         IAttributeManager::SP oldAttrMgr = attrMgr;
         attrMgr = newAttrMgr;
         shouldMatchViewChange = true;