Make AttributeWriter::forceCommit() clear the search cache of all imported attribute vectors.

9 files changed, 87 insertions, 25 deletions

diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp
index ad80f99a6aa..1112cf75570 100644
--- a/searchcore/src/tests/proton/attribute/attribute_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp
@@ -2,44 +2,45 @@
 #include <vespa/log/log.h>
 LOG_SETUP("attribute_test");
 
-#include <vespa/fastos/file.h>
-#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/config-attributes.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/predicate/predicate_slime_builder.h>
 #include <vespa/document/update/arithmeticvalueupdate.h>
+#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/eval/tensor/default_tensor.h>
+#include <vespa/eval/tensor/tensor.h>
+#include <vespa/eval/tensor/tensor_factory.h>
+#include <vespa/eval/tensor/types.h>
+#include <vespa/fastos/file.h>
 #include <vespa/searchcommon/attribute/attributecontent.h>
 #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h>
 #include <vespa/searchcore/proton/attribute/attribute_writer.h>
 #include <vespa/searchcore/proton/attribute/attributemanager.h>
 #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h>
+#include <vespa/searchcore/proton/attribute/imported_attributes_repo.h>
+#include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcore/proton/test/attribute_utils.h>
 #include <vespa/searchcorespi/flush/iflushtarget.h>
 #include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/bitvector_search_cache.h>
+#include <vespa/searchlib/attribute/imported_attribute_vector.h>
 #include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/attribute/predicate_attribute.h>
+#include <vespa/searchlib/attribute/singlenumericattribute.hpp>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
 #include <vespa/searchlib/common/idestructorcallback.h>
+#include <vespa/searchlib/common/sequencedtaskexecutorobserver.h>
 #include <vespa/searchlib/index/docbuilder.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/predicate/predicate_hash.h>
+#include <vespa/searchlib/predicate/predicate_index.h>
+#include <vespa/searchlib/tensor/tensor_attribute.h>
+#include <vespa/searchlib/test/directory_handler.h>
 #include <vespa/searchlib/util/filekit.h>
-#include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/test/insertion_operators.h>
-
-#include <vespa/document/predicate/predicate_slime_builder.h>
-#include <vespa/document/update/assignvalueupdate.h>
-#include <vespa/searchlib/attribute/attributevector.hpp>
-#include <vespa/searchlib/attribute/predicate_attribute.h>
-#include <vespa/searchlib/predicate/predicate_index.h>
-#include <vespa/searchlib/attribute/singlenumericattribute.hpp>
-#include <vespa/searchlib/predicate/predicate_hash.h>
-#include <vespa/searchlib/common/foregroundtaskexecutor.h>
-#include <vespa/searchlib/common/sequencedtaskexecutorobserver.h>
-#include <vespa/searchlib/test/directory_handler.h>
-#include <vespa/eval/tensor/tensor.h>
-#include <vespa/eval/tensor/types.h>
-#include <vespa/eval/tensor/default_tensor.h>
-#include <vespa/eval/tensor/tensor_factory.h>
-#include <vespa/searchlib/tensor/tensor_attribute.h>
-#include <vespa/config-attributes.h>
-
+#include <vespa/vespalib/testkit/testapp.h>
 
 namespace vespa { namespace config { namespace search {}}}
 
@@ -50,8 +51,12 @@ using namespace search::index;
 using namespace search;
 using namespace vespa::config::search;
 
+using proton::ImportedAttributesRepo;
 using proton::test::AttributeUtils;
 using search::TuneFileAttributes;
+using search::attribute::BitVectorSearchCache;
+using search::attribute::ImportedAttributeVector;
+using search::attribute::ReferenceAttribute;
 using search::index::DummyFileHeaderContext;
 using search::index::schema::CollectionType;
 using search::predicate::PredicateHash;
@@ -712,6 +717,34 @@ TEST_F("require that attribute writer spreads write over 3 write contexts", Fixt
     TEST_DO(putAttributes(f, {0, 1, 2}));
 }
 
+ImportedAttributeVector::SP
+createImportedAttribute(const vespalib::string &name)
+{
+    auto result = std::make_shared<ImportedAttributeVector>(name,
+                                                            ReferenceAttribute::SP(),
+                                                            AttributeVector::SP(),
+                                                            true);
+    result->getSearchCache()->insert("foo", BitVectorSearchCache::Entry::SP());
+    return result;
+}
+
+ImportedAttributesRepo::UP
+createImportedAttributesRepo()
+{
+    auto result = std::make_unique<ImportedAttributesRepo>();
+    result->add("imported_a", createImportedAttribute("imported_a"));
+    result->add("imported_b", createImportedAttribute("imported_b"));
+    return result;
+}
+
+TEST_F("require that AttributeWriter::forceCommit() clears search cache in imported attribute vectors", Fixture)
+{
+    f._m->setImportedAttributes(createImportedAttributesRepo());
+    f.commit(10);
+    EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_a")->getSearchCache()->size());
+    EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_b")->getSearchCache()->size());
+}
+
 TEST_MAIN()
 {
     vespalib::rmdir(test_dir, true);
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
index 6eb447ac968..4b4e5b955a1 100644
--- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
@@ -384,6 +384,9 @@ struct MyAttributeManager : public proton::IAttributeManager
     virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo>) override {
         abort();
     }
+    virtual const ImportedAttributesRepo *getImportedAttributes() const override {
+        abort();
+    }
 };
 
 struct MockLidSpaceCompactionHandler : public ILidSpaceCompactionHandler
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
index 98ad981917d..906f71fecb4 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
@@ -2,18 +2,20 @@
 
 #include "attribute_writer.h"
 #include "attributemanager.h"
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/searchcore/proton/attribute/imported_attributes_repo.h>
 #include <vespa/searchcore/proton/common/attrupdate.h>
 #include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/imported_attribute_vector.h>
 #include <vespa/searchlib/common/isequencedtaskexecutor.h>
-#include <vespa/document/datatype/documenttype.h>
 
 #include <vespa/log/log.h>
-#include <vespa/document/base/exceptions.h>
-
 LOG_SETUP(".proton.server.attributeadapter");
 
 using namespace document;
 using namespace search;
+using search::attribute::ImportedAttributeVector;
 
 namespace proton {
 
@@ -461,6 +463,13 @@ AttributeWriter::heartBeat(SerialNum serialNum)
 void
 AttributeWriter::forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone)
 {
+    if (_mgr->getImportedAttributes() != nullptr) {
+        std::vector<std::shared_ptr<ImportedAttributeVector>> importedAttrs;
+        _mgr->getImportedAttributes()->getAll(importedAttrs);
+        for (const auto &attr : importedAttrs) {
+            attr->clearSearchCache();
+        }
+    }
     for (const auto &wc : _writeContexts) {
         auto commitTask = std::make_unique<CommitTask>(wc, serialNum, onWriteDone);
         _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(commitTask));
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
index a7de9e2efa4..7148815ca9e 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
@@ -155,7 +155,6 @@ public:
 
     static void padAttribute(search::AttributeVector &v, uint32_t docIdLimit);
 
-    const ImportedAttributesRepo *getImportedAttributes() const { return _importedAttributes.get(); }
 
     // Implements search::IAttributeManager
     virtual search::AttributeGuard::UP getAttribute(const vespalib::string &name) const override;
@@ -200,6 +199,8 @@ public:
     virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override;
 
     virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override;
+
+    virtual const ImportedAttributesRepo *getImportedAttributes() const override { return _importedAttributes.get(); }
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
index 9f78a431430..af64f47fc0a 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
@@ -213,4 +213,10 @@ FilterAttributeManager::setImportedAttributes(std::unique_ptr<ImportedAttributes
     throw vespalib::IllegalArgumentException("Not implemented");
 }
 
+const ImportedAttributesRepo *
+FilterAttributeManager::getImportedAttributes() const
+{
+    throw vespalib::IllegalArgumentException("Not implemented");
+}
+
 }
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
index 958bf78eeb6..81fafa477ee 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
@@ -53,6 +53,7 @@ public:
     virtual void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const override;
     virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override;
     virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override;
+    virtual const ImportedAttributesRepo *getImportedAttributes() const override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
index eb9bde1048f..31bd7c27c85 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
@@ -101,6 +101,8 @@ struct IAttributeManager : public search::IAttributeManager
     virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const = 0;
 
     virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) = 0;
+
+    virtual const ImportedAttributesRepo *getImportedAttributes() const = 0;
 };
 
 } // namespace proton
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp
index 0ec65f60d41..9efac9ce541 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp
@@ -145,6 +145,12 @@ bool ImportedAttributeVector::hasEnum() const {
     return _target_attribute->hasEnum();
 }
 
+void ImportedAttributeVector::clearSearchCache() {
+    if (_search_cache) {
+        _search_cache->clear();
+    }
+}
+
 long ImportedAttributeVector::onSerializeForAscendingSort(DocId doc,
                                                           void *serTo,
                                                           long available,
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h
index 568c0ff4cce..c94eda3d4db 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h
@@ -75,6 +75,7 @@ public:
     const std::shared_ptr<BitVectorSearchCache> &getSearchCache() const {
         return _search_cache;
     }
+    void clearSearchCache();
 
     /*
      * Create an imported attribute with a snapshot of lid to lid mapping.