Don't load attributes that are too old.

4 files changed, 47 insertions, 24 deletions

diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp
index 37ef21fb700..b11c4f78217 100644
--- a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp
@@ -173,6 +173,15 @@ TEST("require that tensor attributes will not be initialized with mismatching ty
     EXPECT_EQUAL(1, av->getNumDocs());
 }
 
+TEST("require that too old attribute is not loaded")
+{
+    saveAttr("a", int32_sv, 3, 2);
+    Fixture f;
+    auto av = f.createInitializer("a", int32_sv, 5)->init();
+    EXPECT_EQUAL(5, av->getCreateSerialNum());
+    EXPECT_EQUAL(1, av->getNumDocs());
+}
+
 }
 
 TEST_MAIN()
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
index 7232f997b0e..a72e9862353 100644
--- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
@@ -279,20 +279,20 @@ TEST_F("require that attributes are flushed and loaded", BaseFixture)
         proton::AttributeManager &am = amf._m;
         AttributeVector::SP a1 = amf.addAttribute("a1");
         EXPECT_EQUAL(1u, a1->getNumDocs());	// Resized to size of attributemanager
-        fillAttribute(a1, 1, 3, 2, 10);
+        fillAttribute(a1, 1, 3, 2, 100);
         EXPECT_EQUAL(3u, a1->getNumDocs());	// Resized to size of attributemanager
         AttributeVector::SP a2 = amf.addAttribute("a2");
         EXPECT_EQUAL(1u, a2->getNumDocs());	// Not resized to size of attributemanager
-        fillAttribute(a2, 1, 5, 4, 10);
+        fillAttribute(a2, 1, 5, 4, 100);
         EXPECT_EQUAL(5u, a2->getNumDocs());	// Increased
         EXPECT_TRUE(!ia1.load());
         EXPECT_TRUE(!ia2.load());
         EXPECT_TRUE(!ia3.load());
         am.flushAll(0);
         EXPECT_TRUE(ia1.load());
-        EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken);
+        EXPECT_EQUAL(100u, ia1.getBestSnapshot().syncToken);
         EXPECT_TRUE(ia2.load());
-        EXPECT_EQUAL(10u, ia2.getBestSnapshot().syncToken);
+        EXPECT_EQUAL(100u, ia2.getBestSnapshot().syncToken);
     }
     {
         AttributeManagerFixture amf(f);
@@ -300,7 +300,7 @@ TEST_F("require that attributes are flushed and loaded", BaseFixture)
         AttributeVector::SP a1 = amf.addAttribute("a1"); // loaded
 
         EXPECT_EQUAL(3u, a1->getNumDocs());
-        fillAttribute(a1, 1, 2, 20);
+        fillAttribute(a1, 1, 2, 200);
         EXPECT_EQUAL(4u, a1->getNumDocs());
         AttributeVector::SP a2 = amf.addAttribute("a2"); // loaded
         EXPECT_EQUAL(5u, a2->getNumDocs());
@@ -308,26 +308,26 @@ TEST_F("require that attributes are flushed and loaded", BaseFixture)
         amf._aw.onReplayDone(5u);
         EXPECT_EQUAL(5u, a2->getNumDocs());
         EXPECT_EQUAL(5u, a1->getNumDocs());
-        fillAttribute(a2, 1, 4, 20);
+        fillAttribute(a2, 1, 4, 200);
         EXPECT_EQUAL(6u, a2->getNumDocs());
         AttributeVector::SP a3 = amf.addAttribute("a3"); // not-loaded
         EXPECT_EQUAL(1u, a3->getNumDocs());
         amf._aw.onReplayDone(6);
         EXPECT_EQUAL(6u, a3->getNumDocs());
-        fillAttribute(a3, 1, 7, 6, 20);
+        fillAttribute(a3, 1, 7, 6, 200);
         EXPECT_EQUAL(7u, a3->getNumDocs());
         EXPECT_TRUE(ia1.load());
-        EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken);
+        EXPECT_EQUAL(100u, ia1.getBestSnapshot().syncToken);
         EXPECT_TRUE(ia2.load());
-        EXPECT_EQUAL(10u, ia2.getBestSnapshot().syncToken);
+        EXPECT_EQUAL(100u, ia2.getBestSnapshot().syncToken);
         EXPECT_TRUE(!ia3.load());
         am.flushAll(0);
         EXPECT_TRUE(ia1.load());
-        EXPECT_EQUAL(20u, ia1.getBestSnapshot().syncToken);
+        EXPECT_EQUAL(200u, ia1.getBestSnapshot().syncToken);
         EXPECT_TRUE(ia2.load());
-        EXPECT_EQUAL(20u, ia2.getBestSnapshot().syncToken);
+        EXPECT_EQUAL(200u, ia2.getBestSnapshot().syncToken);
         EXPECT_TRUE(ia3.load());
-        EXPECT_EQUAL(20u, ia3.getBestSnapshot().syncToken);
+        EXPECT_EQUAL(200u, ia3.getBestSnapshot().syncToken);
     }
     {
         AttributeManagerFixture amf(f);
@@ -364,14 +364,14 @@ TEST_F("require that predicate attributes are flushed and loaded", BaseFixture)
         uint32_t doc_id;
         a1->addDoc(doc_id);
         index.indexEmptyDocument(doc_id);
-        pa.commit(10, 10);
+        pa.commit(100, 100);
 
         EXPECT_EQUAL(2u, a1->getNumDocs());
 
         EXPECT_TRUE(!ia1.load());
         am.flushAll(0);
         EXPECT_TRUE(ia1.load());
-        EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken);
+        EXPECT_EQUAL(100u, ia1.getBestSnapshot().syncToken);
     }
     {
         AttributeManagerFixture amf(f);
@@ -388,14 +388,14 @@ TEST_F("require that predicate attributes are flushed and loaded", BaseFixture)
         PredicateTreeAnnotations annotations(3);
         annotations.interval_map[123] = {{ 0x0001ffff }};
         index.indexDocument(1, annotations);
-        pa.commit(20, 20);
+        pa.commit(200, 200);
 
         EXPECT_EQUAL(3u, a1->getNumDocs());
         EXPECT_TRUE(ia1.load());
-        EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken);
+        EXPECT_EQUAL(100u, ia1.getBestSnapshot().syncToken);
         am.flushAll(0);
         EXPECT_TRUE(ia1.load());
-        EXPECT_EQUAL(20u, ia1.getBestSnapshot().syncToken);
+        EXPECT_EQUAL(200u, ia1.getBestSnapshot().syncToken);
     }
 }
 
@@ -639,14 +639,14 @@ populateAndFlushAttributes(AttributeManagerFixture &f)
     fillAttribute(a2, 1, 10, attrValue, createSerialNum);
     AttributeVector::SP a3 = f.addAttribute("a3");
     fillAttribute(a3, 1, 10, attrValue, createSerialNum);
-    f._m.flushAll(createSerialNum + 3);
+    f._m.flushAll(createSerialNum + 10);
 }
 
 void
 validateAttribute(const AttributeVector &attr)
 {
     ASSERT_EQUAL(10u, attr.getNumDocs());
-    EXPECT_EQUAL(createSerialNum + 3, attr.getStatus().getLastSyncToken());
+    EXPECT_EQUAL(createSerialNum + 10, attr.getStatus().getLastSyncToken());
     for (uint32_t docId = 1; docId < 10; ++docId) {
         EXPECT_EQUAL(7, attr.getInt(docId));
     }
diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
index f3fd48620a0..79a10b8da37 100644
--- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
@@ -498,8 +498,8 @@ Test::requireThatLastFlushTimeIsReported(void)
         AttributeVector::SP av = amf.addAttribute("a9");
         IFlushTarget::SP ft = am.getFlushable("a9");
         EXPECT_EQUAL(0, ft->getLastFlushTime().time());
-        ft->initFlush(5)->run();
-        EXPECT_TRUE(FastOS_File::Stat("flush/a9/snapshot-5", &stat));
+        ft->initFlush(200)->run();
+        EXPECT_TRUE(FastOS_File::Stat("flush/a9/snapshot-200", &stat));
         EXPECT_EQUAL(stat._modifiedTime, ft->getLastFlushTime().time());
     }
     { // snapshot flushed
@@ -511,7 +511,7 @@ Test::requireThatLastFlushTimeIsReported(void)
         { // updated flush time after nothing to flush
             FastOS_Thread::Sleep(8000);
             fastos::TimeStamp now = fastos::ClockSystem::now();
-            Executor::Task::UP task = ft->initFlush(5);
+            Executor::Task::UP task = ft->initFlush(200);
             EXPECT_TRUE(task.get() == NULL);
             EXPECT_LESS(stat._modifiedTime, ft->getLastFlushTime().time());
             EXPECT_APPROX(now.time(), ft->getLastFlushTime().time(), 8);
@@ -587,7 +587,7 @@ Test::requireThatFlushedAttributeCanBeLoaded(const HwInfo &hwInfo)
         }
         av->commit();
         IFlushTarget::SP ft = am.getFlushable(attrName);
-        ft->initFlush(5)->run();
+        ft->initFlush(200)->run();
     }
     {
         AttributeManagerFixture amf(f);
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp
index 7a23db1a071..0363b191656 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp
@@ -101,6 +101,17 @@ logAttributeTooNew(const AttributeVector::SP &attr,
 }
 
 void
+logAttributeTooOld(const AttributeVector::SP &attr,
+                   search::SerialNum flushedSerialNum,
+                   uint64_t serialNum)
+{
+    LOG(info, "Attribute vector '%s' is too old (%" PRIu64 " < %" PRIu64 ")",
+        attr->getBaseFileName().c_str(),
+        flushedSerialNum,
+        serialNum);
+}
+
+void
 logAttributeWrongType(const AttributeVector::SP &attr,
                       const AttributeHeader &header)
 {
@@ -127,7 +138,7 @@ AttributeInitializer::tryLoadAttribute() const
     AttributeVector::SP attr = _factory.create(attrFileName, _cfg);
     if (serialNum != 0) {
         AttributeHeader header = extractHeader(attrFileName);
-        if (header.getCreateSerialNum() > _currentSerialNum || !headerTypeOK(header, attr->getConfig())) {
+        if (header.getCreateSerialNum() > _currentSerialNum || !headerTypeOK(header, attr->getConfig()) || (serialNum < _currentSerialNum)) {
             setupEmptyAttribute(attr, serialNum, header);
             return attr;
         }
@@ -169,6 +180,9 @@ AttributeInitializer::setupEmptyAttribute(AttributeVector::SP &attr,
     if (header.getCreateSerialNum() > _currentSerialNum) {
         logAttributeTooNew(attr, header, _currentSerialNum);
     }
+    if (serialNum < _currentSerialNum) {
+        logAttributeTooOld(attr, serialNum, _currentSerialNum);
+    }
     if (!headerTypeOK(header, attr->getConfig())) {
         logAttributeWrongType(attr, header);
     }