From d1a59d7e4655ec927f1c6ae932bdcba21cb02b2e Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 29 Mar 2017 14:06:50 +0000 Subject: Don't load attributes that are too old. --- .../attribute_initializer_test.cpp | 9 +++++ .../attribute_manager/attribute_manager_test.cpp | 38 +++++++++++----------- .../tests/proton/attribute/attributeflush_test.cpp | 8 ++--- .../proton/attribute/attribute_initializer.cpp | 16 ++++++++- 4 files changed, 47 insertions(+), 24 deletions(-) (limited to 'searchcore') diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp index 37ef21fb700..b11c4f78217 100644 --- a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp @@ -173,6 +173,15 @@ TEST("require that tensor attributes will not be initialized with mismatching ty EXPECT_EQUAL(1, av->getNumDocs()); } +TEST("require that too old attribute is not loaded") +{ + saveAttr("a", int32_sv, 3, 2); + Fixture f; + auto av = f.createInitializer("a", int32_sv, 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + } TEST_MAIN() diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 7232f997b0e..a72e9862353 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -279,20 +279,20 @@ TEST_F("require that attributes are flushed and loaded", BaseFixture) proton::AttributeManager &am = amf._m; AttributeVector::SP a1 = amf.addAttribute("a1"); EXPECT_EQUAL(1u, a1->getNumDocs()); // Resized to size of attributemanager - fillAttribute(a1, 1, 3, 2, 10); + fillAttribute(a1, 1, 3, 2, 100); EXPECT_EQUAL(3u, a1->getNumDocs()); // Resized to size of attributemanager AttributeVector::SP a2 = amf.addAttribute("a2"); EXPECT_EQUAL(1u, a2->getNumDocs()); // Not resized to size of attributemanager - fillAttribute(a2, 1, 5, 4, 10); + fillAttribute(a2, 1, 5, 4, 100); EXPECT_EQUAL(5u, a2->getNumDocs()); // Increased EXPECT_TRUE(!ia1.load()); EXPECT_TRUE(!ia2.load()); EXPECT_TRUE(!ia3.load()); am.flushAll(0); EXPECT_TRUE(ia1.load()); - EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken); + EXPECT_EQUAL(100u, ia1.getBestSnapshot().syncToken); EXPECT_TRUE(ia2.load()); - EXPECT_EQUAL(10u, ia2.getBestSnapshot().syncToken); + EXPECT_EQUAL(100u, ia2.getBestSnapshot().syncToken); } { AttributeManagerFixture amf(f); @@ -300,7 +300,7 @@ TEST_F("require that attributes are flushed and loaded", BaseFixture) AttributeVector::SP a1 = amf.addAttribute("a1"); // loaded EXPECT_EQUAL(3u, a1->getNumDocs()); - fillAttribute(a1, 1, 2, 20); + fillAttribute(a1, 1, 2, 200); EXPECT_EQUAL(4u, a1->getNumDocs()); AttributeVector::SP a2 = amf.addAttribute("a2"); // loaded EXPECT_EQUAL(5u, a2->getNumDocs()); @@ -308,26 +308,26 @@ TEST_F("require that attributes are flushed and loaded", BaseFixture) amf._aw.onReplayDone(5u); EXPECT_EQUAL(5u, a2->getNumDocs()); EXPECT_EQUAL(5u, a1->getNumDocs()); - fillAttribute(a2, 1, 4, 20); + fillAttribute(a2, 1, 4, 200); EXPECT_EQUAL(6u, a2->getNumDocs()); AttributeVector::SP a3 = amf.addAttribute("a3"); // not-loaded EXPECT_EQUAL(1u, a3->getNumDocs()); amf._aw.onReplayDone(6); EXPECT_EQUAL(6u, a3->getNumDocs()); - fillAttribute(a3, 1, 7, 6, 20); + fillAttribute(a3, 1, 7, 6, 200); EXPECT_EQUAL(7u, a3->getNumDocs()); EXPECT_TRUE(ia1.load()); - EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken); + EXPECT_EQUAL(100u, ia1.getBestSnapshot().syncToken); EXPECT_TRUE(ia2.load()); - EXPECT_EQUAL(10u, ia2.getBestSnapshot().syncToken); + EXPECT_EQUAL(100u, ia2.getBestSnapshot().syncToken); EXPECT_TRUE(!ia3.load()); am.flushAll(0); EXPECT_TRUE(ia1.load()); - EXPECT_EQUAL(20u, ia1.getBestSnapshot().syncToken); + EXPECT_EQUAL(200u, ia1.getBestSnapshot().syncToken); EXPECT_TRUE(ia2.load()); - EXPECT_EQUAL(20u, ia2.getBestSnapshot().syncToken); + EXPECT_EQUAL(200u, ia2.getBestSnapshot().syncToken); EXPECT_TRUE(ia3.load()); - EXPECT_EQUAL(20u, ia3.getBestSnapshot().syncToken); + EXPECT_EQUAL(200u, ia3.getBestSnapshot().syncToken); } { AttributeManagerFixture amf(f); @@ -364,14 +364,14 @@ TEST_F("require that predicate attributes are flushed and loaded", BaseFixture) uint32_t doc_id; a1->addDoc(doc_id); index.indexEmptyDocument(doc_id); - pa.commit(10, 10); + pa.commit(100, 100); EXPECT_EQUAL(2u, a1->getNumDocs()); EXPECT_TRUE(!ia1.load()); am.flushAll(0); EXPECT_TRUE(ia1.load()); - EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken); + EXPECT_EQUAL(100u, ia1.getBestSnapshot().syncToken); } { AttributeManagerFixture amf(f); @@ -388,14 +388,14 @@ TEST_F("require that predicate attributes are flushed and loaded", BaseFixture) PredicateTreeAnnotations annotations(3); annotations.interval_map[123] = {{ 0x0001ffff }}; index.indexDocument(1, annotations); - pa.commit(20, 20); + pa.commit(200, 200); EXPECT_EQUAL(3u, a1->getNumDocs()); EXPECT_TRUE(ia1.load()); - EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken); + EXPECT_EQUAL(100u, ia1.getBestSnapshot().syncToken); am.flushAll(0); EXPECT_TRUE(ia1.load()); - EXPECT_EQUAL(20u, ia1.getBestSnapshot().syncToken); + EXPECT_EQUAL(200u, ia1.getBestSnapshot().syncToken); } } @@ -639,14 +639,14 @@ populateAndFlushAttributes(AttributeManagerFixture &f) fillAttribute(a2, 1, 10, attrValue, createSerialNum); AttributeVector::SP a3 = f.addAttribute("a3"); fillAttribute(a3, 1, 10, attrValue, createSerialNum); - f._m.flushAll(createSerialNum + 3); + f._m.flushAll(createSerialNum + 10); } void validateAttribute(const AttributeVector &attr) { ASSERT_EQUAL(10u, attr.getNumDocs()); - EXPECT_EQUAL(createSerialNum + 3, attr.getStatus().getLastSyncToken()); + EXPECT_EQUAL(createSerialNum + 10, attr.getStatus().getLastSyncToken()); for (uint32_t docId = 1; docId < 10; ++docId) { EXPECT_EQUAL(7, attr.getInt(docId)); } diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp index f3fd48620a0..79a10b8da37 100644 --- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp @@ -498,8 +498,8 @@ Test::requireThatLastFlushTimeIsReported(void) AttributeVector::SP av = amf.addAttribute("a9"); IFlushTarget::SP ft = am.getFlushable("a9"); EXPECT_EQUAL(0, ft->getLastFlushTime().time()); - ft->initFlush(5)->run(); - EXPECT_TRUE(FastOS_File::Stat("flush/a9/snapshot-5", &stat)); + ft->initFlush(200)->run(); + EXPECT_TRUE(FastOS_File::Stat("flush/a9/snapshot-200", &stat)); EXPECT_EQUAL(stat._modifiedTime, ft->getLastFlushTime().time()); } { // snapshot flushed @@ -511,7 +511,7 @@ Test::requireThatLastFlushTimeIsReported(void) { // updated flush time after nothing to flush FastOS_Thread::Sleep(8000); fastos::TimeStamp now = fastos::ClockSystem::now(); - Executor::Task::UP task = ft->initFlush(5); + Executor::Task::UP task = ft->initFlush(200); EXPECT_TRUE(task.get() == NULL); EXPECT_LESS(stat._modifiedTime, ft->getLastFlushTime().time()); EXPECT_APPROX(now.time(), ft->getLastFlushTime().time(), 8); @@ -587,7 +587,7 @@ Test::requireThatFlushedAttributeCanBeLoaded(const HwInfo &hwInfo) } av->commit(); IFlushTarget::SP ft = am.getFlushable(attrName); - ft->initFlush(5)->run(); + ft->initFlush(200)->run(); } { AttributeManagerFixture amf(f); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp index 7a23db1a071..0363b191656 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp @@ -100,6 +100,17 @@ logAttributeTooNew(const AttributeVector::SP &attr, serialNum); } +void +logAttributeTooOld(const AttributeVector::SP &attr, + search::SerialNum flushedSerialNum, + uint64_t serialNum) +{ + LOG(info, "Attribute vector '%s' is too old (%" PRIu64 " < %" PRIu64 ")", + attr->getBaseFileName().c_str(), + flushedSerialNum, + serialNum); +} + void logAttributeWrongType(const AttributeVector::SP &attr, const AttributeHeader &header) @@ -127,7 +138,7 @@ AttributeInitializer::tryLoadAttribute() const AttributeVector::SP attr = _factory.create(attrFileName, _cfg); if (serialNum != 0) { AttributeHeader header = extractHeader(attrFileName); - if (header.getCreateSerialNum() > _currentSerialNum || !headerTypeOK(header, attr->getConfig())) { + if (header.getCreateSerialNum() > _currentSerialNum || !headerTypeOK(header, attr->getConfig()) || (serialNum < _currentSerialNum)) { setupEmptyAttribute(attr, serialNum, header); return attr; } @@ -169,6 +180,9 @@ AttributeInitializer::setupEmptyAttribute(AttributeVector::SP &attr, if (header.getCreateSerialNum() > _currentSerialNum) { logAttributeTooNew(attr, header, _currentSerialNum); } + if (serialNum < _currentSerialNum) { + logAttributeTooOld(attr, serialNum, _currentSerialNum); + } if (!headerTypeOK(header, attr->getConfig())) { logAttributeWrongType(attr, header); } -- cgit v1.2.3