diff options
19 files changed, 110 insertions, 60 deletions
diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 3a91aa9fa41..411730ce188 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -241,9 +241,11 @@ public: const document::DocumentId &docId = doc.getId(); typedef DocumentMetaStore::Result PutRes; IDocumentMetaStore &dms = _ddb->getReadySubDB()->getDocumentMetaStoreContext().get(); + uint32_t docSize = 1; PutRes putRes(dms.put(docId.getGlobalId(), BucketFactory::getBucketId(docId), Timestamp(0u), + docSize, lid)); LOG_ASSERT(putRes.ok()); uint64_t serialNum = _ddb->getFeedHandler().incSerialNum(); diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp index f139bf92e44..0005ca64fbf 100644 --- a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp @@ -44,7 +44,7 @@ struct MySubDb for (size_t i = 0; i < bucketDocs.getDocs().size(); ++i) { const test::Document &testDoc = bucketDocs.getDocs()[i]; _metaStore.put(testDoc.getGid(), testDoc.getBucket(), - testDoc.getTimestamp(), testDoc.getLid()); + testDoc.getTimestamp(), testDoc.getSize(), testDoc.getLid()); } } } diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp index d3d3aa4ac0d..e1a2f9a0008 100644 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp @@ -102,7 +102,7 @@ struct MySubDb for (size_t i = 0; i < docs.getDocs().size(); ++i) { const test::Document &testDoc = docs.getDocs()[i]; _view->_metaStore.put(testDoc.getGid(), testDoc.getBucket(), - testDoc.getTimestamp(), testDoc.getLid()); + testDoc.getTimestamp(), testDoc.getSize(), testDoc.getLid()); } } }; diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp index a7a9042cd54..6ffe9b2494a 100644 --- a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp @@ -38,7 +38,8 @@ struct Fixture const GlobalId &gid = docId.getGlobalId(); DMSResult res = _metaStore.inspect(gid); ASSERT_EQUAL(lid, res._lid); - _metaStore.put(gid, gid.convertToBucketId(), Timestamp(lid), lid); + uint32_t docSize = 1; + _metaStore.put(gid, gid.convertToBucketId(), Timestamp(lid), docSize, lid); return *this; } LidSet scan(uint32_t count, uint32_t compactLidLimit, uint32_t maxDocsToScan = 10) { diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp index 7842125c4c5..bda7f3b6f72 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp @@ -135,7 +135,7 @@ struct MySubDb for (size_t i = 0; i < bucketDocs.getDocs().size(); ++i) { const test::Document &testDoc = bucketDocs.getDocs()[i]; _metaStore.put(testDoc.getGid(), testDoc.getBucket(), - testDoc.getTimestamp(), testDoc.getLid()); + testDoc.getTimestamp(), testDoc.getSize(), testDoc.getLid()); _realRetriever->_docs.push_back(testDoc.getDoc()); ASSERT_EQUAL(testDoc.getLid() + 1, _realRetriever->_docs.size()); diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index f8647153fd9..7e6e7632236 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -643,6 +643,7 @@ MyDocumentSubDB::handlePut(PutOperation &op) PutRes putRes(_metaStore.put(gid, op.getBucketId(), op.getTimestamp(), + op.getSerializedDocSize(), op.getLid())); assert(putRes.ok()); assert(op.getLid() == putRes._lid); @@ -684,6 +685,7 @@ MyDocumentSubDB::handleRemove(RemoveOperation &op) PutRes putRes(_metaStore.put(gid, op.getBucketId(), op.getTimestamp(), + op.getSerializedDocSize(), op.getLid())); assert(putRes.ok()); assert(op.getLid() == putRes._lid); @@ -741,6 +743,7 @@ MyDocumentSubDB::handleMove(const MoveOperation &op) PutRes putRes(_metaStore.put(gid, op.getBucketId(), op.getTimestamp(), + op.getSerializedDocSize(), op.getLid())); assert(putRes.ok()); assert(op.getLid() == putRes._lid); diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index eadc7566162..2c54188af3a 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -159,10 +159,11 @@ struct Fixture { typedef DocumentMetaStore::Result Result; DocumentId id(make_string("groupdoc:test:foo:%d", expected_lid)); Result inspect = meta_store->inspect(id.getGlobalId()); + uint32_t docSize = 1; EXPECT_EQUAL(expected_lid, meta_store->put(id.getGlobalId(), id.getGlobalId().convertToBucketId(), - Timestamp(10), inspect.getLid()).getLid()); + Timestamp(10), docSize, inspect.getLid()).getLid()); } void addDocsToMetaStore(int count) { @@ -219,12 +220,13 @@ TEST_F("require that handleMove handles move within same subdb", Fixture) { Document::SP doc(new Document); DocumentId doc1id("groupdoc:test:foo:1"); + uint32_t docSize = 1; f.runInMaster([&] () { f.meta_store->put(doc1id.getGlobalId(), doc1id.getGlobalId().convertToBucketId(), - Timestamp(9), 1); }); + Timestamp(9), docSize, 1); }); f.runInMaster([&] () { f.meta_store->put(doc->getId().getGlobalId(), doc->getId().getGlobalId().convertToBucketId(), - Timestamp(10), 2); }); + Timestamp(10), docSize, 2); }); f.runInMaster([&] () { f.meta_store->remove(1); }); f.meta_store->removeComplete(1); MoveOperation op(doc->getId().getGlobalId().convertToBucketId(), diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index 43738438fc6..722b0856aa4 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -110,13 +110,14 @@ createBucketDB() bool assertPut(const BucketId &bucketId, const Timestamp ×tamp, + uint32_t docSize, uint32_t lid, const GlobalId &gid, DocumentMetaStore &dms) { Result inspect = dms.inspect(gid); PutRes putRes; - if (!EXPECT_TRUE((putRes = dms.put(gid, bucketId, timestamp, inspect.getLid())). + if (!EXPECT_TRUE((putRes = dms.put(gid, bucketId, timestamp, docSize, inspect.getLid())). ok())) return false; return EXPECT_EQUAL(lid, putRes.getLid()); } @@ -257,28 +258,34 @@ Timestamp time2(2u); Timestamp time3(42u); Timestamp time4(82u); Timestamp time5(141u); +uint32_t docSize1 = 1; +uint32_t docSize2 = 1; +uint32_t docSize3 = 1; +uint32_t docSize4 = 1; +uint32_t docSize5 = 1; uint32_t -addGid(DocumentMetaStore &dms, const GlobalId &gid, const BucketId &bid, Timestamp timestamp = Timestamp()) +addGid(DocumentMetaStore &dms, const GlobalId &gid, const BucketId &bid, Timestamp timestamp, uint32_t docSize) { Result inspect = dms.inspect(gid); PutRes putRes; - EXPECT_TRUE((putRes = dms.put(gid, bid, timestamp, inspect.getLid())).ok()); + EXPECT_TRUE((putRes = dms.put(gid, bid, timestamp, docSize, inspect.getLid())).ok()); return putRes.getLid(); } uint32_t -addGid(DocumentMetaStore &dms, const GlobalId &gid, Timestamp timestamp = Timestamp()) +addGid(DocumentMetaStore &dms, const GlobalId &gid, Timestamp timestamp, uint32_t docSize) { BucketId bid(minNumBits, gid.convertToBucketId().getRawId()); - return addGid(dms, gid, bid, timestamp); + return addGid(dms, gid, bid, timestamp, docSize); } void putGid(DocumentMetaStore &dms, const GlobalId &gid, uint32_t lid, Timestamp timestamp = Timestamp()) { BucketId bid(minNumBits, gid.convertToBucketId().getRawId()); - EXPECT_TRUE(dms.put(gid, bid, timestamp, lid).ok()); + uint32_t docSize = 1; + EXPECT_TRUE(dms.put(gid, bid, timestamp, docSize, lid).ok()); } TEST("require that removed documents are bucketized to bucket 0") @@ -290,9 +297,9 @@ TEST("require that removed documents are bucketized to bucket 0") vespalib::GenerationHandler::Guard guard = dms.getGuard(); EXPECT_EQUAL(BucketId(), dms.getBucketOf(guard, 1)); - EXPECT_TRUE(assertPut(bucketId1, time1, 1, gid1, dms)); + EXPECT_TRUE(assertPut(bucketId1, time1, docSize1, 1, gid1, dms)); EXPECT_EQUAL(bucketId1, dms.getBucketOf(guard, 1)); - EXPECT_TRUE(assertPut(bucketId2, time2, 2, gid2, dms)); + EXPECT_TRUE(assertPut(bucketId2, time2, docSize2, 2, gid2, dms)); EXPECT_EQUAL(bucketId2, dms.getBucketOf(guard, 2)); EXPECT_TRUE(dms.remove(1)); EXPECT_EQUAL(BucketId(), dms.getBucketOf(guard, 1)); @@ -306,16 +313,16 @@ TEST("requireThatGidsCanBeInsertedAndRetrieved") // put() EXPECT_EQUAL(1u, dms.getNumDocs()); EXPECT_EQUAL(0u, dms.getNumUsedLids()); - EXPECT_TRUE(assertPut(bucketId1, time1, 1, gid1, dms)); + EXPECT_TRUE(assertPut(bucketId1, time1, docSize1, 1, gid1, dms)); EXPECT_EQUAL(2u, dms.getNumDocs()); EXPECT_EQUAL(1u, dms.getNumUsedLids()); - EXPECT_TRUE(assertPut(bucketId2, time2, 2, gid2, dms)); + EXPECT_TRUE(assertPut(bucketId2, time2, docSize2, 2, gid2, dms)); EXPECT_EQUAL(3u, dms.getNumDocs()); EXPECT_EQUAL(2u, dms.getNumUsedLids()); // gid1 already inserted - EXPECT_TRUE(assertPut(bucketId1, time1, 1, gid1, dms)); + EXPECT_TRUE(assertPut(bucketId1, time1, docSize1, 1, gid1, dms)); // gid2 already inserted - EXPECT_TRUE(assertPut(bucketId2, time2, 2, gid2, dms)); + EXPECT_TRUE(assertPut(bucketId2, time2, docSize2, 2, gid2, dms)); // getGid() @@ -337,7 +344,7 @@ TEST("requireThatGidsCanBeCleared") GlobalId gid; uint32_t lid = 0u; dms.constructFreeList(); - addGid(dms, gid1, bucketId1, time1); + addGid(dms, gid1, bucketId1, time1, docSize1); EXPECT_TRUE(assertGid(gid1, 1, dms)); EXPECT_TRUE(assertLid(1, gid1, dms)); EXPECT_EQUAL(1u, dms.getNumUsedLids()); @@ -347,7 +354,7 @@ TEST("requireThatGidsCanBeCleared") EXPECT_TRUE(!dms.getGid(1, gid)); EXPECT_TRUE(!dms.getLid(gid1, lid)); // reuse lid - addGid(dms, gid2, bucketId2, time2); + addGid(dms, gid2, bucketId2, time2, docSize2); EXPECT_TRUE(assertGid(gid2, 1, dms)); EXPECT_TRUE(assertLid(1, gid2, dms)); EXPECT_EQUAL(1u, dms.getNumUsedLids()); @@ -367,7 +374,7 @@ TEST("requireThatGenerationHandlingIsWorking") dms->constructFreeList(); const GenerationHandler & gh = dms->getGenerationHandler(); EXPECT_EQUAL(1u, gh.getCurrentGeneration()); - addGid(*dms, gid1, bucketId1, time1); + addGid(*dms, gid1, bucketId1, time1, docSize1); EXPECT_EQUAL(2u, gh.getCurrentGeneration()); EXPECT_EQUAL(0u, gh.getGenerationRefCount()); { @@ -509,10 +516,10 @@ TEST("requireThatLidAndGidSpaceIsReused") dms->constructFreeList(); EXPECT_EQUAL(1u, dms->getNumDocs()); EXPECT_EQUAL(0u, dms->getNumUsedLids()); - EXPECT_TRUE(assertPut(bucketId1, time1, 1, gid1, *dms)); // -> gen 1 + EXPECT_TRUE(assertPut(bucketId1, time1, docSize1, 1, gid1, *dms)); // -> gen 1 EXPECT_EQUAL(2u, dms->getNumDocs()); EXPECT_EQUAL(1u, dms->getNumUsedLids()); - EXPECT_TRUE(assertPut(bucketId2, time2, 2, gid2, *dms)); // -> gen 2 + EXPECT_TRUE(assertPut(bucketId2, time2, docSize2, 2, gid2, *dms)); // -> gen 2 EXPECT_EQUAL(3u, dms->getNumDocs()); EXPECT_EQUAL(2u, dms->getNumUsedLids()); dms->remove(2); // -> gen 3 @@ -520,7 +527,7 @@ TEST("requireThatLidAndGidSpaceIsReused") EXPECT_EQUAL(3u, dms->getNumDocs()); EXPECT_EQUAL(1u, dms->getNumUsedLids()); // -> gen 5 (reuse of lid 2) - EXPECT_TRUE(assertPut(bucketId3, time3, 2, gid3, *dms)); + EXPECT_TRUE(assertPut(bucketId3, time3, docSize3, 2, gid3, *dms)); EXPECT_EQUAL(3u, dms->getNumDocs()); EXPECT_EQUAL(2u, dms->getNumUsedLids()); // reuse EXPECT_TRUE(assertGid(gid3, 2, *dms)); @@ -530,16 +537,16 @@ TEST("requireThatLidAndGidSpaceIsReused") dms->removeComplete(2); EXPECT_EQUAL(3u, dms->getNumDocs()); EXPECT_EQUAL(1u, dms->getNumUsedLids()); // lid 2 free but guarded - EXPECT_TRUE(assertPut(bucketId4, time4, 3, gid4, *dms)); + EXPECT_TRUE(assertPut(bucketId4, time4, docSize4, 3, gid4, *dms)); EXPECT_EQUAL(4u, dms->getNumDocs()); // generation guarded, new lid EXPECT_EQUAL(2u, dms->getNumUsedLids()); EXPECT_TRUE(assertGid(gid4, 3, *dms)); } - EXPECT_TRUE(assertPut(bucketId5, time5, 4, gid5, *dms)); + EXPECT_TRUE(assertPut(bucketId5, time5, docSize5, 4, gid5, *dms)); EXPECT_EQUAL(5u, dms->getNumDocs()); // reuse blocked by previous guard. released at end of put() EXPECT_EQUAL(3u, dms->getNumUsedLids()); EXPECT_TRUE(assertGid(gid5, 4, *dms)); - EXPECT_TRUE(assertPut(bucketId2, time2, 2, gid2, *dms)); // reuse of lid 2 + EXPECT_TRUE(assertPut(bucketId2, time2, docSize2, 2, gid2, *dms)); // reuse of lid 2 EXPECT_EQUAL(5u, dms->getNumDocs()); EXPECT_EQUAL(4u, dms->getNumUsedLids()); EXPECT_TRUE(assertGid(gid2, 2, *dms)); @@ -569,7 +576,8 @@ TEST("requireThatWeCanStoreBucketIdAndTimestamp") GlobalId gid = createGid(lid); BucketId bucketId(gid.convertToBucketId()); bucketId.setUsedBits(numBucketBits); - uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(lid + timestampBias)); + uint32_t docSize = 1; + uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(lid + timestampBias), docSize); EXPECT_EQUAL(lid, addLid); } for (uint32_t lid = 1; lid <= numLids; ++lid) { @@ -596,7 +604,8 @@ TEST("requireThatGidsCanBeSavedAndLoaded") GlobalId gid = createGid(lid); BucketId bucketId(gid.convertToBucketId()); bucketId.setUsedBits(numBucketBits); - uint32_t addLid = addGid(dms1, gid, bucketId, Timestamp(lid + timestampBias)); + uint32_t docSize = 1; + uint32_t addLid = addGid(dms1, gid, bucketId, Timestamp(lid + timestampBias), docSize); EXPECT_EQUAL(lid, addLid); } for (size_t i = 0; i < removeLids.size(); ++i) { @@ -639,7 +648,8 @@ TEST("requireThatGidsCanBeSavedAndLoaded") BucketId bucketId(numBucketBits, gid.convertToBucketId().getRawId()); // re-use removeLid[i] - uint32_t addLid = addGid(dms2, gid, bucketId, Timestamp(43u + i)); + uint32_t docSize = 1; + uint32_t addLid = addGid(dms2, gid, bucketId, Timestamp(43u + i), docSize); EXPECT_EQUAL(removeLids[i], addLid); EXPECT_EQUAL(numLids + 1, dms2.getNumDocs()); EXPECT_EQUAL(numLids - (3 - i), dms2.getNumUsedLids()); @@ -659,7 +669,7 @@ TEST("requireThatStatsAreUpdated") EXPECT_GREATER(lastUsed, perGidUsed); FastOS_Thread::Sleep(2200); - addGid(dms, gid1, bucketId1, time1); + addGid(dms, gid1, bucketId1, time1, docSize1); EXPECT_EQUAL(2u, dms.getStatus().getNumDocs()); EXPECT_EQUAL(2u, dms.getStatus().getNumValues()); EXPECT_GREATER_EQUAL(dms.getStatus().getAllocated(), lastAllocated); @@ -669,7 +679,7 @@ TEST("requireThatStatsAreUpdated") lastAllocated = dms.getStatus().getAllocated(); lastUsed = dms.getStatus().getUsed(); - addGid(dms, gid2, bucketId2, time2); + addGid(dms, gid2, bucketId2, time2, docSize2); dms.commit(true); EXPECT_EQUAL(3u, dms.getStatus().getNumDocs()); EXPECT_EQUAL(3u, dms.getStatus().getNumValues()); @@ -688,23 +698,23 @@ TEST("requireThatStatsAreUpdated") TEST("requireThatWeCanPutAndRemoveBeforeFreeListConstruct") { DocumentMetaStore dms(createBucketDB()); - EXPECT_TRUE(dms.put(gid4, bucketId4, time4, 4).ok()); + EXPECT_TRUE(dms.put(gid4, bucketId4, time4, docSize4, 4).ok()); EXPECT_TRUE(assertLid(4, gid4, dms)); EXPECT_TRUE(assertGid(gid4, 4, dms)); EXPECT_EQUAL(1u, dms.getNumUsedLids()); EXPECT_EQUAL(5u, dms.getNumDocs()); - EXPECT_TRUE(dms.put(gid1, bucketId1, time1, 1).ok()); + EXPECT_TRUE(dms.put(gid1, bucketId1, time1, docSize1, 1).ok()); // already there, nothing changes - EXPECT_TRUE(dms.put(gid1, bucketId1, time1, 1).ok()); + EXPECT_TRUE(dms.put(gid1, bucketId1, time1, docSize1, 1).ok()); EXPECT_TRUE(assertLid(1, gid1, dms)); EXPECT_TRUE(assertGid(gid1, 1, dms)); EXPECT_EQUAL(2u, dms.getNumUsedLids()); EXPECT_EQUAL(5u, dms.getNumDocs()); // gid1 already there with lid 1 - EXPECT_EXCEPTION(!dms.put(gid1, bucketId1, time1, 2).ok(), + EXPECT_EXCEPTION(!dms.put(gid1, bucketId1, time1, docSize1, 2).ok(), vespalib::IllegalStateException, "gid found, but using another lid"); - EXPECT_EXCEPTION(!dms.put(gid5, bucketId5, time5, 1).ok(), + EXPECT_EXCEPTION(!dms.put(gid5, bucketId5, time5, docSize5, 1).ok(), vespalib::IllegalStateException, "gid not found, but lid is used by another gid"); EXPECT_TRUE(assertLid(1, gid1, dms)); @@ -721,8 +731,8 @@ TEST("requireThatWeCanPutAndRemoveBeforeFreeListConstruct") dms.constructFreeList(); EXPECT_EQUAL(1u, dms.getNumUsedLids()); EXPECT_EQUAL(5u, dms.getNumDocs()); - EXPECT_TRUE(assertPut(bucketId2, time2, 2, gid2, dms)); - EXPECT_TRUE(assertPut(bucketId3, time3, 3, gid3, dms)); + EXPECT_TRUE(assertPut(bucketId2, time2, docSize2, 2, gid2, dms)); + EXPECT_TRUE(assertPut(bucketId3, time3, docSize3, 3, gid3, dms)); EXPECT_EQUAL(3u, dms.getNumUsedLids()); EXPECT_EQUAL(5u, dms.getNumDocs()); } @@ -744,9 +754,10 @@ TEST("requireThatWeCanSortGids") Timestamp oldTimestamp; BucketId bucketId(minNumBits, gid.convertToBucketId().getRawId()); - uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(0u)); + uint32_t docSize = 1; + uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(0u), docSize); EXPECT_EQUAL(lid, addLid); - uint32_t addLid2 = addGid(rdms, gid, bucketId, Timestamp(0u)); + uint32_t addLid2 = addGid(rdms, gid, bucketId, Timestamp(0u), docSize); EXPECT_EQUAL(lid, addLid2); } std::vector<uint32_t> lids; @@ -777,7 +788,8 @@ TEST("requireThatBasicBucketInfoWorks") Timestamp oldTimestamp; BucketId bucketId(minNumBits, gid.convertToBucketId().getRawId()); - uint32_t addLid = addGid(dms, gid, bucketId, timestamp); + uint32_t docSize = 1; + uint32_t addLid = addGid(dms, gid, bucketId, timestamp, docSize); EXPECT_EQUAL(lid, addLid); m[std::make_pair(bucketId, gid)] = timestamp; } @@ -787,7 +799,8 @@ TEST("requireThatBasicBucketInfoWorks") Timestamp oldTimestamp; BucketId bucketId(minNumBits, gid.convertToBucketId().getRawId()); - uint32_t addLid = addGid(dms, gid, bucketId, timestamp); + uint32_t docSize = 1; + uint32_t addLid = addGid(dms, gid, bucketId, timestamp, docSize); EXPECT_EQUAL(lid, addLid); m[std::make_pair(bucketId, gid)] = timestamp; } @@ -844,7 +857,8 @@ TEST("requireThatWeCanRetrieveListOfLidsFromBucketId") GlobalId gid = createGid(lid); BucketId bucketId(bucketBits, gid.convertToBucketId().getRawId()); - uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(0)); + uint32_t docSize = 1; + uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(0), docSize); EXPECT_EQUAL(lid, addLid); m[bucketId].push_back(lid); } @@ -914,7 +928,8 @@ struct UserDocFixture { bid3 = BucketId(minNumBits, gids[7].convertToBucketId().getRawId()); } void addGlobalId(const GlobalId &gid, uint32_t expLid, uint32_t timestampConst = 100) { - uint32_t actLid = addGid(dms, gid, Timestamp(expLid + timestampConst)); + uint32_t docSize = 1; + uint32_t actLid = addGid(dms, gid, Timestamp(expLid + timestampConst), docSize); EXPECT_EQUAL(expLid, actLid); } void putGlobalId(const GlobalId &gid, uint32_t lid, uint32_t timestampConst = 100) { @@ -1191,14 +1206,18 @@ struct SplitAndJoinFixture : public SplitAndJoinEmptyFixture bid32Gids = &bid3s[bid32]; } void insertGids1() { + uint32_t docSize = 1; for (size_t i = 0; i < gids.size(); ++i) { EXPECT_TRUE(dms.put(gids[i].gid, gids[i].bid1, Timestamp(0), + docSize, gids[i].lid).ok()); } } void insertGids2() { + uint32_t docSize = 1; for (size_t i = 0; i < gids.size(); ++i) { EXPECT_TRUE(dms.put(gids[i].gid, gids[i].bid2, Timestamp(0), + docSize, gids[i].lid).ok()); } } @@ -1206,20 +1225,22 @@ struct SplitAndJoinFixture : public SplitAndJoinEmptyFixture void insertGids1Mostly(const BucketId &alt) { + uint32_t docSize = 1; for (size_t i = 0; i < gids.size(); ++i) { const GlobalIdEntry &g(gids[i]); BucketId b(g.bid3 == alt ? g.bid2 : g.bid1); - EXPECT_TRUE(dms.put(g.gid, b, Timestamp(0), g.lid).ok()); + EXPECT_TRUE(dms.put(g.gid, b, Timestamp(0), docSize, g.lid).ok()); } } void insertGids2Mostly(const BucketId &alt) { + uint32_t docSize = 1; for (size_t i = 0; i < gids.size(); ++i) { const GlobalIdEntry &g(gids[i]); BucketId b(g.bid3 == alt ? g.bid1 : g.bid2); - EXPECT_TRUE(dms.put(g.gid, b, Timestamp(0), g.lid).ok()); + EXPECT_TRUE(dms.put(g.gid, b, Timestamp(0), docSize, g.lid).ok()); } } }; @@ -1654,9 +1675,10 @@ TEST("requireThatRemoveChangedBucketWorks") GlobalIdEntry g(1); f.dms.constructFreeList(); f._bucketDBHandler.handleCreateBucket(g.bid1); - uint32_t addLid1 = addGid(f.dms, g.gid, g.bid1, Timestamp(0)); + uint32_t docSize = 1; + uint32_t addLid1 = addGid(f.dms, g.gid, g.bid1, Timestamp(0), docSize); EXPECT_EQUAL(1u, addLid1); - uint32_t addLid2 = addGid(f.dms, g.gid, g.bid2, Timestamp(0)); + uint32_t addLid2 = addGid(f.dms, g.gid, g.bid2, Timestamp(0), docSize); EXPECT_TRUE(1u == addLid2); EXPECT_TRUE(f.dms.remove(1u)); f.dms.removeComplete(1u); @@ -1749,10 +1771,10 @@ TEST("requireThatMoveWorks") EXPECT_EQUAL(1u, dms.getNumDocs()); EXPECT_EQUAL(0u, dms.getNumUsedLids()); - EXPECT_TRUE(assertPut(bucketId1, time1, 1u, gid1, dms)); + EXPECT_TRUE(assertPut(bucketId1, time1, docSize1, 1u, gid1, dms)); EXPECT_EQUAL(2u, dms.getNumDocs()); EXPECT_EQUAL(1u, dms.getNumUsedLids()); - EXPECT_TRUE(assertPut(bucketId2, time2, 2u, gid2, dms)); + EXPECT_TRUE(assertPut(bucketId2, time2, docSize2, 2u, gid2, dms)); EXPECT_EQUAL(3u, dms.getNumDocs()); EXPECT_EQUAL(2u, dms.getNumUsedLids()); EXPECT_TRUE(dms.getGid(1u, gid)); @@ -1899,7 +1921,8 @@ addLid(DocumentMetaStore &dms, uint32_t lid) GlobalId gid = createGid(lid); BucketId bucketId(gid.convertToBucketId()); bucketId.setUsedBits(numBucketBits); - uint32_t addedLid = addGid(dms, gid, bucketId, Timestamp(lid + timestampBias)); + uint32_t docSize = 1; + uint32_t addedLid = addGid(dms, gid, bucketId, Timestamp(lid + timestampBias), docSize); EXPECT_EQUAL(lid, addedLid); } diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp index ad1eb837c3d..18c92312446 100644 --- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp @@ -64,7 +64,7 @@ public: } virtual Result put(const GlobalId &, const BucketId &, const Timestamp &, - DocId) override + uint32_t, DocId) override { return Result(); } diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index b483b4ea8f5..c67b1c0a1c9 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -198,9 +198,11 @@ struct MyWorld { const document::GlobalId &gid = docId.getGlobalId(); typedef DocumentMetaStore::Result PutRes; document::BucketId bucketId(BucketFactory::getBucketId(docId)); + uint32_t docSize = 1; PutRes putRes(metaStore.put(gid, bucketId, Timestamp(0u), + docSize, i)); metaStore.setBucketState(bucketId, true); } diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp index 53758985ddd..f76f542adab 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp @@ -64,7 +64,8 @@ struct Fixture void put(vespalib::stringref docId, uint32_t lid) { bumpTimeStamp(); const GlobalId gid(toGid(docId)); - _dms->put(gid, toBucketId(gid), _timestamp, lid); + uint32_t docSize = 1; + _dms->put(gid, toBucketId(gid), _timestamp, docSize, lid); _dms->commit(); } diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp index ac505699eaf..8f94f382870 100644 --- a/searchcore/src/tests/proton/server/documentretriever_test.cpp +++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp @@ -273,7 +273,8 @@ struct Fixture { typedef DocumentMetaStore::Result Result; meta_store.constructFreeList(); Result inspect = meta_store.get().inspect(gid); - Result putRes(meta_store.get().put(gid, bucket_id, timestamp, inspect.getLid())); + uint32_t docSize = 1; + Result putRes(meta_store.get().put(gid, bucket_id, timestamp, docSize, inspect.getLid())); lid = putRes.getLid(); ASSERT_TRUE(putRes.ok()); schema::CollectionType ct = schema::SINGLE; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 5152f0d0be6..9c9905b70e7 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -492,10 +492,11 @@ DocumentMetaStore::Result DocumentMetaStore::put(const GlobalId &gid, const BucketId &bucketId, const Timestamp ×tamp, + uint32_t size, DocId lid) { Result res; - RawDocumentMetaData metaData(gid, bucketId, timestamp); + RawDocumentMetaData metaData(gid, bucketId, timestamp, size); KeyComp comp(metaData, _metaDataStore, *_gidCompare); TreeType::Iterator itr = _gidToLidMap.lowerBound(KeyComp::FIND_DOC_ID, comp); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index 0b4272da7a9..430c2113bf2 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -168,6 +168,7 @@ public: Result put(const GlobalId &gid, const BucketId &bucketId, const Timestamp ×tamp, + uint32_t size, DocId lid) override; bool updateMetaData(DocId lid, const BucketId &bucketId, diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/i_store.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_store.h index 74442a25599..803ae1dc3b4 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/i_store.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_store.h @@ -73,6 +73,7 @@ struct IStore virtual Result put(const GlobalId &gid, const BucketId &bucketId, const Timestamp ×tamp, + uint32_t size, DocId lid) = 0; /* diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h index 6e91e9a673c..35c0f14c6d2 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h @@ -19,17 +19,23 @@ struct RawDocumentMetaData using Timestamp = storage::spi::Timestamp; GlobalId _gid; uint8_t _bucketUsedBits; + uint8_t _sizeLow; + uint16_t _sizeHigh; Timestamp _timestamp; RawDocumentMetaData(void) : _gid(), _bucketUsedBits(BucketId::minNumBits), + _sizeLow(0), + _sizeHigh(0), _timestamp() { } - RawDocumentMetaData(const GlobalId &gid, const BucketId &bucketId, const Timestamp ×tamp) + RawDocumentMetaData(const GlobalId &gid, const BucketId &bucketId, const Timestamp ×tamp, uint32_t size) : _gid(gid), _bucketUsedBits(bucketId.getUsedBits()), + _sizeLow(size), + _sizeHigh(size >> 8), _timestamp(timestamp) { assert(bucketId.valid()); @@ -37,6 +43,10 @@ struct RawDocumentMetaData verId.setUsedBits(_bucketUsedBits); assert(bucketId.getRawId() == verId.getRawId() || bucketId.getRawId() == verId.getId()); + if (size >= (1u << 24)) { + _sizeLow = 0xff; + _sizeHigh = 0xffff; + } } bool operator<(const GlobalId &rhs) const { return _gid < rhs; } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index d7dd6238d20..c0bfcaf0024 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -570,7 +570,7 @@ void putMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id const DocumentOperation &op, bool is_removed_doc) { documentmetastore::IStore::Result putRes( meta_store.put(doc_id.getGlobalId(), - op.getBucketId(), op.getTimestamp(), op.getLid())); + op.getBucketId(), op.getTimestamp(), op.getSerializedDocSize(), op.getLid())); if (!putRes.ok()) { throw IllegalStateException( make_string("Could not put <lid, gid> pair for " diff --git a/searchcore/src/vespa/searchcore/proton/test/document.h b/searchcore/src/vespa/searchcore/proton/test/document.h index aacf67c45bd..dcde2e543bf 100644 --- a/searchcore/src/vespa/searchcore/proton/test/document.h +++ b/searchcore/src/vespa/searchcore/proton/test/document.h @@ -40,6 +40,7 @@ public: } search::DocumentIdT getLid() const { return _lid; } storage::spi::Timestamp getTimestamp() const { return _tstamp; } + uint32_t getSize() const { return 1; } }; typedef std::vector<Document> DocumentVector; diff --git a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h index 070ffc901a2..13b9d62bf02 100644 --- a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h @@ -66,8 +66,9 @@ struct DocumentMetaStoreObserver : public IDocumentMetaStore virtual Result put(const GlobalId &gid, const BucketId &bucketId, const Timestamp ×tamp, + uint32_t size, DocId lid) override { - return _store.put(gid, bucketId, timestamp, lid); + return _store.put(gid, bucketId, timestamp, size, lid); } virtual bool updateMetaData(DocId lid, const BucketId &bucketId, |