diff options
Diffstat (limited to 'memfilepersistence/src/tests/spi')
22 files changed, 0 insertions, 7763 deletions
diff --git a/memfilepersistence/src/tests/spi/.gitignore b/memfilepersistence/src/tests/spi/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/memfilepersistence/src/tests/spi/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/memfilepersistence/src/tests/spi/CMakeLists.txt b/memfilepersistence/src/tests/spi/CMakeLists.txt deleted file mode 100644 index 25c4acf2c32..00000000000 --- a/memfilepersistence/src/tests/spi/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(memfilepersistence_testspi - SOURCES - memfiletestutils.cpp - providerconformancetest.cpp - memfilev1serializertest.cpp - memfilev1verifiertest.cpp - basicoperationhandlertest.cpp - splitoperationhandlertest.cpp - joinoperationhandlertest.cpp - iteratorhandlertest.cpp - memfiletest.cpp - memcachetest.cpp - simplememfileiobuffertest.cpp - memfileautorepairtest.cpp - shared_data_location_tracker_test.cpp - buffered_file_writer_test.cpp - buffer_test.cpp - simulatedfailurefile.cpp - DEPENDS - memfilepersistence_testhelper - memfilepersistence -) diff --git a/memfilepersistence/src/tests/spi/basicoperationhandlertest.cpp b/memfilepersistence/src/tests/spi/basicoperationhandlertest.cpp deleted file mode 100644 index 28cc56aa44e..00000000000 --- a/memfilepersistence/src/tests/spi/basicoperationhandlertest.cpp +++ /dev/null @@ -1,743 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memfiletestutils.h" -#include "simulatedfailurefile.h" -#include "options_builder.h" -#include <vespa/document/fieldset/fieldsetrepo.h> -#include <vespa/document/fieldset/fieldsets.h> -#include <vespa/persistence/spi/test.h> -#include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/vdstestlib/cppunit/macros.h> - -using storage::spi::test::makeSpiBucket; - -namespace storage { -namespace memfile { -namespace { - spi::LoadType defaultLoadType(0, "default"); -} - -class BasicOperationHandlerTest : public SingleDiskMemFileTestUtils -{ - CPPUNIT_TEST_SUITE(BasicOperationHandlerTest); - CPPUNIT_TEST(testGetHeaderOnly); - CPPUNIT_TEST(testGetFieldFiltering); - CPPUNIT_TEST(testRemove); - CPPUNIT_TEST(testRemoveWithNonMatchingTimestamp); - CPPUNIT_TEST(testRemoveWithNonMatchingTimestampAlwaysPersist); - CPPUNIT_TEST(testRemoveForExistingRemoveSameTimestamp); - CPPUNIT_TEST(testRemoveForExistingRemoveNewTimestamp); - CPPUNIT_TEST(testRemoveForExistingRemoveNewTimestampAlwaysPersist); - CPPUNIT_TEST(testRemoveDocumentNotFound); - CPPUNIT_TEST(testRemoveDocumentNotFoundAlwaysPersist); - CPPUNIT_TEST(testRemoveExistingOlderDocumentVersion); - CPPUNIT_TEST(testPutSameTimestampAsRemove); - CPPUNIT_TEST(testUpdateBody); - CPPUNIT_TEST(testUpdateHeaderOnly); - CPPUNIT_TEST(testUpdateTimestampExists); - CPPUNIT_TEST(testUpdateForNonExistentDocWillFail); - CPPUNIT_TEST(testUpdateMayCreateDoc); - CPPUNIT_TEST(testRemoveEntry); - CPPUNIT_TEST(testEraseFromCacheOnFlushException); - CPPUNIT_TEST(testEraseFromCacheOnMaintainException); - CPPUNIT_TEST(testEraseFromCacheOnDeleteBucketException); - CPPUNIT_TEST(list_buckets_returns_empty_set_for_non_default_bucketspace); - CPPUNIT_TEST(get_modified_buckets_returns_empty_set_for_non_default_bucketspace); - CPPUNIT_TEST_SUITE_END(); - - void doTestRemoveDocumentNotFound( - OperationHandler::RemoveType persistRemove); - void doTestRemoveWithNonMatchingTimestamp( - OperationHandler::RemoveType persistRemove); - void doTestRemoveForExistingRemoveNewTimestamp( - OperationHandler::RemoveType persistRemove); -public: - void setupTestConfig(); - void testPutHeadersOnly(); - void testPutHeadersOnlyDocumentNotFound(); - void testPutHeadersOnlyTimestampNotFound(); - void testGetHeaderOnly(); - void testGetFieldFiltering(); - void testRemove(); - void testRemoveWithNonMatchingTimestamp(); - void testRemoveWithNonMatchingTimestampAlwaysPersist(); - void testRemoveForExistingRemoveSameTimestamp(); - void testRemoveForExistingRemoveNewTimestamp(); - void testRemoveForExistingRemoveNewTimestampAlwaysPersist(); - void testRemoveDocumentNotFound(); - void testRemoveDocumentNotFoundAlwaysPersist(); - void testRemoveExistingOlderDocumentVersion(); - void testPutSameTimestampAsRemove(); - void testUpdateBody(); - void testUpdateHeaderOnly(); - void testUpdateTimestampExists(); - void testUpdateForNonExistentDocWillFail(); - void testUpdateMayCreateDoc(); - void testRemoveEntry(); - void testEraseFromCacheOnFlushException(); - void testEraseFromCacheOnMaintainException(); - void testEraseFromCacheOnDeleteBucketException(); - void list_buckets_returns_empty_set_for_non_default_bucketspace(); - void get_modified_buckets_returns_empty_set_for_non_default_bucketspace(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(BasicOperationHandlerTest); - -/** - * Test that doing a header-only get gives back a document containing - * only the document header - */ -void -BasicOperationHandlerTest::testGetHeaderOnly() -{ - document::BucketId bucketId(16, 4); - - Document::SP doc(createRandomDocumentAtLocation(4)); - doc->setValue(doc->getField("hstringval"), document::StringFieldValue("hypnotoad")); - doc->setValue(doc->getField("headerval"), document::IntFieldValue(42)); - - doPut(doc, bucketId, Timestamp(4567), 0); - flush(bucketId); - - spi::GetResult reply = doGet(bucketId, doc->getId(), document::HeaderFields()); - - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, reply.getErrorCode()); - CPPUNIT_ASSERT(reply.hasDocument()); - CPPUNIT_ASSERT_EQUAL(std::string("headerval: 42\nhstringval: hypnotoad\n"), - stringifyFields(reply.getDocument())); - CPPUNIT_ASSERT_EQUAL( - size_t(1), - getPersistenceProvider().getMetrics().headerOnlyGets.getValue()); -} - -void -BasicOperationHandlerTest::testGetFieldFiltering() -{ - document::BucketId bucketId(16, 4); - Document::SP doc(createRandomDocumentAtLocation(4)); - doc->setValue(doc->getField("headerval"), document::IntFieldValue(42)); - doc->setValue(doc->getField("hstringval"), - document::StringFieldValue("groovy")); - - document::FieldSetRepo repo; - - doPut(doc, bucketId, Timestamp(4567), 0); - flush(bucketId); - spi::GetResult reply(doGet(bucketId, - doc->getId(), - *repo.parse(*getTypeRepo(), "testdoctype1:hstringval"))); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, reply.getErrorCode()); - CPPUNIT_ASSERT(reply.hasDocument()); - CPPUNIT_ASSERT_EQUAL(std::string("hstringval: groovy\n"), - stringifyFields(reply.getDocument())); - CPPUNIT_ASSERT_EQUAL( - size_t(1), - getPersistenceProvider().getMetrics().headerOnlyGets.getValue()); -} - -void -BasicOperationHandlerTest::testRemove() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - - document::Document::SP doc = doPut(4, Timestamp(1)); - - CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId, - doc->getId(), - Timestamp(2), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - env()._cache.clear(); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(*doc, *file->getDocument((*file)[0], ALL)); - - CPPUNIT_ASSERT_EQUAL(Timestamp(2), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT((*file)[1].deleted()); - CPPUNIT_ASSERT_EQUAL(DataLocation(0, 0), (*file)[1].getLocation(BODY)); - CPPUNIT_ASSERT_EQUAL((*file)[0].getLocation(HEADER), - (*file)[1].getLocation(HEADER)); -} - -/** - * Test that removing a document with a max timestamp for which there - * is no matching document does not add a remove slot to the memfile - */ -void -BasicOperationHandlerTest::doTestRemoveWithNonMatchingTimestamp( - OperationHandler::RemoveType persistRemove) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::Document::SP doc = doPut(4, Timestamp(1234)); - - CPPUNIT_ASSERT_EQUAL(false, doRemove(bucketId, - doc->getId(), - Timestamp(1233), - persistRemove)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL( - uint32_t(persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE - ? 2 : 1), - file->getSlotCount()); - - int i = 0; - if (persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE) { - CPPUNIT_ASSERT_EQUAL(Timestamp(1233), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT((*file)[0].deleted()); - CPPUNIT_ASSERT_EQUAL(DataLocation(0, 0), (*file)[0].getLocation(BODY)); - CPPUNIT_ASSERT((*file)[0].getLocation(HEADER) - != (*file)[1].getLocation(HEADER)); - CPPUNIT_ASSERT_EQUAL(doc->getId(), file->getDocumentId((*file)[0])); - ++i; - } - - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[i].getTimestamp()); - CPPUNIT_ASSERT(!(*file)[i].deleted()); - CPPUNIT_ASSERT(file->getDocument((*file)[i], ALL)->getValue("content").get()); -} - -/** - * Test that removing a document with a max timestamp for which there - * is no matching document does not add a remove slot to the memfile - */ -void -BasicOperationHandlerTest::testRemoveWithNonMatchingTimestamp() -{ - doTestRemoveWithNonMatchingTimestamp( - OperationHandler::PERSIST_REMOVE_IF_FOUND); -} - -void -BasicOperationHandlerTest::testRemoveWithNonMatchingTimestampAlwaysPersist() -{ - doTestRemoveWithNonMatchingTimestamp( - OperationHandler::ALWAYS_PERSIST_REMOVE); -} - -/** - * Test that doing a remove with a timestamp for which there already - * exists a remove does not add another remove slot - */ -void -BasicOperationHandlerTest::testRemoveForExistingRemoveSameTimestamp() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::Document::SP doc = doPut(4, Timestamp(1234)); - - CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId, - doc->getId(), - Timestamp(1235), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - CPPUNIT_ASSERT_EQUAL(false, doRemove(bucketId, - doc->getId(), - Timestamp(1235), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - // Should only be one remove entry still - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("content").get()); - - CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT((*file)[1].deleted()); -} - -void -BasicOperationHandlerTest::doTestRemoveForExistingRemoveNewTimestamp( - OperationHandler::RemoveType persistRemove) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::Document::SP doc = doPut(4, Timestamp(1234)); - - CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId, - doc->getId(), - Timestamp(1235), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - CPPUNIT_ASSERT_EQUAL(false, doRemove(bucketId, - doc->getId(), - Timestamp(1236), - persistRemove)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL( - uint32_t(persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE - ? 3 : 2), - file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("content").get()); - - CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT((*file)[1].deleted()); - - if (persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE) { - CPPUNIT_ASSERT_EQUAL(Timestamp(1236), (*file)[2].getTimestamp()); - CPPUNIT_ASSERT((*file)[2].deleted()); - } -} - -/** - * Test that doing a second remove with a newer timestamp does not add - * another remove slot when PERSIST_REMOVE_IF_FOUND is specified - */ -void -BasicOperationHandlerTest::testRemoveForExistingRemoveNewTimestamp() -{ - doTestRemoveForExistingRemoveNewTimestamp( - OperationHandler::PERSIST_REMOVE_IF_FOUND); -} - -void -BasicOperationHandlerTest::testRemoveForExistingRemoveNewTimestampAlwaysPersist() -{ - doTestRemoveForExistingRemoveNewTimestamp( - OperationHandler::ALWAYS_PERSIST_REMOVE); -} - -/** - * Test removing an older version of a document. Older version should be removed - * in-place without attempting to add a new slot (which would fail). - */ -void -BasicOperationHandlerTest::testRemoveExistingOlderDocumentVersion() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::Document::SP doc = doPut(4, Timestamp(1234)); - - CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId, - doc->getId(), - Timestamp(1235), - OperationHandler::ALWAYS_PERSIST_REMOVE)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId, - doc->getId(), - Timestamp(1234), - OperationHandler::ALWAYS_PERSIST_REMOVE)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - // Should now be two remove entries. - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(doc->getId(), file->getDocumentId((*file)[0])); - CPPUNIT_ASSERT((*file)[0].deleted()); - - CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(doc->getId(), file->getDocumentId((*file)[1])); - CPPUNIT_ASSERT((*file)[1].deleted()); -} - -void -BasicOperationHandlerTest::doTestRemoveDocumentNotFound( - OperationHandler::RemoveType persistRemove) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::DocumentId docId("userdoc:test:4:0"); - doPut(4, Timestamp(1234)); - - CPPUNIT_ASSERT_EQUAL(false, - doRemove(bucketId, - docId, - Timestamp(1235), - persistRemove)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL( - uint32_t(persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE - ? 2 : 1), - file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - if (persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE) { - CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT((*file)[1].deleted()); - CPPUNIT_ASSERT_EQUAL(docId, file->getDocumentId((*file)[1])); - } -/* TODO: Test this in service layer tests. - CPPUNIT_ASSERT_EQUAL( - uint64_t(1), - env()._metrics.remove[documentapi::LoadType::DEFAULT].notFound.getValue()); -*/ -} - -/** - * Test that removing a non-existing document when PERSIST_EXISTING_ONLY is - * specified does not add a remove entry - */ -void -BasicOperationHandlerTest::testRemoveDocumentNotFound() -{ - doTestRemoveDocumentNotFound( - OperationHandler::PERSIST_REMOVE_IF_FOUND); -} - -void -BasicOperationHandlerTest::testRemoveDocumentNotFoundAlwaysPersist() -{ - doTestRemoveDocumentNotFound( - OperationHandler::ALWAYS_PERSIST_REMOVE); -} - -void -BasicOperationHandlerTest::testPutSameTimestampAsRemove() -{ - document::BucketId bucketId(16, 4); - - document::Document::SP doc = doPut(4, Timestamp(1234)); - - CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId, - doc->getId(), - Timestamp(1235), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - - // Flush here to avoid put+remove being thrown away by duplicate timestamp - // exception evicting the cache and unpersisted changes. - flush(bucketId); - - doPut(4, Timestamp(1235)); - flush(bucketId); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("content").get()); - - CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT((*file)[1].deleted()); -} - -/** - * Test that updating body results in a new memfile slot containing - * an updated document - */ -void -BasicOperationHandlerTest::testUpdateBody() -{ - document::BucketId bucketId(16, 4); - document::StringFieldValue updateValue("foo"); - document::Document::SP doc = doPut(4, Timestamp(1234)); - document::Document originalDoc(*doc); - - document::DocumentUpdate::SP update = createBodyUpdate( - doc->getId(), updateValue); - - spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(5678)); - flush(bucketId); - CPPUNIT_ASSERT_EQUAL(1234, (int)result.getExistingTimestamp()); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("content").get()); - CPPUNIT_ASSERT_EQUAL(*(originalDoc.getValue("content")), - *file->getDocument((*file)[0], ALL)->getValue("content")); - - CPPUNIT_ASSERT_EQUAL(Timestamp(5678), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[1], ALL)->getValue("content").get()); - CPPUNIT_ASSERT_EQUAL(updateValue, - dynamic_cast<document::StringFieldValue&>( - *file->getDocument((*file)[1], ALL)->getValue( - "content"))); - CPPUNIT_ASSERT_EQUAL( - size_t(0), - getPersistenceProvider().getMetrics().headerOnlyUpdates.getValue()); -} - -void -BasicOperationHandlerTest::testUpdateHeaderOnly() -{ - document::BucketId bucketId(16, 4); - document::IntFieldValue updateValue(42); - document::Document::SP doc = doPut(4, Timestamp(1234)); - - document::DocumentUpdate::SP update = createHeaderUpdate( - doc->getId(), updateValue); - - spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(5678)); - flush(bucketId); - CPPUNIT_ASSERT_EQUAL(1234, (int)result.getExistingTimestamp()); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("headerval").get() == - NULL); - - CPPUNIT_ASSERT_EQUAL(Timestamp(5678), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[1], ALL)->getValue("headerval").get()); - CPPUNIT_ASSERT_EQUAL(updateValue, - dynamic_cast<document::IntFieldValue&>( - *file->getDocument((*file)[1], ALL)->getValue( - "headerval"))); - CPPUNIT_ASSERT_EQUAL( - size_t(1), - getPersistenceProvider().getMetrics().headerOnlyUpdates.getValue()); -} - -void -BasicOperationHandlerTest::testUpdateTimestampExists() -{ - document::BucketId bucketId(16, 4); - document::IntFieldValue updateValue(42); - document::Document::SP doc = doPut(4, Timestamp(1234)); - - document::DocumentUpdate::SP update = createHeaderUpdate( - doc->getId(), updateValue); - - spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(1234)); - flush(bucketId); - CPPUNIT_ASSERT_EQUAL(spi::Result::TRANSIENT_ERROR, result.getErrorCode()); -} - -void -BasicOperationHandlerTest::testUpdateForNonExistentDocWillFail() -{ - document::BucketId bucketId(16, 4); - document::IntFieldValue updateValue(42); - Timestamp timestamp(5678); - - // Is there an easier way to get a DocumentId? - document::Document::UP doc( - createRandomDocumentAtLocation(4, timestamp.getTime())); - const DocumentId& documentId = doc->getId(); - - document::DocumentUpdate::SP update = createHeaderUpdate( - documentId, updateValue); - - spi::UpdateResult result = doUpdate(bucketId, update, timestamp); - flush(bucketId); - CPPUNIT_ASSERT_EQUAL(0, (int)result.getExistingTimestamp()); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), file->getSlotCount()); -} - -void -BasicOperationHandlerTest::testUpdateMayCreateDoc() -{ - document::BucketId bucketId(16, 4); - document::IntFieldValue updateValue(42); - Timestamp timestamp(5678); - - // Is there an easier way to get a DocumentId? - document::Document::UP doc( - createRandomDocumentAtLocation(4, timestamp.getTime())); - const DocumentId& documentId = doc->getId(); - - document::DocumentUpdate::SP update = createHeaderUpdate( - documentId, updateValue); - update->setCreateIfNonExistent(true); - - spi::UpdateResult result = doUpdate(bucketId, update, timestamp); - flush(bucketId); - CPPUNIT_ASSERT_EQUAL(timestamp.getTime(), - (uint64_t)result.getExistingTimestamp()); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(timestamp, (*file)[0].getTimestamp()); - - auto headerval = file->getDocument((*file)[0], ALL)->getValue("headerval"); - CPPUNIT_ASSERT(headerval.get() != nullptr); - CPPUNIT_ASSERT_EQUAL(updateValue, - dynamic_cast<document::IntFieldValue&>(*headerval)); -} - -void -BasicOperationHandlerTest::testRemoveEntry() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - - doPut(4, Timestamp(1234)); - Document::SP doc = doPut(4, Timestamp(2345)); - doPut(4, Timestamp(3456)); - - getPersistenceProvider().removeEntry(makeSpiBucket(bucketId), spi::Timestamp(1234), context); - getPersistenceProvider().removeEntry(makeSpiBucket(bucketId), spi::Timestamp(3456), context); - flush(bucketId); - - memfile::MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(2345), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(*doc, *file->getDocument((*file)[0], ALL)); -} - -void -BasicOperationHandlerTest::setupTestConfig() -{ - using MemFileConfig = vespa::config::storage::StorMemfilepersistenceConfig; - using MemFileConfigBuilder - = vespa::config::storage::StorMemfilepersistenceConfigBuilder; - MemFileConfigBuilder builder( - *env().acquireConfigReadLock().memFilePersistenceConfig()); - builder.minimumFileMetaSlots = 2; - builder.minimumFileHeaderBlockSize = 3000; - auto newConfig = std::unique_ptr<MemFileConfig>(new MemFileConfig(builder)); - env().acquireConfigWriteLock().setMemFilePersistenceConfig( - std::move(newConfig)); -} - -void -BasicOperationHandlerTest::testEraseFromCacheOnFlushException() -{ - document::BucketId bucketId(16, 4); - - setupTestConfig(); - - document::Document::SP doc( - createRandomDocumentAtLocation(4, 2345, 1024, 1024)); - doPut(doc, bucketId, Timestamp(2345)); - flush(bucketId); - // Must throw out cache to re-create lazyfile - env()._cache.clear(); - - env()._lazyFileFactory = - std::unique_ptr<Environment::LazyFileFactory>( - new SimulatedFailureLazyFile::Factory); - - // Try partial write, followed by full rewrite - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < i+1; ++j) { - document::Document::SP doc2( - createRandomDocumentAtLocation(4, 4000 + j, 1500, 1500)); - doPut(doc2, bucketId, Timestamp(4000 + j)); - } - spi::Result result = flush(bucketId); - CPPUNIT_ASSERT(result.hasError()); - CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O write") - != vespalib::string::npos); - - CPPUNIT_ASSERT(!env()._cache.contains(bucketId)); - - // Check that we still have first persisted put - memfile::MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(2345), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(*doc, *file->getDocument((*file)[0], ALL)); - } -} - -void -BasicOperationHandlerTest::testEraseFromCacheOnMaintainException() -{ - document::BucketId bucketId(16, 4); - - setupTestConfig(); - - getFakeClock()._absoluteTime = framework::MicroSecTime(2000 * 1000000); - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options) - .revertTimePeriod(framework::MicroSecTime(100000ULL * 1000000)) - .build()); - // Put a doc twice to allow for revert time compaction to be done - document::Document::SP doc1( - createRandomDocumentAtLocation(4, 2345, 1024, 1024)); - document::Document::SP doc2( - createRandomDocumentAtLocation(4, 2345, 1024, 1024)); - doPut(doc1, bucketId, Timestamp(1000 * 1000000)); - doPut(doc2, bucketId, Timestamp(1500 * 1000000)); - flush(bucketId); - env()._cache.clear(); - - options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options) - .revertTimePeriod(framework::MicroSecTime(100ULL * 1000000)) - .build()); - - env()._lazyFileFactory = - std::unique_ptr<Environment::LazyFileFactory>( - new SimulatedFailureLazyFile::Factory); - - spi::Result result = getPersistenceProvider().maintain(makeSpiBucket(bucketId), spi::HIGH); - CPPUNIT_ASSERT(result.hasError()); - CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O write") - != vespalib::string::npos); - - CPPUNIT_ASSERT(!env()._cache.contains(bucketId)); - - // Check that we still have both persisted puts - memfile::MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1000 * 1000000), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(*doc1, *file->getDocument((*file)[0], ALL)); - CPPUNIT_ASSERT_EQUAL(Timestamp(1500 * 1000000), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(*doc2, *file->getDocument((*file)[1], ALL)); -} - -void -BasicOperationHandlerTest::testEraseFromCacheOnDeleteBucketException() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::Document::SP doc( - createRandomDocumentAtLocation(4, 2345, 1024, 1024)); - doPut(doc, bucketId, Timestamp(2345)); - flush(bucketId); - env()._cache.clear(); - - SimulatedFailureLazyFile::Factory* factory( - new SimulatedFailureLazyFile::Factory); - factory->setReadOpsBeforeFailure(0); - env()._lazyFileFactory = - std::unique_ptr<Environment::LazyFileFactory>(factory); - - // loadFile will fail - spi::Result result = getPersistenceProvider().deleteBucket(makeSpiBucket(bucketId), context); - CPPUNIT_ASSERT(result.hasError()); - CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O read") - != vespalib::string::npos); - - CPPUNIT_ASSERT(!env()._cache.contains(bucketId)); - -} - -void BasicOperationHandlerTest::list_buckets_returns_empty_set_for_non_default_bucketspace() { - document::BucketId bucket(16, 4); - doPut(createRandomDocumentAtLocation(4), bucket, Timestamp(4567), 0); - flush(bucket); - - auto buckets = getPersistenceProvider().listBuckets(document::FixedBucketSpaces::global_space(), spi::PartitionId(0)); - CPPUNIT_ASSERT_EQUAL(size_t(0), buckets.getList().size()); -} - -void BasicOperationHandlerTest::get_modified_buckets_returns_empty_set_for_non_default_bucketspace() { - env().addModifiedBucket(document::BucketId(16, 1234)); - auto buckets = getPersistenceProvider().getModifiedBuckets(document::FixedBucketSpaces::global_space()); - CPPUNIT_ASSERT_EQUAL(size_t(0), buckets.getList().size()); -} - -} - -} diff --git a/memfilepersistence/src/tests/spi/buffer_test.cpp b/memfilepersistence/src/tests/spi/buffer_test.cpp deleted file mode 100644 index bb31577bf17..00000000000 --- a/memfilepersistence/src/tests/spi/buffer_test.cpp +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/memfilepersistence/mapper/buffer.h> - -namespace storage { -namespace memfile { - -class BufferTest : public CppUnit::TestFixture -{ -public: - void getSizeReturnsInitiallyAllocatedSize(); - void getSizeReturnsUnAlignedSizeForMMappedAllocs(); - void resizeRetainsExistingDataWhenSizingUp(); - void resizeRetainsExistingDataWhenSizingDown(); - void bufferAddressIs512ByteAligned(); - - CPPUNIT_TEST_SUITE(BufferTest); - CPPUNIT_TEST(getSizeReturnsInitiallyAllocatedSize); - CPPUNIT_TEST(getSizeReturnsUnAlignedSizeForMMappedAllocs); - CPPUNIT_TEST(resizeRetainsExistingDataWhenSizingUp); - CPPUNIT_TEST(resizeRetainsExistingDataWhenSizingDown); - CPPUNIT_TEST(bufferAddressIs512ByteAligned); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(BufferTest); - -void -BufferTest::getSizeReturnsInitiallyAllocatedSize() -{ - Buffer buf(1234); - CPPUNIT_ASSERT_EQUAL(size_t(1234), buf.getSize()); -} - -void -BufferTest::getSizeReturnsUnAlignedSizeForMMappedAllocs() -{ - Buffer buf(vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE + 1); - CPPUNIT_ASSERT_EQUAL(size_t(vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE + 1), buf.getSize()); -} - -void -BufferTest::resizeRetainsExistingDataWhenSizingUp() -{ - std::string src = "hello world"; - Buffer buf(src.size()); - memcpy(buf.getBuffer(), src.data(), src.size()); - buf.resize(src.size() * 2); - CPPUNIT_ASSERT_EQUAL(src.size() * 2, buf.getSize()); - CPPUNIT_ASSERT_EQUAL(0, memcmp(buf.getBuffer(), src.data(), src.size())); -} - -void -BufferTest::resizeRetainsExistingDataWhenSizingDown() -{ - std::string src = "hello world"; - Buffer buf(src.size()); - memcpy(buf.getBuffer(), src.data(), src.size()); - buf.resize(src.size() / 2); - CPPUNIT_ASSERT_EQUAL(src.size() / 2, buf.getSize()); - CPPUNIT_ASSERT_EQUAL(0, memcmp(buf.getBuffer(), src.data(), src.size() / 2)); -} - -void -BufferTest::bufferAddressIs512ByteAligned() -{ - Buffer buf(32); - CPPUNIT_ASSERT(reinterpret_cast<size_t>(buf.getBuffer()) % 512 == 0); -} - -} // memfile -} // storage - diff --git a/memfilepersistence/src/tests/spi/buffered_file_writer_test.cpp b/memfilepersistence/src/tests/spi/buffered_file_writer_test.cpp deleted file mode 100644 index 36270335fda..00000000000 --- a/memfilepersistence/src/tests/spi/buffered_file_writer_test.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/memfilepersistence/mapper/bufferedfilewriter.h> -#include <vespa/memfilepersistence/mapper/buffer.h> -#include <vespa/vespalib/io/fileutil.h> - -namespace storage { -namespace memfile { - -class BufferedFileWriterTest : public CppUnit::TestFixture -{ -public: - void noImplicitFlushingWhenDestructing(); - - CPPUNIT_TEST_SUITE(BufferedFileWriterTest); - CPPUNIT_TEST(noImplicitFlushingWhenDestructing); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(BufferedFileWriterTest); - -namespace { - -// Partial mock of vespalib::File. Unfortunately, there's currently no -// base interface to implement so have to override a class that already has -// implementation code present. -class MockFile : public vespalib::File -{ -public: - bool _didWrite; - - MockFile(const std::string& filename) - : File(filename), - _didWrite(false) - { - } - - void open(int flags, bool autoCreateDirectories) override { - (void) flags; - (void) autoCreateDirectories; - // Don't do anything here to prevent us from actually opening a file - // on disk. - } - - off_t write(const void *buf, size_t bufsize, off_t offset) override { - (void) buf; - (void) bufsize; - (void) offset; - _didWrite = true; - return 0; - } -}; - -} - -void -BufferedFileWriterTest::noImplicitFlushingWhenDestructing() -{ - MockFile file("foo"); - { - Buffer buffer(1024); - BufferedFileWriter writer(file, buffer, buffer.getSize()); - // Do a buffered write. This fits well within the buffer and should - // consequently not be immediately written out to the backing file. - writer.write("blarg", 5); - // Escape scope without having flushed anything. - } - // Since BufferedFileWriter is meant to be used with O_DIRECT files, - // flushing just implies writing rather than syncing (this is a half truth - // since you still sync directories etc to ensure metadata is written, but - // this constrained assumption works fine in the context of this test). - CPPUNIT_ASSERT(!file._didWrite); -} - -} // memfile -} // storage - diff --git a/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp b/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp deleted file mode 100644 index 418ead076fd..00000000000 --- a/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp +++ /dev/null @@ -1,929 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h> -#include <tests/spi/memfiletestutils.h> -#include <tests/spi/simulatedfailurefile.h> -#include <tests/spi/options_builder.h> -#include <vespa/persistence/spi/test.h> -#include <vespa/document/select/parser.h> - -using storage::spi::test::makeSpiBucket; - -namespace storage { -namespace memfile { -namespace { - spi::LoadType defaultLoadType(0, "default"); -} - -class IteratorHandlerTest : public SingleDiskMemFileTestUtils -{ - CPPUNIT_TEST_SUITE(IteratorHandlerTest); - CPPUNIT_TEST(testCreateIterator); - CPPUNIT_TEST(testSomeSlotsRemovedBetweenInvocations); - CPPUNIT_TEST(testAllSlotsRemovedBetweenInvocations); - CPPUNIT_TEST(testIterateMetadataOnly); - CPPUNIT_TEST(testIterateHeadersOnly); - CPPUNIT_TEST(testIterateLargeDocument); - CPPUNIT_TEST(testDocumentsRemovedBetweenInvocations); - CPPUNIT_TEST(testUnrevertableRemoveBetweenInvocations); - CPPUNIT_TEST(testUnrevertableRemoveBetweenInvocationsIncludeRemoves); - CPPUNIT_TEST(testMatchTimestampRangeDocAltered); - CPPUNIT_TEST(testIterateAllVersions); - CPPUNIT_TEST(testFieldSetFiltering); - CPPUNIT_TEST(testIteratorInactiveOnException); - CPPUNIT_TEST(testDocsCachedBeforeDocumentSelection); - CPPUNIT_TEST(testTimestampRangeLimitedPrefetch); - CPPUNIT_TEST(testCachePrefetchRequirements); - CPPUNIT_TEST(testBucketEvictedFromCacheOnIterateException); - CPPUNIT_TEST_SUITE_END(); - -public: - void testCreateIterator(); - void testSomeSlotsRemovedBetweenInvocations(); - void testAllSlotsRemovedBetweenInvocations(); - void testIterateMetadataOnly(); - void testIterateHeadersOnly(); - void testIterateLargeDocument(); - void testDocumentsRemovedBetweenInvocations(); - void testUnrevertableRemoveBetweenInvocations(); - void testUnrevertableRemoveBetweenInvocationsIncludeRemoves(); - void testMatchTimestampRangeDocAltered(); - void testIterateAllVersions(); - void testFieldSetFiltering(); - void testIteratorInactiveOnException(); - void testDocsCachedBeforeDocumentSelection(); - void testTimestampRangeLimitedPrefetch(); - void testCachePrefetchRequirements(); - void testBucketEvictedFromCacheOnIterateException(); - - void setUp() override; - void tearDown() override; - - struct Chunk - { - std::vector<spi::DocEntry::UP> _entries; - }; - -private: - spi::Selection createSelection(const std::string& docSel) const; - - - spi::CreateIteratorResult create( - const spi::Bucket& b, - const spi::Selection& sel, - spi::IncludedVersions versions = spi::NEWEST_DOCUMENT_ONLY, - const document::FieldSet& fieldSet = document::AllFields()) - { - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - return getPersistenceProvider().createIterator(b, fieldSet, sel, - versions, context); - } - - typedef std::pair<Document::SP, spi::Timestamp> DocAndTimestamp; - - std::vector<DocAndTimestamp> feedDocs(size_t numDocs, - uint32_t minSize = 110, - uint32_t maxSize = 110); - - std::vector<Chunk> doIterate(spi::IteratorId id, - uint64_t maxByteSize, - size_t maxChunks = 0, - bool allowEmptyResult = false); - - void verifyDocs(const std::vector<DocAndTimestamp>& wanted, - const std::vector<IteratorHandlerTest::Chunk>& chunks, - const std::set<vespalib::string>& removes - = std::set<vespalib::string>()) const; - - void doTestUnrevertableRemoveBetweenInvocations(bool includeRemoves); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(IteratorHandlerTest); - -void -IteratorHandlerTest::setUp() -{ - SingleDiskMemFileTestUtils::setUp(); -} - -void -IteratorHandlerTest::tearDown() -{ - SingleDiskMemFileTestUtils::tearDown(); -} - -spi::Selection -IteratorHandlerTest::createSelection(const std::string& docSel) const -{ - return spi::Selection(spi::DocumentSelection(docSel)); -} - -void -IteratorHandlerTest::testCreateIterator() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 1234))); - - spi::CreateIteratorResult iter1(create(b, createSelection("true"))); - CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1), iter1.getIteratorId()); - - spi::CreateIteratorResult iter2(create(b, createSelection("true"))); - CPPUNIT_ASSERT_EQUAL(spi::IteratorId(2), iter2.getIteratorId()); -} - -std::vector<IteratorHandlerTest::Chunk> -IteratorHandlerTest::doIterate(spi::IteratorId id, - uint64_t maxByteSize, - size_t maxChunks, - bool allowEmptyResult) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - std::vector<Chunk> chunks; - - while (true) { - spi::IterateResult result(getPersistenceProvider().iterate( - id, maxByteSize, context)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT(result.getEntries().size() > 0 || allowEmptyResult); - - chunks.push_back(Chunk{result.steal_entries()}); - if (result.isCompleted() - || (maxChunks != 0 && chunks.size() >= maxChunks)) - { - break; - } - } - return chunks; -} - -namespace { - -size_t -getDocCount(const std::vector<IteratorHandlerTest::Chunk>& chunks) -{ - size_t count = 0; - for (size_t i=0; i<chunks.size(); ++i) { - count += chunks[i]._entries.size(); - } - return count; -} - -size_t -getRemoveEntryCount(const std::vector<spi::DocEntry::UP>& entries) -{ - size_t ret = 0; - for (size_t i = 0; i < entries.size(); ++i) { - if (entries[i]->isRemove()) { - ++ret; - } - } - return ret; -} - -struct DocEntryIndirectTimestampComparator -{ - bool operator()(const spi::DocEntry::UP& e1, - const spi::DocEntry::UP& e2) const - { - return e1->getTimestamp() < e2->getTimestamp(); - } -}; - -std::vector<spi::DocEntry::UP> -getEntriesFromChunks(const std::vector<IteratorHandlerTest::Chunk>& chunks) -{ - std::vector<spi::DocEntry::UP> ret; - for (size_t chunk = 0; chunk < chunks.size(); ++chunk) { - for (size_t i = 0; i < chunks[chunk]._entries.size(); ++i) { - ret.push_back(spi::DocEntry::UP(chunks[chunk]._entries[i]->clone())); - } - } - std::sort(ret.begin(), - ret.end(), - DocEntryIndirectTimestampComparator()); - return ret; -} - -const vespalib::LazyFile& -getFileHandle(const MemFile& mf1) -{ - return static_cast<const SimpleMemFileIOBuffer&>( - mf1.getMemFileIO()).getFileHandle(); -} - -const LoggingLazyFile& -getLoggerFile(const MemFile& file) -{ - return dynamic_cast<const LoggingLazyFile&>(getFileHandle(file)); -} - -} - -void -IteratorHandlerTest::verifyDocs(const std::vector<DocAndTimestamp>& wanted, - const std::vector<IteratorHandlerTest::Chunk>& chunks, - const std::set<vespalib::string>& removes) const -{ - std::vector<spi::DocEntry::UP> retrieved( - getEntriesFromChunks(chunks)); - size_t removeCount = getRemoveEntryCount(retrieved); - // Ensure that we've got the correct number of puts and removes - CPPUNIT_ASSERT_EQUAL(removes.size(), removeCount); - CPPUNIT_ASSERT_EQUAL(wanted.size(), retrieved.size() - removeCount); - - size_t wantedIdx = 0; - for (size_t i = 0; i < retrieved.size(); ++i) { - spi::DocEntry& entry(*retrieved[i]); - if (entry.getDocument() != 0) { - if (!(*wanted[wantedIdx].first == *entry.getDocument())) { - std::ostringstream ss; - ss << "Documents differ! Wanted:\n" - << wanted[wantedIdx].first->toString(true) - << "\n\nGot:\n" - << entry.getDocument()->toString(true); - CPPUNIT_FAIL(ss.str()); - } - CPPUNIT_ASSERT_EQUAL(wanted[wantedIdx].second, entry.getTimestamp()); - CPPUNIT_ASSERT_EQUAL(wanted[wantedIdx].first->serialize()->getLength() - + sizeof(spi::DocEntry), - size_t(entry.getSize())); - ++wantedIdx; - } else { - // Remove-entry - CPPUNIT_ASSERT(entry.getDocumentId() != 0); - CPPUNIT_ASSERT_EQUAL(entry.getDocumentId()->getSerializedSize() - + sizeof(spi::DocEntry), - size_t(entry.getSize())); - if (removes.find(entry.getDocumentId()->toString()) == removes.end()) { - std::ostringstream ss; - ss << "Got unexpected remove entry for document id " - << *entry.getDocumentId(); - CPPUNIT_FAIL(ss.str()); - } - } - } -} - -// Feed numDocs documents, starting from timestamp 1000 -std::vector<IteratorHandlerTest::DocAndTimestamp> -IteratorHandlerTest::feedDocs(size_t numDocs, - uint32_t minSize, - uint32_t maxSize) -{ - std::vector<DocAndTimestamp> docs; - for (uint32_t i = 0; i < numDocs; ++i) { - docs.push_back( - DocAndTimestamp( - doPut(4, - framework::MicroSecTime(1000 + i), - minSize, - maxSize), - spi::Timestamp(1000 + i))); - } - flush(document::BucketId(16, 4)); - return docs; -} - -void -IteratorHandlerTest::testSomeSlotsRemovedBetweenInvocations() -{ - std::vector<DocAndTimestamp> docs = feedDocs(100, 4096, 4096); - - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - spi::Selection sel(createSelection("true")); - - spi::CreateIteratorResult iter(create(b, sel)); - CPPUNIT_ASSERT(env()._cache.contains(b.getBucketId())); - - std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 10000, 25); - CPPUNIT_ASSERT_EQUAL(size_t(25), chunks.size()); - - { - MemFilePtr file(getMemFile(b.getBucketId())); - - for (int i = 0 ; i < 2; ++i) { - const MemSlot* slot = file->getSlotWithId(docs.front().first->getId()); - CPPUNIT_ASSERT(slot != 0); - file->removeSlot(*slot); - docs.erase(docs.begin()); - } - file->flushToDisk(); - } - - std::vector<Chunk> chunks2 = doIterate(iter.getIteratorId(), 10000); - CPPUNIT_ASSERT_EQUAL(size_t(24), chunks2.size()); - std::move(chunks2.begin(), chunks2.end(), std::back_inserter(chunks)); - - verifyDocs(docs, chunks); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - - // Bucket should not be evicted from cache during normal operation. - CPPUNIT_ASSERT(env()._cache.contains(b.getBucketId())); -} - -void -IteratorHandlerTest::testAllSlotsRemovedBetweenInvocations() -{ - std::vector<DocAndTimestamp> docs = feedDocs(100, 4096, 4096); - - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - spi::Selection sel(createSelection("true")); - - spi::CreateIteratorResult iter(create(b, sel)); - - std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 1, 25); - CPPUNIT_ASSERT_EQUAL(size_t(25), chunks.size()); - - { - MemFilePtr file(getMemFile(b.getBucketId())); - - for (int i = 0 ; i < 75; ++i) { - const MemSlot* slot = file->getSlotWithId(docs[i].first->getId()); - CPPUNIT_ASSERT(slot != 0); - file->removeSlot(*slot); - } - file->flushToDisk(); - docs.erase(docs.begin(), docs.begin() + 75); - } - - std::vector<Chunk> chunks2 = doIterate(iter.getIteratorId(), 1, 0, true); - CPPUNIT_ASSERT_EQUAL(size_t(0), getDocCount(chunks2)); - verifyDocs(docs, chunks); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testIterateMetadataOnly() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - std::vector<DocAndTimestamp> docs = feedDocs(10); - - CPPUNIT_ASSERT( - doUnrevertableRemove(b.getBucketId(), - docs[docs.size() - 2].first->getId(), - Timestamp(1008))); - - CPPUNIT_ASSERT( - doRemove(b.getBucketId(), - docs[docs.size() - 1].first->getId(), - framework::MicroSecTime(3001), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - - flush(b.getBucketId()); - - spi::Selection sel(createSelection("true")); - spi::CreateIteratorResult iter( - create(b, sel, spi::NEWEST_DOCUMENT_OR_REMOVE, document::NoFields())); - - std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 4096); - std::vector<spi::DocEntry::UP> entries = getEntriesFromChunks(chunks); - CPPUNIT_ASSERT_EQUAL(docs.size(), entries.size()); - std::vector<DocAndTimestamp>::const_iterator docIter( - docs.begin()); - for (size_t i = 0; i < entries.size(); ++i, ++docIter) { - const spi::DocEntry& entry = *entries[i]; - - CPPUNIT_ASSERT(entry.getDocument() == 0); - CPPUNIT_ASSERT(entry.getDocumentId() == 0); - if (i == 9) { - CPPUNIT_ASSERT(entry.isRemove()); - CPPUNIT_ASSERT_EQUAL(spi::Timestamp(3001), entry.getTimestamp()); - } else if (i == 8) { - CPPUNIT_ASSERT(entry.isRemove()); - CPPUNIT_ASSERT_EQUAL(spi::Timestamp(1008), entry.getTimestamp()); - } else { - CPPUNIT_ASSERT(!entry.isRemove()); - CPPUNIT_ASSERT_EQUAL(docIter->second, entry.getTimestamp()); - } - } - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testIterateHeadersOnly() -{ - std::vector<DocAndTimestamp> docs = feedDocs(20); - // Remove all bodies. - for (size_t i = 0; i < docs.size(); ++i) { - clearBody(*docs[i].first); - } - - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - spi::Selection sel(createSelection("true")); - - spi::CreateIteratorResult iter(create(b, sel, spi::NEWEST_DOCUMENT_ONLY, - document::HeaderFields())); - - std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 1024); - verifyDocs(docs, chunks); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testIterateLargeDocument() -{ - std::vector<DocAndTimestamp> docs = feedDocs(10, 10000, 10000); - std::vector<DocAndTimestamp> largedoc; - largedoc.push_back(docs.back()); - - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - spi::Selection sel(createSelection("true")); - - spi::CreateIteratorResult iter(create(b, sel)); - - std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 100, 1); - verifyDocs(largedoc, chunks); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testDocumentsRemovedBetweenInvocations() -{ - int docCount = 100; - std::vector<DocAndTimestamp> docs = feedDocs(docCount); - - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - spi::Selection sel(createSelection("true")); - - spi::CreateIteratorResult iter(create(b, sel)); - - std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 1, 25); - CPPUNIT_ASSERT_EQUAL(size_t(25), chunks.size()); - - // Remove a subset of the documents. We should still get all the - // original documents from the iterator, assuming no compactions. - std::vector<DocumentId> removedDocs; - std::vector<DocAndTimestamp> nonRemovedDocs; - for (int i = 0; i < docCount; ++i) { - if (i % 3 == 0) { - removedDocs.push_back(docs[i].first->getId()); - CPPUNIT_ASSERT(doRemove(b.getBucketId(), - removedDocs.back(), - framework::MicroSecTime(2000 + i), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - } else { - nonRemovedDocs.push_back(docs[i]); - } - } - flush(b.getBucketId()); - - std::vector<Chunk> chunks2 = doIterate(iter.getIteratorId(), 1); - CPPUNIT_ASSERT_EQUAL(size_t(75), chunks2.size()); - std::move(chunks2.begin(), chunks2.end(), std::back_inserter(chunks)); - - verifyDocs(docs, chunks); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::doTestUnrevertableRemoveBetweenInvocations(bool includeRemoves) -{ - int docCount = 100; - std::vector<DocAndTimestamp> docs = feedDocs(docCount); - - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - spi::Selection sel(createSelection("true")); - spi::CreateIteratorResult iter( - create(b, sel, - includeRemoves ? - spi::NEWEST_DOCUMENT_OR_REMOVE : spi::NEWEST_DOCUMENT_ONLY)); - - std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 1, 25); - CPPUNIT_ASSERT_EQUAL(size_t(25), chunks.size()); - - // Remove a subset of the documents unrevertably. - std::vector<DocumentId> removedDocs; - std::vector<DocAndTimestamp> nonRemovedDocs; - for (int i = 0; i < docCount - 25; ++i) { - if (i < 10) { - removedDocs.push_back(docs[i].first->getId()); - CPPUNIT_ASSERT( - doUnrevertableRemove(b.getBucketId(), - removedDocs.back(), - Timestamp(1000+i))); - } else { - nonRemovedDocs.push_back(docs[i]); - } - } - flush(b.getBucketId()); - - std::vector<Chunk> chunks2 = doIterate(iter.getIteratorId(), 1); - std::vector<spi::DocEntry::UP> entries = getEntriesFromChunks(chunks2); - if (!includeRemoves) { - CPPUNIT_ASSERT_EQUAL(nonRemovedDocs.size(), chunks2.size()); - verifyDocs(nonRemovedDocs, chunks2); - } else { - CPPUNIT_ASSERT_EQUAL(size_t(75), entries.size()); - for (int i = 0; i < docCount - 25; ++i) { - spi::DocEntry& entry(*entries[i]); - if (i < 10) { - CPPUNIT_ASSERT(entry.isRemove()); - } else { - CPPUNIT_ASSERT(!entry.isRemove()); - } - } - } - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testUnrevertableRemoveBetweenInvocations() -{ - doTestUnrevertableRemoveBetweenInvocations(false); -} - -void -IteratorHandlerTest::testUnrevertableRemoveBetweenInvocationsIncludeRemoves() -{ - doTestUnrevertableRemoveBetweenInvocations(true); -} - -void -IteratorHandlerTest::testMatchTimestampRangeDocAltered() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::StringFieldValue updateValue1("update1"); - document::StringFieldValue updateValue2("update2"); - - Document::SP originalDoc = doPut(4, Timestamp(1234)); - - { - document::DocumentUpdate::SP update = createBodyUpdate( - originalDoc->getId(), updateValue1); - - spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(2345)); - CPPUNIT_ASSERT_EQUAL(1234, (int)result.getExistingTimestamp()); - } - - { - document::DocumentUpdate::SP update = createBodyUpdate( - originalDoc->getId(), updateValue2); - - spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(3456)); - CPPUNIT_ASSERT_EQUAL(2345, (int)result.getExistingTimestamp()); - } - - CPPUNIT_ASSERT( - doRemove(bucketId, - originalDoc->getId(), - Timestamp(4567), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - flush(bucketId); - - spi::Bucket b(makeSpiBucket(bucketId)); - - { - spi::Selection sel(createSelection("true")); - sel.setFromTimestamp(spi::Timestamp(0)); - sel.setToTimestamp(spi::Timestamp(10)); - spi::CreateIteratorResult iter(create(b, sel)); - - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 4096, context)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT_EQUAL(size_t(0), result.getEntries().size()); - CPPUNIT_ASSERT(result.isCompleted()); - - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - } - - { - spi::Selection sel(createSelection("true")); - sel.setFromTimestamp(spi::Timestamp(10000)); - sel.setToTimestamp(spi::Timestamp(20000)); - spi::CreateIteratorResult iter(create(b, sel)); - - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 4096, context)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT_EQUAL(size_t(0), result.getEntries().size()); - CPPUNIT_ASSERT(result.isCompleted()); - - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - } - - { - spi::Selection sel(createSelection("true")); - sel.setFromTimestamp(spi::Timestamp(0)); - sel.setToTimestamp(spi::Timestamp(1234)); - spi::CreateIteratorResult iter(create(b, sel)); - - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 4096, context)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT_EQUAL(size_t(1), result.getEntries().size()); - CPPUNIT_ASSERT(result.isCompleted()); - - const Document& receivedDoc(*result.getEntries()[0]->getDocument()); - if (!(*originalDoc == receivedDoc)) { - std::ostringstream ss; - ss << "Documents differ! Wanted:\n" - << originalDoc->toString(true) - << "\n\nGot:\n" - << receivedDoc.toString(true); - CPPUNIT_FAIL(ss.str()); - } - - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - } - - { - spi::Selection sel(createSelection("true")); - sel.setFromTimestamp(spi::Timestamp(0)); - sel.setToTimestamp(spi::Timestamp(2345)); - spi::CreateIteratorResult iter(create(b, sel)); - - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 4096, context)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT_EQUAL(size_t(1), result.getEntries().size()); - CPPUNIT_ASSERT(result.isCompleted()); - - const Document& receivedDoc(*result.getEntries()[0]->getDocument()); - CPPUNIT_ASSERT(receivedDoc.getValue("content").get()); - CPPUNIT_ASSERT_EQUAL(updateValue1, - dynamic_cast<document::StringFieldValue&>( - *receivedDoc.getValue( - "content"))); - - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - } - - { - spi::Selection sel(createSelection("true")); - sel.setFromTimestamp(spi::Timestamp(0)); - sel.setToTimestamp(spi::Timestamp(3456)); - spi::CreateIteratorResult iter(create(b, sel)); - - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 4096, context)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT_EQUAL(size_t(1), result.getEntries().size()); - CPPUNIT_ASSERT(result.isCompleted()); - - const Document& receivedDoc(*result.getEntries()[0]->getDocument()); - CPPUNIT_ASSERT(receivedDoc.getValue("content").get()); - CPPUNIT_ASSERT_EQUAL(updateValue2, - dynamic_cast<document::StringFieldValue&>( - *receivedDoc.getValue( - "content"))); - - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - } -} - -void -IteratorHandlerTest::testIterateAllVersions() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - std::vector<DocAndTimestamp> docs; - - Document::SP originalDoc(createRandomDocumentAtLocation( - 4, 1001, 110, 110)); - - doPut(originalDoc, framework::MicroSecTime(1001), 0); - - document::StringFieldValue updateValue1("update1"); - { - document::DocumentUpdate::SP update = createBodyUpdate( - originalDoc->getId(), updateValue1); - - spi::UpdateResult result = doUpdate(b.getBucketId(), update, Timestamp(2345)); - CPPUNIT_ASSERT_EQUAL(1001, (int)result.getExistingTimestamp()); - } - flush(b.getBucketId()); - - Document::SP updatedDoc(new Document(*originalDoc)); - updatedDoc->setValue("content", document::StringFieldValue("update1")); - docs.push_back(DocAndTimestamp(originalDoc, spi::Timestamp(1001))); - docs.push_back(DocAndTimestamp(updatedDoc, spi::Timestamp(2345))); - - spi::Selection sel(createSelection("true")); - spi::CreateIteratorResult iter(create(b, sel, spi::ALL_VERSIONS)); - - std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 4096); - verifyDocs(docs, chunks); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testFieldSetFiltering() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - Document::SP doc(createRandomDocumentAtLocation( - 4, 1001, 110, 110)); - doc->setValue(doc->getField("headerval"), document::IntFieldValue(42)); - doc->setValue(doc->getField("hstringval"), - document::StringFieldValue("groovy, baby!")); - doc->setValue(doc->getField("content"), - document::StringFieldValue("fancy content")); - doPut(doc, framework::MicroSecTime(1001), 0); - flush(b.getBucketId()); - - document::FieldSetRepo repo; - spi::Selection sel(createSelection("true")); - spi::CreateIteratorResult iter( - create(b, sel, spi::NEWEST_DOCUMENT_ONLY, - *repo.parse(*getTypeRepo(), "testdoctype1:hstringval,content"))); - std::vector<spi::DocEntry::UP> entries( - getEntriesFromChunks(doIterate(iter.getIteratorId(), 4096))); - CPPUNIT_ASSERT_EQUAL(size_t(1), entries.size()); - CPPUNIT_ASSERT_EQUAL(std::string("content: fancy content\n" - "hstringval: groovy, baby!\n"), - stringifyFields(*entries[0]->getDocument())); -} - -void -IteratorHandlerTest::testIteratorInactiveOnException() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - feedDocs(10); - - env()._cache.clear(); - - simulateIoErrorsForSubsequentlyOpenedFiles(IoErrors().afterReads(1)); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - spi::CreateIteratorResult iter(create(b, createSelection("true"))); - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 100000, context)); - CPPUNIT_ASSERT(result.hasError()); - // Check that iterator is marked as inactive - const SharedIteratorHandlerState& state( - getPersistenceProvider().getIteratorHandler().getState()); - CPPUNIT_ASSERT(state._iterators.find(iter.getIteratorId().getValue()) - != state._iterators.end()); - CPPUNIT_ASSERT(state._iterators.find(iter.getIteratorId().getValue()) - ->second.isActive() == false); - - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testDocsCachedBeforeDocumentSelection() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - std::vector<DocAndTimestamp> docs = feedDocs(100, 4096, 4096); - - env()._cache.clear(); - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options).maximumReadThroughGap(1024*1024).build()); - env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>( - new LoggingLazyFile::Factory()); - - spi::Selection sel(createSelection("id.user=4")); - spi::CreateIteratorResult iter(create(b, sel, spi::NEWEST_DOCUMENT_ONLY, - document::BodyFields())); - - std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 4096); - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - { - MemFilePtr file(getMemFile(b.getBucketId())); - // Should have 3 read ops; metadata, (precached) headers and bodies - CPPUNIT_ASSERT_EQUAL(size_t(3), - getLoggerFile(*file).operations.size()); - } -} - -void -IteratorHandlerTest::testTimestampRangeLimitedPrefetch() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - // Feed docs with timestamp range [1000, 1100) - feedDocs(100, 4096, 4096); - - env()._cache.clear(); - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options).maximumReadThroughGap(512).build()); - env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>( - new LoggingLazyFile::Factory()); - - spi::Selection sel(createSelection("id.user=4")); - sel.setFromTimestamp(spi::Timestamp(1050)); - sel.setToTimestamp(spi::Timestamp(1059)); - spi::CreateIteratorResult iter(create(b, sel, spi::NEWEST_DOCUMENT_ONLY, - document::BodyFields())); - std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 4096); - CPPUNIT_ASSERT_EQUAL(size_t(10), getDocCount(chunks)); - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - // Iterate over all slots, ensuring that only those that fall within the - // timestamp range have actually been cached. - { - MemFilePtr file(getMemFile(b.getBucketId())); - // Should have 3 read ops; metadata, (precached) headers and bodies - CPPUNIT_ASSERT_EQUAL(size_t(3), - getLoggerFile(*file).operations.size()); - for (size_t i = 0; i < file->getSlotCount(); ++i) { - const MemSlot& slot((*file)[i]); - if (slot.getTimestamp() >= Timestamp(1050) - && slot.getTimestamp() <= Timestamp(1059)) - { - CPPUNIT_ASSERT(file->partAvailable(slot, HEADER)); - CPPUNIT_ASSERT(file->partAvailable(slot, BODY)); - } else { - CPPUNIT_ASSERT(!file->partAvailable(slot, HEADER)); - CPPUNIT_ASSERT(!file->partAvailable(slot, BODY)); - } - } - } -} - -void -IteratorHandlerTest::testCachePrefetchRequirements() -{ - document::select::Parser parser( - env().repo(), env()._bucketFactory); - { - // No prefetch required. - // NOTE: since stuff like id.user=1234 won't work, we have to handle - // that explicitly in createIterator based on the assumption that a - // non-empty document selection at _least_ requires header to be read. - std::unique_ptr<document::select::Node> sel( - parser.parse("true")); - CachePrefetchRequirements req( - CachePrefetchRequirements::createFromSelection(env().repo(), - *sel)); - CPPUNIT_ASSERT(!req.isHeaderPrefetchRequired()); - CPPUNIT_ASSERT(!req.isBodyPrefetchRequired()); - } - - { - // Header prefetch required. - std::unique_ptr<document::select::Node> sel( - parser.parse("testdoctype1.hstringval='blarg'")); - CachePrefetchRequirements req( - CachePrefetchRequirements::createFromSelection(env().repo(), - *sel)); - CPPUNIT_ASSERT(req.isHeaderPrefetchRequired()); - CPPUNIT_ASSERT(!req.isBodyPrefetchRequired()); - } - - { - // Body prefetch required. - std::unique_ptr<document::select::Node> sel( - parser.parse("testdoctype1.content='foobar'")); - CachePrefetchRequirements req( - CachePrefetchRequirements::createFromSelection(env().repo(), - *sel)); - CPPUNIT_ASSERT(!req.isHeaderPrefetchRequired()); - CPPUNIT_ASSERT(req.isBodyPrefetchRequired()); - } -} - -void -IteratorHandlerTest::testBucketEvictedFromCacheOnIterateException() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - feedDocs(10); - env()._cache.clear(); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - spi::CreateIteratorResult iter(create(b, createSelection("true"))); - simulateIoErrorsForSubsequentlyOpenedFiles(IoErrors().afterReads(1)); - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 100000, context)); - CPPUNIT_ASSERT(result.hasError()); - - // This test is actually a bit disingenuous since calling iterate will - // implicitly invoke maintain() on an IO exception, which will subsequently - // evict the bucket due to the exception happening again in its context. - CPPUNIT_ASSERT(!env()._cache.contains(b.getBucketId())); -} - -} -} diff --git a/memfilepersistence/src/tests/spi/joinoperationhandlertest.cpp b/memfilepersistence/src/tests/spi/joinoperationhandlertest.cpp deleted file mode 100644 index 2a9395767af..00000000000 --- a/memfilepersistence/src/tests/spi/joinoperationhandlertest.cpp +++ /dev/null @@ -1,504 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memfiletestutils.h" -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/persistence/spi/test.h> - -using document::DocumentType; -using storage::spi::test::makeSpiBucket; - -namespace storage { -namespace memfile { -namespace { - spi::LoadType defaultLoadType(0, "default"); -} - -class JoinOperationHandlerTest : public MemFileTestUtils -{ - CPPUNIT_TEST_SUITE(JoinOperationHandlerTest); - CPPUNIT_TEST(testSimple); - CPPUNIT_TEST(testTargetExists); - CPPUNIT_TEST(testTargetWithOverlap); - CPPUNIT_TEST(testMultiDisk); - CPPUNIT_TEST(testMultiDiskFlushed); - CPPUNIT_TEST(testInternalJoin); - CPPUNIT_TEST(testInternalJoinDiskFull); - CPPUNIT_TEST(testTargetIoWriteExceptionEvictsTargetFromCache); - CPPUNIT_TEST(test1stSourceIoReadExceptionEvictsSourceFromCache); - CPPUNIT_TEST(test2ndSourceExceptionEvictsExistingTargetFromCache); - CPPUNIT_TEST_SUITE_END(); - -public: - void testSimple(); - void testTargetExists(); - void testTargetWithOverlap(); - void testMultiDisk(); - void testMultiDiskFlushed(); - void testInternalJoin(); - void testInternalJoinDiskFull(); - void testTargetIoWriteExceptionEvictsTargetFromCache(); - void test1stSourceIoReadExceptionEvictsSourceFromCache(); - void test2ndSourceExceptionEvictsExistingTargetFromCache(); - - void insertDocumentInBucket(uint64_t location, - Timestamp timestamp, - document::BucketId bucket); - -private: - void feedSingleDisk(); - void feedMultiDisk(); - std::string getStandardMemFileStatus(uint32_t disk = 0); - - spi::Result doJoin(const document::BucketId to, - const document::BucketId from1, - const document::BucketId from2); -}; - -namespace { - -document::BucketId TARGET = document::BucketId(15, 4); -document::BucketId SOURCE1 = document::BucketId(16, 4); -document::BucketId SOURCE2 = document::BucketId(16, (uint64_t)4 | ((uint64_t)1 << 15)); -} - -CPPUNIT_TEST_SUITE_REGISTRATION(JoinOperationHandlerTest); - -void -JoinOperationHandlerTest::feedSingleDisk() -{ - for (uint32_t i = 0; i < 100; i++) { - std::ostringstream ost; - ost << "userdoc:storage_test:1234:" << i; - const DocumentType& type( - *getTypeRepo()->getDocumentType("testdoctype1")); - document::Document::SP doc( - new document::Document(type, document::DocumentId(ost.str()))); - - document::BucketId bucket( - getBucketIdFactory().getBucketId(doc->getId())); - bucket.setUsedBits(33); - doPut(doc, Timestamp(1000 + i), 0, 33); - flush(bucket); - } -} - -void -JoinOperationHandlerTest::feedMultiDisk() -{ - for (uint32_t i = 0; i < 100; i += 2) { - doPutOnDisk(7, 4 | (1 << 15), Timestamp(1000 + i)); - } - flush(SOURCE2); - - for (uint32_t i = 1; i < 100; i += 2) { - doPutOnDisk(4, 4, Timestamp(1000 + i)); - } - flush(SOURCE1); - - { - MemFilePtr file(getMemFile(SOURCE1, 4)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(4, (int)file->getDisk()); - } - - { - MemFilePtr file(getMemFile(SOURCE2, 7)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(7, (int)file->getDisk()); - } -} - -std::string -JoinOperationHandlerTest::getStandardMemFileStatus(uint32_t disk) -{ - std::ostringstream ost; - - ost << getMemFileStatus(TARGET, disk) << "\n" - << getMemFileStatus(SOURCE1, disk ) << "\n" - << getMemFileStatus(SOURCE2, disk) << "\n"; - - return ost.str(); -} - -void -JoinOperationHandlerTest::insertDocumentInBucket( - uint64_t location, - Timestamp timestamp, - document::BucketId bucket) -{ - Document::SP doc( - createRandomDocumentAtLocation( - location, timestamp.getTime(), 100, 100)); - doPut(doc, bucket, timestamp); -} - -spi::Result -JoinOperationHandlerTest::doJoin(const document::BucketId to, - const document::BucketId from1, - const document::BucketId from2) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - return getPersistenceProvider().join( - makeSpiBucket(from1), - makeSpiBucket(from2), - makeSpiBucket(to), - context); -} - -void -JoinOperationHandlerTest::testSimple() -{ - setupDisks(1); - feedSingleDisk(); - - { - MemFilePtr file(getMemFile(document::BucketId(33, 1234))); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - } - - { - MemFilePtr file(getMemFile(document::BucketId(33, (uint64_t)1234 | ((uint64_t)1 << 32)))); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - } - - spi::Result result = - doJoin(document::BucketId(32, 1234), - document::BucketId(33, 1234), - document::BucketId(33, (uint64_t)1234 | ((uint64_t)1 << 32))); - - { - MemFilePtr file(getMemFile(document::BucketId(32, (uint64_t)1234))); - CPPUNIT_ASSERT_EQUAL(100, (int)file->getSlotCount()); - CPPUNIT_ASSERT(!file->slotsAltered()); - } -} - -void -JoinOperationHandlerTest::testTargetExists() -{ - setupDisks(1); - - for (uint32_t i = 0; i < 100; i += 2) { - doPut(4 | (1 << 15), Timestamp(1000 + i)); - } - flush(SOURCE2); - - for (uint32_t i = 1; i < 100; i += 2) { - doPut(4, Timestamp(1000 + i)); - } - flush(SOURCE1); - - for (uint32_t i = 0; i < 100; i++) { - uint32_t location = 4; - if (i % 2 == 0) { - location |= (1 << 15); - } - - insertDocumentInBucket(location, Timestamp(500 + i), TARGET); - } - flush(TARGET); - - doJoin(TARGET, SOURCE1, SOURCE2); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x3c00000000000004): 200,0\n" - "BucketId(0x4000000000000004): 0,0\n" - "BucketId(0x4000000000008004): 0,0\n"), - getStandardMemFileStatus()); -} - -void -JoinOperationHandlerTest::testTargetWithOverlap() -{ - setupDisks(1); - - for (uint32_t i = 0; i < 100; i += 2) { - doPut(4 | (1 << 15), Timestamp(1000 + i)); - } - flush(SOURCE2); - - for (uint32_t i = 1; i < 100; i += 2) { - doPut(4, Timestamp(1000 + i)); - } - flush(SOURCE1); - - for (uint32_t i = 0; i < 100; i++) { - uint32_t location = 4; - if (i % 2 == 0) { - location |= (1 << 15); - } - - insertDocumentInBucket(location, Timestamp(950 + i), TARGET); - } - flush(TARGET); - - doJoin(TARGET, SOURCE1, SOURCE2); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x3c00000000000004): 150,0\n" - "BucketId(0x4000000000000004): 0,0\n" - "BucketId(0x4000000000008004): 0,0\n"), - getStandardMemFileStatus()); -} - -void -JoinOperationHandlerTest::testMultiDisk() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(10); - feedMultiDisk(); - - getPersistenceProvider().join(makeSpiBucket(SOURCE2, spi::PartitionId(7)), - makeSpiBucket(SOURCE1, spi::PartitionId(4)), - makeSpiBucket(TARGET, spi::PartitionId(3)), - context); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x3c00000000000004): 100,3\n" - "BucketId(0x4000000000000004): 0,0\n" - "BucketId(0x4000000000008004): 0,0\n"), - getStandardMemFileStatus()); -} - -void -JoinOperationHandlerTest::testMultiDiskFlushed() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(10); - feedMultiDisk(); - - // Flush everything to disk, to check that we can join even - // if it's not in cache before. - env()._cache.flushDirtyEntries(); - env()._cache.clear(); - - getPersistenceProvider().join(makeSpiBucket(SOURCE2, spi::PartitionId(7)), - makeSpiBucket(SOURCE1, spi::PartitionId(4)), - makeSpiBucket(TARGET, spi::PartitionId(3)), - context); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x3c00000000000004): 100,3\n" - "BucketId(0x4000000000000004): 0,3\n" - "BucketId(0x4000000000008004): 0,3\n"), - getStandardMemFileStatus(3)); -} - -void -JoinOperationHandlerTest::testInternalJoin() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(10); - - for (uint32_t i = 4; i < 6; i++) { - for (uint32_t j = 0; j < 10; j++) { - uint32_t location = 4; - doPutOnDisk(i, location, Timestamp(i * 1000 + j)); - } - flush(document::BucketId(16, 4), i); - env()._cache.clear(); - } - - std::string fileName1 = - env().calculatePathInDir(SOURCE1, (*env()._mountPoints)[4]); - std::string fileName2 = - env().calculatePathInDir(SOURCE1, (*env()._mountPoints)[5]); - - CPPUNIT_ASSERT(vespalib::stat(fileName1).get()); - vespalib::FileInfo::UP file2(vespalib::stat(fileName2)); - - CPPUNIT_ASSERT(file2.get()); - CPPUNIT_ASSERT(file2->_size > 0); - - PartitionMonitor* mon = env().getDirectory(5).getPartition().getMonitor(); - // Set disk under 80% full. Over 80%, we shouldn't move buckets to the target. - mon->setStatOncePolicy(); - mon->overrideRealStat(512, 100000, 50000); - CPPUNIT_ASSERT(!mon->isFull(0, .80f)); - - getPersistenceProvider().join(makeSpiBucket(SOURCE1, spi::PartitionId(4)), - makeSpiBucket(SOURCE1, spi::PartitionId(4)), - makeSpiBucket(SOURCE1, spi::PartitionId(5)), - context); - - env()._cache.clear(); - - CPPUNIT_ASSERT(!vespalib::stat(fileName1).get()); - CPPUNIT_ASSERT(vespalib::stat(fileName2).get()); -} - -void -JoinOperationHandlerTest::testInternalJoinDiskFull() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(10); - - for (uint32_t i = 4; i < 6; i++) { - for (uint32_t j = 0; j < 10; j++) { - uint32_t location = 4; - doPutOnDisk(i, location, Timestamp(i * 1000 + j)); - } - flush(document::BucketId(16, 4), i); - env()._cache.clear(); - } - - std::string fileName1 = - env().calculatePathInDir(SOURCE1, (*env()._mountPoints)[4]); - std::string fileName2 = - env().calculatePathInDir(SOURCE1, (*env()._mountPoints)[5]); - - CPPUNIT_ASSERT(vespalib::stat(fileName1).get()); - vespalib::FileInfo::UP file2(vespalib::stat(fileName2)); - - CPPUNIT_ASSERT(file2.get()); - CPPUNIT_ASSERT(file2->_size > 0); - - PartitionMonitor* mon = env().getDirectory(5).getPartition().getMonitor(); - // Set disk to 81% full. Over 80%, we shouldn't move buckets to the target. - mon->setStatOncePolicy(); - mon->overrideRealStat(512, 100000, 81000); - CPPUNIT_ASSERT(!mon->isFull()); - CPPUNIT_ASSERT(mon->isFull(0, .08f)); - - spi::Result result = - getPersistenceProvider().join(makeSpiBucket(SOURCE1, spi::PartitionId(4)), - makeSpiBucket(SOURCE1, spi::PartitionId(4)), - makeSpiBucket(SOURCE1, spi::PartitionId(5)), - context); - - CPPUNIT_ASSERT(result.hasError()); -} - -void -JoinOperationHandlerTest::testTargetIoWriteExceptionEvictsTargetFromCache() -{ - setupDisks(1); - feedSingleDisk(); - - document::BucketId src1(33, 1234); - document::BucketId src2(33, 1234ULL | (1ULL << 32)); - document::BucketId target(32, 1234); - - CPPUNIT_ASSERT(env()._cache.contains(src1)); - CPPUNIT_ASSERT(env()._cache.contains(src2)); - CPPUNIT_ASSERT(!env()._cache.contains(target)); - - // Reading existing (fully cached) files will go fine, but writing - // new file will not. - simulateIoErrorsForSubsequentlyOpenedFiles(); - - spi::Result result = doJoin(target, src1, src2); - CPPUNIT_ASSERT(result.hasError()); - CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O write") - != vespalib::string::npos); - - CPPUNIT_ASSERT(!env()._cache.contains(target)); - // NOTE: since we end up renaming src1 -> target during the first - // iteration of join, src1 will actually be empty. This should not - // matter since the service layer will query the bucket info for - // all these afterwards and will thus pick up on this automatically. - unSimulateIoErrorsForSubsequentlyOpenedFiles(); - { - MemFilePtr file(getMemFile(src1)); - CPPUNIT_ASSERT_EQUAL(0, (int)file->getSlotCount()); - CPPUNIT_ASSERT(!file->slotsAltered()); - } - { - MemFilePtr file(getMemFile(src2)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - CPPUNIT_ASSERT(!file->slotsAltered()); - } - { - MemFilePtr file(getMemFile(target)); - // Renamed from src1 - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - CPPUNIT_ASSERT(!file->slotsAltered()); - } -} - -void -JoinOperationHandlerTest::test1stSourceIoReadExceptionEvictsSourceFromCache() -{ - setupDisks(1); - feedSingleDisk(); - - document::BucketId src1(33, 1234); - document::BucketId src2(33, 1234ULL | (1ULL << 32)); - document::BucketId target(32, 1234); - - env()._cache.clear(); - // Allow for reading in initial metadata so that loadFile itself doesn't - // fail. This could otherwise cause a false negative since that happens - // during initial cache lookup on a cache miss, at which point any - // exception will always stop a file from being added to the cache. Here - // we want to test the case where a file has been successfully hoisted - // out of the cache initially. - simulateIoErrorsForSubsequentlyOpenedFiles(IoErrors().afterReads(1)); - - spi::Result result = doJoin(target, src1, src2); - CPPUNIT_ASSERT(result.hasError()); - CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O read") - != vespalib::string::npos); - - CPPUNIT_ASSERT(!env()._cache.contains(src1)); - CPPUNIT_ASSERT(!env()._cache.contains(src2)); - CPPUNIT_ASSERT(!env()._cache.contains(target)); -} - -/** - * It must be exception safe for any source bucket to throw an exception during - * processing. Otherwise the node will core due to cache sanity checks. - * - * See VESPA-674 for context. In this scenario, it was not possible to write - * to the target file when attempting to join in the 2nd source bucket due to - * the disk fill ratio exceeding configured limits. - */ -void -JoinOperationHandlerTest::test2ndSourceExceptionEvictsExistingTargetFromCache() -{ - setupDisks(1); - feedSingleDisk(); - - constexpr uint64_t location = 1234; - - document::BucketId src1(33, location); - document::BucketId src2(33, location | (1ULL << 32)); - document::BucketId target(32, location); - - // Ensure target file is _not_ empty so that copySlots is triggered for - // each source bucket (rather than just renaming the file, which does not - // invoke the file read/write paths). - insertDocumentInBucket(location, Timestamp(100000), target); - flush(target); - - env()._cache.clear(); - // File rewrites are buffered before ever reaching the failure simulation - // layer, so only 1 actual write is used to flush the target file after - // the first source file has been processed. Attempting to flush the writes - // for the second source file should fail with an exception. - simulateIoErrorsForSubsequentlyOpenedFiles( - IoErrors().afterReads(INT_MAX).afterWrites(1)); - - spi::Result result = doJoin(target, src1, src2); - CPPUNIT_ASSERT(result.hasError()); - CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O write") - != vespalib::string::npos); - - CPPUNIT_ASSERT(!env()._cache.contains(src1)); - CPPUNIT_ASSERT(!env()._cache.contains(src2)); - CPPUNIT_ASSERT(!env()._cache.contains(target)); -} - -} - -} diff --git a/memfilepersistence/src/tests/spi/logginglazyfile.h b/memfilepersistence/src/tests/spi/logginglazyfile.h deleted file mode 100644 index d525f85b165..00000000000 --- a/memfilepersistence/src/tests/spi/logginglazyfile.h +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/io/fileutil.h> -#include <sstream> - -namespace storage::memfile { - -class LoggingLazyFile : public vespalib::LazyFile { -public: - class Factory : public Environment::LazyFileFactory { - public: - vespalib::LazyFile::UP createFile(const std::string& fileName) const override { - return vespalib::LazyFile::UP( - new LoggingLazyFile(fileName, vespalib::File::DIRECTIO)); - } - }; - - enum OpType { - READ = 0, - WRITE - }; - - struct Entry { - OpType opType; - size_t bufsize; - off_t offset; - - std::string toString() const { - std::ostringstream ost; - ost << (opType == READ ? "Reading " : "Writing ") - << bufsize - << " bytes at " - << offset; - return ost.str(); - } - }; - - mutable std::vector<Entry> operations; - - LoggingLazyFile(const std::string& filename, int flags) - : LazyFile(filename, flags) {}; - - size_t getOperationCount() const { - return operations.size(); - } - - off_t write(const void *buf, size_t bufsize, off_t offset) override { - Entry e; - e.opType = WRITE; - e.bufsize = bufsize; - e.offset = offset; - - operations.push_back(e); - - return vespalib::LazyFile::write(buf, bufsize, offset); - } - - size_t read(void *buf, size_t bufsize, off_t offset) const override { - Entry e; - e.opType = READ; - e.bufsize = bufsize; - e.offset = offset; - - operations.push_back(e); - - return vespalib::LazyFile::read(buf, bufsize, offset); - } - - std::string toString() const { - std::ostringstream ost; - for (uint32_t i = 0; i < operations.size(); i++) { - ost << operations[i].toString() << "\n"; - } - - return ost.str(); - } - -}; - -} diff --git a/memfilepersistence/src/tests/spi/memcachetest.cpp b/memfilepersistence/src/tests/spi/memcachetest.cpp deleted file mode 100644 index 5e9f1a28225..00000000000 --- a/memfilepersistence/src/tests/spi/memcachetest.cpp +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/memfilepersistence/memfile/memfilecache.h> -#include <tests/spi/memfiletestutils.h> - - -namespace storage { -namespace memfile { - -class MemCacheTest : public SingleDiskMemFileTestUtils -{ - CPPUNIT_TEST_SUITE(MemCacheTest); - CPPUNIT_TEST(testSimpleLRU); - CPPUNIT_TEST(testCacheSize); - CPPUNIT_TEST(testEvictBody); - CPPUNIT_TEST(testEvictHeader); - CPPUNIT_TEST(testKeepBodyWhenLessThanOneFourth); - CPPUNIT_TEST(testComplexEviction); - CPPUNIT_TEST(testEraseEmptyOnReturn); - CPPUNIT_TEST(testDeleteDoesNotReAddMemoryUsage); - CPPUNIT_TEST(testEraseDoesNotReAddMemoryUsage); - CPPUNIT_TEST(testGetWithNoCreation); - CPPUNIT_TEST_SUITE_END(); - -public: - void testSimpleLRU(); - void testCacheSize(); - void testReduceCacheSizeCallback(); - void testReduceCacheSizeCallbackWhileActive(); - void testEvictBody(); - void testEvictHeader(); - void testKeepBodyWhenLessThanOneFourth(); - void testComplexEviction(); - void testEraseEmptyOnReturn(); - void testDeleteDoesNotReAddMemoryUsage(); - void testEraseDoesNotReAddMemoryUsage(); - void testGetWithNoCreation(); - -private: - framework::defaultimplementation::ComponentRegisterImpl::UP _register; - framework::Component::UP _component; - FakeClock::UP _clock; - std::unique_ptr<MemFilePersistenceMetrics> _metrics; - - std::unique_ptr<MemFileCache> _cache; - - void setSize(const document::BucketId& id, - uint64_t metaSize, - uint64_t headerSz = 0, - uint64_t bodySz = 0, - bool createIfNotInCache = true) - { - MemFilePtr file(_cache->get(id, env(), env().getDirectory(), - createIfNotInCache)); - CPPUNIT_ASSERT(file.get()); - - file->_cacheSizeOverride.metaSize = metaSize; - file->_cacheSizeOverride.headerSize = headerSz; - file->_cacheSizeOverride.bodySize = bodySz; - } - - std::string - getBucketStatus(uint32_t buckets) - { - std::ostringstream ost; - for (uint32_t i = 1; i < buckets + 1; i++) { - document::BucketId id(16, i); - ost << id << " "; - if (!_cache->contains(id)) { - ost << "<nil>\n"; - } else { - MemFilePtr file(_cache->get(id, env(), env().getDirectory())); - if (file->_cacheSizeOverride.bodySize > 0) { - ost << "body,"; - } - if (file->_cacheSizeOverride.headerSize > 0) { - ost << "header\n"; - } else { - ost << "meta only\n"; - } - } - } - - return ost.str(); - } - - uint64_t cacheSize() { - return _cache->size(); - } - - document::BucketId getLRU() { - return _cache->getLeastRecentlyUsedBucket()->_bid; - } - - void setCacheSize(uint64_t sz) { - MemFileCache::MemoryUsage usage; - usage.metaSize = sz / 3; - usage.headerSize = sz / 3; - usage.bodySize = sz - usage.metaSize - usage.headerSize; - - _cache->setCacheSize(usage); - } - - void stealMemory(uint64_t memToSteal) { - setCacheSize(_cache->getCacheSize() - memToSteal); - } - - void setup(uint64_t maxMemory) { - tearDown(); - _register.reset( - new framework::defaultimplementation::ComponentRegisterImpl); - _clock.reset(new FakeClock); - _register->setClock(*_clock); - _component.reset(new framework::Component(*_register, "testcomponent")); - _metrics.reset(new MemFilePersistenceMetrics(*_component)); - _cache.reset(new MemFileCache(*_register, _metrics->_cache)); - setCacheSize(maxMemory); - } - -public: - void tearDown() override { - _cache.reset(0); - _metrics.reset(0); - _component.reset(0); - _register.reset(0); - _clock.reset(0); - } -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemCacheTest); - -namespace { - FakeClock clock; -} - -void -MemCacheTest::testSimpleLRU() -{ - setup(2000); - - for (uint32_t i = 1; i < 4; i++) { - setSize(document::BucketId(16, i), 100); - } - - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1), getLRU()); - - setSize(document::BucketId(16, 1), 100); - - CPPUNIT_ASSERT_EQUAL(1UL, _cache->getMetrics().hits.getValue()); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 2), getLRU()); -} - -void -MemCacheTest::testCacheSize() -{ - setup(400); - - setSize(document::BucketId(16, 2), 100); - setSize(document::BucketId(16, 1), 150); - - CPPUNIT_ASSERT_EQUAL(0UL, _cache->getMetrics().hits.getValue()); - CPPUNIT_ASSERT_EQUAL(2UL, _cache->getMetrics().misses.getValue()); - - CPPUNIT_ASSERT_EQUAL(250ul, cacheSize()); - - setSize(document::BucketId(16, 1), 200); - - CPPUNIT_ASSERT_EQUAL(1UL, _cache->getMetrics().hits.getValue()); - CPPUNIT_ASSERT_EQUAL(2UL, _cache->getMetrics().misses.getValue()); - - CPPUNIT_ASSERT_EQUAL(300ul, cacheSize()); - - CPPUNIT_ASSERT(_cache->contains(document::BucketId(16, 2))); - CPPUNIT_ASSERT(_cache->contains(document::BucketId(16, 1))); - - setSize(document::BucketId(16, 1), 301); - - CPPUNIT_ASSERT_EQUAL(2UL, _cache->getMetrics().hits.getValue()); - CPPUNIT_ASSERT_EQUAL(2UL, _cache->getMetrics().misses.getValue()); - - CPPUNIT_ASSERT(!_cache->contains(document::BucketId(16, 2))); - CPPUNIT_ASSERT(_cache->contains(document::BucketId(16, 1))); - - _cache->clear(); - CPPUNIT_ASSERT_EQUAL(0ul, cacheSize()); -} - -void -MemCacheTest::testEvictBody() -{ - setup(1400); - - CPPUNIT_ASSERT_EQUAL(0UL, _cache->getMetrics().body_evictions.getValue()); - - setSize(BucketId(16, 1), 150, 100, 0); - setSize(BucketId(16, 2), 100, 100, 900); - - CPPUNIT_ASSERT_EQUAL(1350ul, cacheSize()); - - stealMemory(150); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) header\n" - "BucketId(0x4000000000000002) header\n"), - getBucketStatus(2)); - CPPUNIT_ASSERT_EQUAL(1UL, _cache->getMetrics().body_evictions.getValue()); -} - -void -MemCacheTest::testKeepBodyWhenLessThanOneFourth() -{ - setup(450); - - setSize(BucketId(16, 1), 150, 0, 0); - setSize(BucketId(16, 2), 100, 50, 50); - - stealMemory(150); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) <nil>\n" - "BucketId(0x4000000000000002) body,header\n"), - getBucketStatus(2)); -} - -void -MemCacheTest::testEvictHeader() -{ - setup(550); - - CPPUNIT_ASSERT_EQUAL(0UL, _cache->getMetrics().header_evictions.getValue()); - - setSize(BucketId(16, 1), 150, 0, 0); - setSize(BucketId(16, 2), 100, 200, 100); - - stealMemory(150); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) meta only\n" - "BucketId(0x4000000000000002) meta only\n"), - getBucketStatus(2)); - CPPUNIT_ASSERT_EQUAL(1UL, _cache->getMetrics().header_evictions.getValue()); -} - -#define ASSERT_CACHE_EVICTIONS(meta, header, body) \ - CPPUNIT_ASSERT_EQUAL(size_t(meta), _cache->getMetrics().body_evictions.getValue()); \ - CPPUNIT_ASSERT_EQUAL(size_t(header), _cache->getMetrics().header_evictions.getValue()); \ - CPPUNIT_ASSERT_EQUAL(size_t(body), _cache->getMetrics().meta_evictions.getValue()); - -void -MemCacheTest::testComplexEviction() -{ - setup(4200); - - setSize(BucketId(16, 1), 150, 0, 0); - setSize(BucketId(16, 2), 100, 200, 200); - setSize(BucketId(16, 3), 100, 200, 0); - setSize(BucketId(16, 4), 100, 400, 0); - setSize(BucketId(16, 5), 100, 200, 400); - setSize(BucketId(16, 6), 100, 200, 300); - setSize(BucketId(16, 7), 100, 0, 0); - setSize(BucketId(16, 8), 100, 200, 400); - setSize(BucketId(16, 9), 100, 200, 250); - - CPPUNIT_ASSERT_EQUAL(4100ul, cacheSize()); - - ASSERT_CACHE_EVICTIONS(0, 0, 0); - - stealMemory(600); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) meta only\n" - "BucketId(0x4000000000000002) header\n" - "BucketId(0x4000000000000003) header\n" - "BucketId(0x4000000000000004) header\n" - "BucketId(0x4000000000000005) header\n" - "BucketId(0x4000000000000006) body,header\n" - "BucketId(0x4000000000000007) meta only\n" - "BucketId(0x4000000000000008) body,header\n" - "BucketId(0x4000000000000009) body,header\n"), - getBucketStatus(9)); - - CPPUNIT_ASSERT_EQUAL(3500ul, cacheSize()); - - ASSERT_CACHE_EVICTIONS(2, 0, 0); - - stealMemory(500); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) meta only\n" - "BucketId(0x4000000000000002) meta only\n" - "BucketId(0x4000000000000003) meta only\n" - "BucketId(0x4000000000000004) header\n" - "BucketId(0x4000000000000005) header\n" - "BucketId(0x4000000000000006) body,header\n" - "BucketId(0x4000000000000007) meta only\n" - "BucketId(0x4000000000000008) body,header\n" - "BucketId(0x4000000000000009) body,header\n"), - getBucketStatus(9)); - - CPPUNIT_ASSERT_EQUAL(3100ul, cacheSize()); - - ASSERT_CACHE_EVICTIONS(2, 2, 0); - - stealMemory(1000); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) <nil>\n" - "BucketId(0x4000000000000002) meta only\n" - "BucketId(0x4000000000000003) meta only\n" - "BucketId(0x4000000000000004) meta only\n" - "BucketId(0x4000000000000005) meta only\n" - "BucketId(0x4000000000000006) header\n" - "BucketId(0x4000000000000007) meta only\n" - "BucketId(0x4000000000000008) body,header\n" - "BucketId(0x4000000000000009) body,header\n"), - getBucketStatus(9)); - - CPPUNIT_ASSERT_EQUAL(2050ul, cacheSize()); - - ASSERT_CACHE_EVICTIONS(3, 4, 1); - - stealMemory(1100); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) <nil>\n" - "BucketId(0x4000000000000002) <nil>\n" - "BucketId(0x4000000000000003) <nil>\n" - "BucketId(0x4000000000000004) <nil>\n" - "BucketId(0x4000000000000005) <nil>\n" - "BucketId(0x4000000000000006) <nil>\n" - "BucketId(0x4000000000000007) meta only\n" - "BucketId(0x4000000000000008) header\n" - "BucketId(0x4000000000000009) body,header\n"), - getBucketStatus(9)); - - CPPUNIT_ASSERT_EQUAL(950ul, cacheSize()); -} - -#undef ASSERT_CACHE_EVICTIONS - -void -MemCacheTest::testEraseEmptyOnReturn() -{ - setup(4200); - setSize(BucketId(16, 1), 0, 0, 0); - CPPUNIT_ASSERT(!_cache->contains(document::BucketId(16, 1))); -} - -void -MemCacheTest::testDeleteDoesNotReAddMemoryUsage() -{ - BucketId id(16, 1); - setup(1000); - setSize(id, 100, 200, 300); - CPPUNIT_ASSERT_EQUAL(600ul, cacheSize()); - { - MemFilePtr file(_cache->get(id, env(), env().getDirectory())); - file.deleteFile(); - } - CPPUNIT_ASSERT_EQUAL(0ul, cacheSize()); - -} - -void -MemCacheTest::testGetWithNoCreation() -{ - BucketId id(16, 1); - setup(1000); - setSize(id, 100, 200, 300, false); - CPPUNIT_ASSERT_EQUAL(0ul, cacheSize()); -} - - -void -MemCacheTest::testEraseDoesNotReAddMemoryUsage() -{ - BucketId id(16, 1); - setup(1000); - setSize(id, 100, 200, 300); - CPPUNIT_ASSERT_EQUAL(600ul, cacheSize()); - { - MemFilePtr file(_cache->get(id, env(), env().getDirectory())); - file.eraseFromCache(); - } - CPPUNIT_ASSERT_EQUAL(0ul, cacheSize()); - -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp b/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp deleted file mode 100644 index b7fbeba1649..00000000000 --- a/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp +++ /dev/null @@ -1,409 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/memfilepersistence/mapper/memfilemapper.h> -#include <vespa/memfilepersistence/mapper/memfile_v1_serializer.h> -#include <vespa/memfilepersistence/mapper/memfile_v1_verifier.h> -#include <tests/spi/memfiletestutils.h> -#include <vespa/persistence/spi/test.h> - -using storage::spi::test::makeSpiBucket; - -namespace storage { -namespace memfile { - -class MemFileAutoRepairTest : public SingleDiskMemFileTestUtils -{ -public: - void setUp() override; - void tearDown() override; - - void testFileMetadataCorruptionIsAutoRepaired(); - void testDocumentContentCorruptionIsAutoRepaired(); - void testCorruptionEvictsBucketFromCache(); - void testRepairFailureInMaintainEvictsBucketFromCache(); - void testZeroLengthFileIsDeleted(); - void testTruncatedBodyLocationIsAutoRepaired(); - void testTruncatedHeaderLocationIsAutoRepaired(); - void testTruncatedHeaderBlockIsAutoRepaired(); - - void corruptBodyBlock(); - - CPPUNIT_TEST_SUITE(MemFileAutoRepairTest); - CPPUNIT_TEST(testFileMetadataCorruptionIsAutoRepaired); - CPPUNIT_TEST(testDocumentContentCorruptionIsAutoRepaired); - CPPUNIT_TEST(testCorruptionEvictsBucketFromCache); - CPPUNIT_TEST(testRepairFailureInMaintainEvictsBucketFromCache); - CPPUNIT_TEST(testZeroLengthFileIsDeleted); - CPPUNIT_TEST(testTruncatedBodyLocationIsAutoRepaired); - CPPUNIT_TEST(testTruncatedHeaderLocationIsAutoRepaired); - CPPUNIT_TEST(testTruncatedHeaderBlockIsAutoRepaired); - CPPUNIT_TEST_SUITE_END(); - -private: - void assertDocumentIsSilentlyRemoved( - const document::BucketId& bucket, - const document::DocumentId& docId); - - void reconfigureMinimumHeaderBlockSize(uint32_t newMinSize); - - document::BucketId _bucket; - std::unique_ptr<FileSpecification> _file; - std::vector<document::DocumentId> _slotIds; -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemFileAutoRepairTest); - -namespace { - // A totall uncached memfile with content to use for verify testing - std::unique_ptr<MemFile> _memFile; - - // Clear old content. Create new file. Make sure nothing is cached. - void prepareBucket(SingleDiskMemFileTestUtils& util, - const FileSpecification& file) { - _memFile.reset(); - util.env()._cache.clear(); - vespalib::unlink(file.getPath()); - util.createTestBucket(file.getBucketId(), 0); - util.env()._cache.clear(); - _memFile.reset(new MemFile(file, util.env())); - _memFile->getMemFileIO().close(); - - } - - MetaSlot getSlot(uint32_t index) { - assert(_memFile.get()); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - MetaSlot result; - file.read(&result, sizeof(MetaSlot), - sizeof(Header) + sizeof(MetaSlot) * index); - return result; - } - - void setSlot(uint32_t index, MetaSlot slot, - bool updateFileChecksum = true) - { - (void)updateFileChecksum; - assert(_memFile.get()); - //if (updateFileChecksum) slot.updateFileChecksum(); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - file.write(&slot, sizeof(MetaSlot), - sizeof(Header) + sizeof(MetaSlot) * index); - } -} - -void -MemFileAutoRepairTest::setUp() -{ - SingleDiskMemFileTestUtils::setUp(); - _bucket = BucketId(16, 0xa); - createTestBucket(_bucket, 0); - - { - MemFilePtr memFilePtr(env()._cache.get(_bucket, env(), env().getDirectory())); - _file.reset(new FileSpecification(memFilePtr->getFile())); - CPPUNIT_ASSERT(memFilePtr->getSlotCount() >= 2); - for (size_t i = 0; i < memFilePtr->getSlotCount(); ++i) { - _slotIds.push_back(memFilePtr->getDocumentId((*memFilePtr)[i])); - } - } - env()._cache.clear(); -} - -void -MemFileAutoRepairTest::tearDown() -{ - _file.reset(0); - _memFile.reset(0); - SingleDiskMemFileTestUtils::tearDown(); -}; - -void -MemFileAutoRepairTest::testFileMetadataCorruptionIsAutoRepaired() -{ - // Test corruption detected in initial metadata load - prepareBucket(*this, *_file); - document::DocumentId id(_slotIds[1]); - MetaSlot slot(getSlot(1)); - CPPUNIT_ASSERT(slot._gid == id.getGlobalId()); // Sanity checking... - { - MetaSlot s(slot); - s.setTimestamp(Timestamp(40)); - setSlot(1, s); - } - - CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets()); - - // File not in cache; should be detected in initial load - spi::GetResult res(doGet(_bucket, id, document::AllFields())); - // FIXME: currently loadFile is silently fixing corruptions! - //CPPUNIT_ASSERT_EQUAL(spi::Result::TRANSIENT_ERROR, res.getErrorCode()); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res.getErrorCode()); - CPPUNIT_ASSERT(!res.hasDocument()); - - CPPUNIT_ASSERT_EQUAL(std::string("400000000000000a"), getModifiedBuckets()); - CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets()); - - // File should now have been repaired, so a subsequent get for - // the same document should just return an empty (but OK) result. - spi::GetResult res2(doGet(_bucket, id, document::AllFields())); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res2.getErrorCode()); - CPPUNIT_ASSERT(!res2.hasDocument()); - - CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets()); -} - -void -MemFileAutoRepairTest::corruptBodyBlock() -{ - CPPUNIT_ASSERT(!env()._cache.contains(_bucket)); - // Corrupt body block of slot 1 - MetaSlot slot(getSlot(1)); - { - MetaSlot s(slot); - s.setBodyPos(52); - s.setBodySize(18); - s.updateChecksum(); - setSlot(1, s); - } -} - -void -MemFileAutoRepairTest::testDocumentContentCorruptionIsAutoRepaired() -{ - // Corrupt body block - prepareBucket(*this, *_file); - document::DocumentId id(_slotIds[1]); - corruptBodyBlock(); - - CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets()); - - spi::GetResult res(doGet(_bucket, id, document::AllFields())); - CPPUNIT_ASSERT_EQUAL(spi::Result::TRANSIENT_ERROR, res.getErrorCode()); - CPPUNIT_ASSERT(!res.hasDocument()); - - CPPUNIT_ASSERT(!env()._cache.contains(_bucket)); - - CPPUNIT_ASSERT_EQUAL(std::string("400000000000000a"), getModifiedBuckets()); - CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets()); - - // File should now have been repaired, so a subsequent get for - // the same document should just return an empty (but OK) result. - spi::GetResult res2(doGet(_bucket, id, document::AllFields())); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res2.getErrorCode()); - CPPUNIT_ASSERT(!res2.hasDocument()); - - // File should now be in cache OK - CPPUNIT_ASSERT(env()._cache.contains(_bucket)); - CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets()); -} - -// Ideally we'd test this for each spi operation that accesses MemFiles, but -// they all use the same eviction+auto-repair logic... -void -MemFileAutoRepairTest::testCorruptionEvictsBucketFromCache() -{ - prepareBucket(*this, *_file); - corruptBodyBlock(); - - // Read slot 0 and shove file into cache - spi::GetResult res(doGet(_bucket, _slotIds[0], document::AllFields())); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res.getErrorCode()); - CPPUNIT_ASSERT(res.hasDocument()); - CPPUNIT_ASSERT(env()._cache.contains(_bucket)); - - spi::GetResult res2(doGet(_bucket, _slotIds[1], document::AllFields())); - CPPUNIT_ASSERT_EQUAL(spi::Result::TRANSIENT_ERROR, res2.getErrorCode()); - CPPUNIT_ASSERT(!res2.hasDocument()); - - // Out of the cache! Begone! Shoo! - CPPUNIT_ASSERT(!env()._cache.contains(_bucket)); - -} - -void -MemFileAutoRepairTest::testRepairFailureInMaintainEvictsBucketFromCache() -{ - prepareBucket(*this, *_file); - corruptBodyBlock(); - spi::Result result(getPersistenceProvider().maintain( - makeSpiBucket(_bucket), spi::HIGH)); - // File being successfully repaired does not constitute a failure of - // the maintain() call. - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - // It should, however, shove it out of the cache. - CPPUNIT_ASSERT(!env()._cache.contains(_bucket)); -} - -void -MemFileAutoRepairTest::testZeroLengthFileIsDeleted() -{ - // Completely truncate auto-created file - vespalib::LazyFile file(_file->getPath(), 0); - file.resize(0); - - // No way to deal with zero-length files aside from deleting them. - spi::Result result(getPersistenceProvider().maintain( - makeSpiBucket(_bucket), spi::HIGH)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT(!env()._cache.contains(_bucket)); - CPPUNIT_ASSERT(!vespalib::fileExists(_file->getPath())); -} - -namespace { - -uint32_t -alignDown(uint32_t value) -{ - uint32_t blocks = value / 512; - return blocks * 512; -}; - -FileInfo -fileInfoFromMemFile(const MemFilePtr& mf) -{ - auto& ioBuf(dynamic_cast<const SimpleMemFileIOBuffer&>( - mf->getMemFileIO())); - return ioBuf.getFileInfo(); -} - -} - -void -MemFileAutoRepairTest::assertDocumentIsSilentlyRemoved( - const document::BucketId& bucket, - const document::DocumentId& docId) -{ - // Corrupted (truncated) slot should be transparently removed during - // loadFile and it should be as if it was never there! - spi::Bucket spiBucket(makeSpiBucket(bucket)); - spi::GetResult res(doGet(spiBucket, docId, document::AllFields())); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res.getErrorCode()); - CPPUNIT_ASSERT(!res.hasDocument()); -} - -void -MemFileAutoRepairTest::testTruncatedBodyLocationIsAutoRepaired() -{ - document::BucketId bucket(16, 4); - document::Document::SP doc( - createRandomDocumentAtLocation(4, 1234, 1024, 1024)); - - doPut(doc, bucket, framework::MicroSecTime(1000)); - flush(bucket); - FileInfo fileInfo; - { - MemFilePtr mf(getMemFile(bucket)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), mf->getSlotCount()); - fileInfo = fileInfoFromMemFile(mf); - - const uint32_t bodyBlockStart( - sizeof(Header) - + fileInfo._metaDataListSize * sizeof(MetaSlot) - + fileInfo._headerBlockSize); - - vespalib::LazyFile file(mf->getFile().getPath(), 0); - uint32_t slotBodySize = (*mf)[0].getLocation(BODY)._size; - CPPUNIT_ASSERT(slotBodySize > 0); - // Align down to nearest sector alignment to avoid unrelated DirectIO - // checks to kick in. Since the body block is always aligned on a - // sector boundary, we know this cannot truncate into the header block. - file.resize(alignDown(bodyBlockStart + slotBodySize - 1)); - } - env()._cache.clear(); - assertDocumentIsSilentlyRemoved(bucket, doc->getId()); -} - -void -MemFileAutoRepairTest::testTruncatedHeaderLocationIsAutoRepaired() -{ - document::BucketId bucket(16, 4); - document::Document::SP doc( - createRandomDocumentAtLocation(4, 1234, 1024, 1024)); - // Ensure header has a bunch of data (see alignment comments below). - doc->setValue(doc->getField("hstringval"), - document::StringFieldValue(std::string(1024, 'A'))); - - doPut(doc, bucket, framework::MicroSecTime(1000)); - flush(bucket); - FileInfo fileInfo; - { - MemFilePtr mf(getMemFile(bucket)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), mf->getSlotCount()); - fileInfo = fileInfoFromMemFile(mf); - - const uint32_t headerBlockStart( - sizeof(Header) - + fileInfo._metaDataListSize * sizeof(MetaSlot)); - - vespalib::LazyFile file(mf->getFile().getPath(), 0); - uint32_t slotHeaderSize = (*mf)[0].getLocation(HEADER)._size; - CPPUNIT_ASSERT(slotHeaderSize > 0); - // Align down to nearest sector alignment to avoid unrelated DirectIO - // checks to kick in. The header block is not guaranteed to start on - // sector boundary, but we assume there is enough slack in the header - // section for the metadata slots themselves to be untouched since we - // have a minimum header size of 1024 for the doc in question. - file.resize(alignDown(headerBlockStart + slotHeaderSize - 1)); - } - env()._cache.clear(); - assertDocumentIsSilentlyRemoved(bucket, doc->getId()); -} - -void -MemFileAutoRepairTest::reconfigureMinimumHeaderBlockSize(uint32_t newMinSize) -{ - using MemFileConfig = vespa::config::storage::StorMemfilepersistenceConfig; - using MemFileConfigBuilder - = vespa::config::storage::StorMemfilepersistenceConfigBuilder; - MemFileConfigBuilder builder( - *env().acquireConfigReadLock().memFilePersistenceConfig()); - builder.minimumFileMetaSlots = 2; - builder.minimumFileHeaderBlockSize = newMinSize; - auto newConfig = std::unique_ptr<MemFileConfig>(new MemFileConfig(builder)); - env().acquireConfigWriteLock().setMemFilePersistenceConfig( - std::move(newConfig)); -} - -void -MemFileAutoRepairTest::testTruncatedHeaderBlockIsAutoRepaired() -{ - document::BucketId bucket(16, 4); - document::Document::SP doc( - createRandomDocumentAtLocation(4, 1234, 1, 1)); - // Ensure header block is large enough that free space is added to the end. - reconfigureMinimumHeaderBlockSize(8192); - // Add header field and remove randomly generated body field, ensuring - // we have no data to add to body field. This will prevent slot body - // location checking from detecting a header truncation. - doc->setValue(doc->getField("hstringval"), - document::StringFieldValue("foo")); - doc->remove(doc->getField("content")); - - doPut(doc, bucket, framework::MicroSecTime(1000)); - flush(bucket); - FileInfo fileInfo; - { - MemFilePtr mf(getMemFile(bucket)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), mf->getSlotCount()); - fileInfo = fileInfoFromMemFile(mf); - - const uint32_t headerBlockEnd( - sizeof(Header) - + fileInfo._metaDataListSize * sizeof(MetaSlot) - + fileInfo._headerBlockSize); - - vespalib::LazyFile file(mf->getFile().getPath(), 0); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), - (*mf)[0].getLocation(BODY)._size); // No body. - const auto headerLoc((*mf)[0].getLocation(HEADER)); - const uint32_t extent(headerLoc._pos + headerLoc._size); - // Make sure we don't intersect an existing slot range. - CPPUNIT_ASSERT(extent < alignDown(headerBlockEnd - 1)); - file.resize(alignDown(headerBlockEnd - 1)); - } - env()._cache.clear(); - assertDocumentIsSilentlyRemoved(bucket, doc->getId()); -} - -} -} diff --git a/memfilepersistence/src/tests/spi/memfiletest.cpp b/memfilepersistence/src/tests/spi/memfiletest.cpp deleted file mode 100644 index 019b20de2df..00000000000 --- a/memfilepersistence/src/tests/spi/memfiletest.cpp +++ /dev/null @@ -1,987 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/memfilepersistence/memfile/memfile.h> -#include <tests/spi/memfiletestutils.h> -#include <tests/spi/logginglazyfile.h> -#include <tests/spi/options_builder.h> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/memfilepersistence/memfile/memfilecompactor.h> -#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h> -#include <vespa/vespalib/util/exceptions.h> -#include <limits> - -namespace storage { -namespace memfile { - -struct MemFileTest : public SingleDiskMemFileTestUtils -{ - typedef MemFileCompactor::SlotList SlotList; - - /** - * Feed a document whose ID is deterministically generated from `seed` to - * bucket (16, 4) at time `timestamp`. - */ - document::DocumentId feedDocument( - uint64_t seed, - uint64_t timestamp, - uint32_t headerSize = 0, - uint32_t minBodySize = 10, - uint32_t maxBodySize = 100); - - /** - * Feed n instances of documents with the same ID to bucket (16, 4) using - * a timestamp range of [1000, 1000+n). - */ - void feedSameDocNTimes(uint32_t n); - - void setMaxDocumentVersionsOption(uint32_t n); - - std::vector<Types::Timestamp> compactWithVersionLimit(uint32_t maxVersions); - - void testCompactRemoveDoublePut(); - void testCompactPutRemove(); - void testCompactGidCollision(); - void testCompactGidCollisionAndNot(); - void testCompactWithMemFile(); - void testCompactCombined(); - void testCompactDifferentPuts(); - void testNoCompactionWhenDocumentVersionsWithinLimit(); - void testCompactWhenDocumentVersionsExceedLimit(); - void testCompactLimit1KeepsNewestVersionOnly(); - void testCompactionOptionsArePropagatedFromConfig(); - void testZeroDocumentVersionConfigIsCorrected(); - void testResizeToFreeSpace(); - void testNoFileWriteOnNoOpCompaction(); - void testCacheSize(); - void testClearCache(); - void testGetSlotsByTimestamp(); - void testCacheInconsistentSlot(); - void testEnsureCached(); - void testAddSlotWhenDiskFull(); - void testGetSerializedSize(); - void testGetBucketInfo(); - void testCopySlotsPreservesLocationSharing(); - void testFlushingToNonExistingFileAlwaysRunsCompaction(); - void testOrderDocSchemeDocumentsCanBeAddedToFile(); - - CPPUNIT_TEST_SUITE(MemFileTest); - CPPUNIT_TEST(testCompactRemoveDoublePut); - CPPUNIT_TEST(testCompactPutRemove); - CPPUNIT_TEST(testCompactGidCollision); - CPPUNIT_TEST(testCompactGidCollisionAndNot); - CPPUNIT_TEST(testCompactWithMemFile); - CPPUNIT_TEST(testCompactCombined); - CPPUNIT_TEST(testCompactDifferentPuts); - CPPUNIT_TEST(testNoCompactionWhenDocumentVersionsWithinLimit); - CPPUNIT_TEST(testCompactWhenDocumentVersionsExceedLimit); - CPPUNIT_TEST(testCompactLimit1KeepsNewestVersionOnly); - CPPUNIT_TEST(testCompactionOptionsArePropagatedFromConfig); - CPPUNIT_TEST(testZeroDocumentVersionConfigIsCorrected); - CPPUNIT_TEST(testNoFileWriteOnNoOpCompaction); - CPPUNIT_TEST(testCacheSize); - CPPUNIT_TEST(testClearCache); - CPPUNIT_TEST(testGetSlotsByTimestamp); - CPPUNIT_TEST(testEnsureCached); - CPPUNIT_TEST(testResizeToFreeSpace); - CPPUNIT_TEST(testAddSlotWhenDiskFull); - CPPUNIT_TEST(testGetSerializedSize); - CPPUNIT_TEST(testGetBucketInfo); - CPPUNIT_TEST(testCopySlotsPreservesLocationSharing); - CPPUNIT_TEST(testFlushingToNonExistingFileAlwaysRunsCompaction); - CPPUNIT_TEST(testOrderDocSchemeDocumentsCanBeAddedToFile); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemFileTest); - -/** - * Slots should actually be the same pointer. Use this assert to do correct - * check, and still print content of slots on failure. - */ -#define ASSERT_SLOT_EQUAL(slotptra, slotptrb) \ -{ \ - CPPUNIT_ASSERT(slotptra != 0); \ - CPPUNIT_ASSERT(slotptrb != 0); \ - std::ostringstream slotdiff; \ - slotdiff << "Expected: " << *slotptra << ", but got " << *slotptrb; \ - CPPUNIT_ASSERT_EQUAL_MSG(slotdiff.str(), slotptra, slotptrb); \ -} - -namespace { - -framework::MicroSecTime sec(uint64_t n) { - return framework::MicroSecTime(n * 1000000ULL); -} - -/** - * Utility functions for tests to call to do compacting, such that the - * tests themselves are not bound to the current interface. - * - * Also, this function translates second time to microsecond time. - */ -MemFileTest::SlotList getSlotsToRemove( - const MemFile& file, uint64_t currentTime, - uint64_t revertTime, uint64_t keepRemoveTime) -{ - MemFileCompactor compactor( - sec(currentTime), - CompactionOptions() - .maxDocumentVersions( - std::numeric_limits<uint32_t>::max()) - .revertTimePeriod(sec(revertTime)) - .keepRemoveTimePeriod(sec(keepRemoveTime))); - return compactor.getSlotsToRemove(file); -} - -class AutoFlush -{ -public: - AutoFlush(MemFilePtr& ptr) : _ptr(ptr) {} - ~AutoFlush() { _ptr->flushToDisk(); } -private: - MemFilePtr& _ptr; -}; - -} - -document::DocumentId -MemFileTest::feedDocument( - uint64_t seed, - uint64_t timestamp, - uint32_t headerSize, - uint32_t minDocSize, - uint32_t maxDocSize) { - document::Document::SP doc(createRandomDocumentAtLocation( - 4, seed, minDocSize, maxDocSize)); - - if (headerSize > 0) { - std::string val(headerSize, 'A'); - doc->setValue(doc->getField("hstringval"), - document::StringFieldValue(val)); - } - - doPut(doc, - document::BucketId(16, 4), - Timestamp(timestamp * 1000000)); - - return doc->getId(); -} - -void -MemFileTest::feedSameDocNTimes(uint32_t n) -{ - for (uint32_t i = 0; i < n; ++i) { - feedDocument(1234, 1000 + i); - } -} - -void -MemFileTest::setMaxDocumentVersionsOption(uint32_t n) -{ - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options) - .maxDocumentVersions(n) - .build()); -} - -void -MemFileTest::testCacheSize() -{ - // Feed some puts - for (uint32_t i = 0; i < 4; i++) { - feedDocument(1234 * (i % 2), 1000 + 200 * i); - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - - CPPUNIT_ASSERT(file->getCacheSize().sum() > 0); -} - -void -MemFileTest::testClearCache() -{ - // Feed some puts - for (uint32_t i = 0; i < 4; i++) { - feedDocument(1234 * (i % 2), 1000 + 200 * i); - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - file->flushToDisk(); - - CPPUNIT_ASSERT(file->getCacheSize().bodySize > 0); - CPPUNIT_ASSERT(file->getCacheSize().headerSize > 0); - - file->clearCache(HEADER); - - CPPUNIT_ASSERT(file->getCacheSize().bodySize > 0); - CPPUNIT_ASSERT(file->getMemFileIO().getCachedSize(BODY) > 0); - CPPUNIT_ASSERT_EQUAL(0, (int)file->getCacheSize().headerSize); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), file->getMemFileIO().getCachedSize(HEADER)); - - file->clearCache(BODY); - - CPPUNIT_ASSERT_EQUAL(0, (int)file->getCacheSize().bodySize); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), file->getMemFileIO().getCachedSize(BODY)); -} - - -void -MemFileTest::testCompactGidCollision() -{ - // Feed two puts - for (uint32_t i = 0; i < 2; i++) { - feedDocument(1234 * i, 1000 + 200 * i); - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - const_cast<MemSlot&>((*file)[1]).setGlobalId((*file)[0].getGlobalId()); - - CPPUNIT_ASSERT_EQUAL(2, (int)file->getSlotCount()); - - { - SlotList toRemove(getSlotsToRemove(*file, 1600, 300, 86400)); - CPPUNIT_ASSERT_EQUAL(0, (int)toRemove.size()); - file->removeSlots(toRemove); - } -} - -void -MemFileTest::testCompactGidCollisionAndNot() -{ - // Feed some puts - for (uint32_t i = 0; i < 4; i++) { - feedDocument(1234 * (i % 2), 1000 + 200 * i); - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - const_cast<MemSlot&>((*file)[2]).setGlobalId((*file)[0].getGlobalId()); - const_cast<MemSlot&>((*file)[3]).setGlobalId((*file)[1].getGlobalId()); - - CPPUNIT_ASSERT_EQUAL(4, (int)file->getSlotCount()); - - { - SlotList toRemove(getSlotsToRemove(*file, 2000, 300, 86400)); - - CPPUNIT_ASSERT_EQUAL(2, (int)toRemove.size()); - ASSERT_SLOT_EQUAL(&(*file)[0], toRemove[0]); - ASSERT_SLOT_EQUAL(&(*file)[1], toRemove[1]); - file->removeSlots(toRemove); - } -} - - -void -MemFileTest::testCompactRemoveDoublePut() -{ - // Feed two puts at time 1000 and 1200 - for (uint32_t i = 0; i < 2; i++) { - feedDocument(1234, 1000 + 200 * i); - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - CPPUNIT_ASSERT_EQUAL(2, (int)file->getSlotCount()); - - { - // Not time to collect yet, newest is still revertable - SlotList toRemove(getSlotsToRemove(*file, 1300, 300, 86400)); - CPPUNIT_ASSERT_EQUAL(0, (int)toRemove.size()); - } - - { - SlotList toRemove(getSlotsToRemove(*file, 1600, 300, 86400)); - - CPPUNIT_ASSERT_EQUAL(1, (int)toRemove.size()); - ASSERT_SLOT_EQUAL(&(*file)[0], toRemove[0]); - file->removeSlots(toRemove); - } -} - -void -MemFileTest::testCompactPutRemove() -{ - document::DocumentId docId = feedDocument(1234, 1000); - - doRemove(docId, Timestamp(1200*1000000), 0); - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - - { - // Since remove can still be reverted, we can't revert anything. - SlotList toRemove(getSlotsToRemove(*file, 1300, 300, 600)); - - CPPUNIT_ASSERT_EQUAL(0, (int)toRemove.size()); - } - - { - SlotList toRemove(getSlotsToRemove(*file, 1600, 300, 600)); - - CPPUNIT_ASSERT_EQUAL(1, (int)toRemove.size()); - ASSERT_SLOT_EQUAL(&(*file)[0], toRemove[0]); - file->removeSlots(toRemove); - } - - { - SlotList toRemove(getSlotsToRemove(*file, 1900, 300, 600)); - - CPPUNIT_ASSERT_EQUAL(1, (int)toRemove.size()); - ASSERT_SLOT_EQUAL(&(*file)[0], toRemove[0]); - file->removeSlots(toRemove); - } -} - -void -MemFileTest::testCompactCombined() -{ - document::DocumentId docId; - - // Feed some puts at time 1000, 1200, 1400, 1600 and 1800 for same doc. - for (uint32_t i = 0; i < 5; i++) { - docId = feedDocument(1234, 1000 + i * 200); - } - flush(document::BucketId(16, 4)); - - // Now add remove at time 2000. - doRemove(docId, Timestamp(2000 * 1000000), 0); - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - CPPUNIT_ASSERT_EQUAL(6, (int)file->getSlotCount()); - - { - // Compact all redundant slots that are older than revert period of 300. - // This includes 1000, 1200, 1400 and 1600. - SlotList toRemove(getSlotsToRemove(*file, 2001, 300, 86400)); - CPPUNIT_ASSERT_EQUAL(4, (int)toRemove.size()); - for (int i = 0; i < 4; ++i) { - ASSERT_SLOT_EQUAL(&(*file)[i], toRemove[i]); - } - file->removeSlots(toRemove); - } -} - -void -MemFileTest::testCompactDifferentPuts() -{ - document::DocumentId docId; - - // Feed some puts - for (uint32_t i = 0; i < 2; i++) { - for (uint32_t j = 0; j < 3; j++) { - feedDocument(1234 * j, 1000 + (i * 3 + j) * 200); - } - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - CPPUNIT_ASSERT_EQUAL(6, (int)file->getSlotCount()); - - { - SlotList toRemove(getSlotsToRemove(*file, 3000, 300, 86400)); - CPPUNIT_ASSERT_EQUAL(3, (int)toRemove.size()); - - for (uint32_t i = 0; i < 3; i++) { - bool found = false; - for (uint32_t j = 0; j < 3; j++) { - if ((*file)[j] == *toRemove[i]) { - found = true; - } - } - - CPPUNIT_ASSERT(found); - } - file->removeSlots(toRemove); - } -} - -void -MemFileTest::testCompactWithMemFile() -{ - // Feed two puts - for (uint32_t i = 0; i < 2; i++) { - document::Document::SP doc(createRandomDocumentAtLocation( - 4, 1234, 10, 100)); - - doPut(doc, document::BucketId(16, 4), Timestamp((1000 + i * 200)*1000000), 0); - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - CPPUNIT_ASSERT_EQUAL(2, (int)file->getSlotCount()); - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options) - .revertTimePeriod(framework::MicroSecTime(1000)) - .build()); - - getFakeClock()._absoluteTime = framework::MicroSecTime(2000ULL * 1000000); - - CPPUNIT_ASSERT(file->compact()); - CPPUNIT_ASSERT(!file->compact()); - - CPPUNIT_ASSERT_EQUAL(1, (int)file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1200 * 1000000), (*file)[0].getTimestamp()); -} - -/** - * Feed 5 versions of a single document at absolute times 0 through 4 seconds - * and run compaction using the provided max document version option. - * Revert time/keep remove time options are effectively disabled for this test. - * Returns timestamps of all slots that are marked as compactable. - */ -std::vector<Types::Timestamp> -MemFileTest::compactWithVersionLimit(uint32_t maxVersions) -{ - document::BucketId bucket(16, 4); - std::shared_ptr<Document> doc( - createRandomDocumentAtLocation(4, 1234, 10, 100)); - uint32_t versionLimit = 5; - for (uint32_t i = 0; i < versionLimit; ++i) { - Timestamp ts(sec(i).getTime()); - doPut(doc, bucket, ts, 0); - } - flush(bucket); - - MemFilePtr file(getMemFile(bucket)); - CPPUNIT_ASSERT_EQUAL(versionLimit, file->getSlotCount()); - - framework::MicroSecTime currentTime(sec(versionLimit)); - MemFileCompactor compactor( - currentTime, - CompactionOptions() - .revertTimePeriod(sec(versionLimit)) - .keepRemoveTimePeriod(sec(versionLimit)) - .maxDocumentVersions(maxVersions)); - auto slots = compactor.getSlotsToRemove(*file); - // Convert to timestamps since caller won't have access to actual MemFile. - std::vector<Timestamp> timestamps; - for (const MemSlot* slot : slots) { - timestamps.push_back(slot->getTimestamp()); - } - return timestamps; -} - -void -MemFileTest::testNoCompactionWhenDocumentVersionsWithinLimit() -{ - auto timestamps = compactWithVersionLimit(5); - CPPUNIT_ASSERT(timestamps.empty()); -} - -void -MemFileTest::testCompactWhenDocumentVersionsExceedLimit() -{ - auto timestamps = compactWithVersionLimit(2); - CPPUNIT_ASSERT_EQUAL(size_t(3), timestamps.size()); - std::vector<Timestamp> expected = { - sec(0), sec(1), sec(2) - }; - CPPUNIT_ASSERT_EQUAL(expected, timestamps); -} - -void -MemFileTest::testCompactLimit1KeepsNewestVersionOnly() -{ - auto timestamps = compactWithVersionLimit(1); - CPPUNIT_ASSERT_EQUAL(size_t(4), timestamps.size()); - std::vector<Timestamp> expected = { - sec(0), sec(1), sec(2), sec(3) - }; - CPPUNIT_ASSERT_EQUAL(expected, timestamps); -} - -void -MemFileTest::testCompactionOptionsArePropagatedFromConfig() -{ - vespa::config::storage::StorMemfilepersistenceConfigBuilder mfcBuilder; - vespa::config::content::PersistenceConfigBuilder pcBuilder; - - pcBuilder.maximumVersionsOfSingleDocumentStored = 12345; - pcBuilder.revertTimePeriod = 555; - pcBuilder.keepRemoveTimePeriod = 777; - - vespa::config::storage::StorMemfilepersistenceConfig mfc(mfcBuilder); - vespa::config::content::PersistenceConfig pc(pcBuilder); - Options opts(mfc, pc); - - CPPUNIT_ASSERT_EQUAL(framework::MicroSecTime(555 * 1000000), - opts._revertTimePeriod); - CPPUNIT_ASSERT_EQUAL(framework::MicroSecTime(777 * 1000000), - opts._keepRemoveTimePeriod); - CPPUNIT_ASSERT_EQUAL(uint32_t(12345), opts._maxDocumentVersions); -} - -void -MemFileTest::testZeroDocumentVersionConfigIsCorrected() -{ - vespa::config::storage::StorMemfilepersistenceConfigBuilder mfcBuilder; - vespa::config::content::PersistenceConfigBuilder pcBuilder; - - pcBuilder.maximumVersionsOfSingleDocumentStored = 0; - - vespa::config::storage::StorMemfilepersistenceConfig mfc(mfcBuilder); - vespa::config::content::PersistenceConfig pc(pcBuilder); - Options opts(mfc, pc); - - CPPUNIT_ASSERT_EQUAL(uint32_t(1), opts._maxDocumentVersions); -} - -void -MemFileTest::testGetSlotsByTimestamp() -{ - for (uint32_t i = 0; i < 10; i++) { - feedDocument(i, 1000 + i); - } - flush(document::BucketId(16, 4)); - - std::vector<Timestamp> timestamps; - timestamps.push_back(Timestamp(999 * 1000000)); - timestamps.push_back(Timestamp(1001 * 1000000)); - timestamps.push_back(Timestamp(1002 * 1000000)); - timestamps.push_back(Timestamp(1007 * 1000000)); - timestamps.push_back(Timestamp(1100 * 1000000)); - std::vector<const MemSlot*> slots; - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - file->getSlotsByTimestamp(timestamps, slots); - CPPUNIT_ASSERT_EQUAL(std::size_t(3), slots.size()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1001 * 1000000), slots[0]->getTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1002 * 1000000), slots[1]->getTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1007 * 1000000), slots[2]->getTimestamp()); -} - -void -MemFileTest::testEnsureCached() -{ - // Feed some puts - for (uint32_t i = 0; i < 5; i++) { - feedDocument(i, 1000 + i * 200, 600, 600, 600); - } - flush(document::BucketId(16, 4)); - - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options).maximumReadThroughGap(512).build()); - env()._cache.clear(); - - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - CPPUNIT_ASSERT(file.get()); - CPPUNIT_ASSERT_EQUAL(5, (int)file->getSlotCount()); - - file->ensureDocumentIdCached((*file)[1]); - - for (std::size_t i = 0; i < file->getSlotCount(); ++i) { - if (i == 1) { - CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i])); - } else { - CPPUNIT_ASSERT(!file->documentIdAvailable((*file)[i])); - } - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY)); - } - } - - env()._cache.clear(); - - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - file->ensureDocumentCached((*file)[2], true); - - for (std::size_t i = 0; i < file->getSlotCount(); ++i) { - if (i == 2) { - CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER)); - } else { - CPPUNIT_ASSERT(!file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], HEADER)); - } - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY)); - } - } - - env()._cache.clear(); - - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - - file->ensureDocumentCached((*file)[3], false); - - for (std::size_t i = 0; i < file->getSlotCount(); ++i) { - if (i == 3) { - CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER)); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], BODY)); - } else { - CPPUNIT_ASSERT(!file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], HEADER)); - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY)); - } - } - } - - env()._cache.clear(); - - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - - std::vector<Timestamp> ts; - for (int i = 2; i < 5; ++i) { - ts.push_back((*file)[i].getTimestamp()); - } - - file->ensureDocumentCached(ts, false); - - for (std::size_t i = 0; i < file->getSlotCount(); ++i) { - if (i > 1 && i < 5) { - CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER)); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], BODY)); - } else { - CPPUNIT_ASSERT(!file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], HEADER)); - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY)); - } - } - } - - env()._cache.clear(); - - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - - file->ensureHeaderBlockCached(); - - for (std::size_t i = 0; i < file->getSlotCount(); ++i) { - CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER)); - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY)); - } - } - - env()._cache.clear(); - - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - - file->ensureBodyBlockCached(); - - for (std::size_t i = 0; i < file->getSlotCount(); ++i) { - CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER)); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], BODY)); - } - } -} - -void -MemFileTest::testResizeToFreeSpace() -{ - /** - * This test tests that files are resized to a smaller size when they need - * to be. This should happen during a call to flushToDisk() in MemFile, - * which is either dirty or if passed flag to check even if clean. (Which - * the integrity checker cycle uses). A clean file is used for testing to - * ensure that no part of the code only works for dirty files. This test - * only test for the case where body block is too large. The real - * implementation here will be in the flushUpdatesToFile() function for the - * given file formats. (VersionSerializer's) If more cases wants to be - * tested add those as unit tests for the versionserializers themselves. - */ - - // Create a test bucket to test with. - BucketId bucket(16, 0xa); - createTestBucket(bucket, 0); - - off_t file_size = - ((SimpleMemFileIOBuffer&)getMemFile(bucket)->getMemFileIO()). - getFileHandle().getFileSize(); - - // Clear cache so we can manually modify backing file to increase the - // size of it. - FileSpecification file(getMemFile(bucket)->getFile()); - env()._cache.clear(); - { - // Extend file to 1 MB, which should create an excessively large - // body block such that file should be resized to be smaller - vespalib::LazyFile fileHandle(file.getPath(), 0); - fileHandle.write("foobar", 6, 2 * 1024 * 1024 - 6); - } - MemFilePtr memFile(getMemFile(bucket)); - memFile->flushToDisk(CHECK_NON_DIRTY_FILE_FOR_SPACE); - CPPUNIT_ASSERT_EQUAL(file_size, - ((SimpleMemFileIOBuffer&)memFile->getMemFileIO()). - getFileHandle().getFileSize()); -} - -namespace { - -const vespalib::LazyFile& -getFileHandle(const MemFile& mf1) -{ - return dynamic_cast<const SimpleMemFileIOBuffer&>( - mf1.getMemFileIO()).getFileHandle(); -} - -const LoggingLazyFile& -getLoggerFile(const MemFile& file) -{ - return dynamic_cast<const LoggingLazyFile&>(getFileHandle(file)); -} - -} - -void -MemFileTest::testNoFileWriteOnNoOpCompaction() -{ - BucketId bucket(16, 4); - env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>( - new LoggingLazyFile::Factory()); - - // Feed some unique puts, none of which can be compacted away. - for (uint32_t i = 0; i < 2; i++) { - document::Document::SP doc(createRandomDocumentAtLocation( - 4, i, 10, 100)); - - doPut(doc, bucket, Timestamp((1000 + i * 200)*1000000), 0); - } - flush(bucket); - - MemFilePtr file(getMemFile(bucket)); - - size_t opsBeforeFlush = getLoggerFile(*file).getOperationCount(); - file->flushToDisk(CHECK_NON_DIRTY_FILE_FOR_SPACE); - size_t opsAfterFlush = getLoggerFile(*file).getOperationCount(); - - // Disk should not have been touched, since no slots have been - // compacted away. - if (opsBeforeFlush != opsAfterFlush) { - std::cerr << "\n" << getLoggerFile(*file).toString() << "\n"; - } - CPPUNIT_ASSERT_EQUAL(opsBeforeFlush, opsAfterFlush); -} - -void -MemFileTest::testAddSlotWhenDiskFull() -{ - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - { - // Add a dummy-slot that can later be removed - Document::SP doc(createRandomDocumentAtLocation(4)); - file->addPutSlot(*doc, Timestamp(1001)); - } - } - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - PartitionMonitor* mon = env().getDirectory().getPartition().getMonitor(); - // Set disk to 99% full - mon->setStatOncePolicy(); - mon->setMaxFillness(.98f); - mon->overrideRealStat(512, 100000, 99000); - CPPUNIT_ASSERT(mon->isFull()); - - // Test that addSlot with a non-persisted Put fails - { - Document::SP doc(createRandomDocumentAtLocation(4)); - try { - file->addPutSlot(*doc, Timestamp(10003)); - CPPUNIT_ASSERT(false); - } catch (vespalib::IoException& e) { - CPPUNIT_ASSERT_EQUAL(vespalib::IoException::NO_SPACE, e.getType()); - } - } - - // Slots with valid header and body locations should also - // not fail, as these are added when the file is loaded - { - // Just steal parts from existing slot to ensure they're persisted - const MemSlot* existing = file->getSlotAtTime(Timestamp(1001)); - - MemSlot slot(existing->getGlobalId(), - Timestamp(1005), - existing->getLocation(HEADER), - existing->getLocation(BODY), - IN_USE, - 0x1234); - file->addSlot(slot); - } - - // Removes should not fail when disk is full - { - file->addRemoveSlot(*file->getSlotAtTime(Timestamp(1001)), Timestamp(1003)); - } -} - -void -MemFileTest::testGetSerializedSize() { - document::Document::SP doc(createRandomDocumentAtLocation( - 4, 1234, 1024, 1024)); - - std::string val("Header"); - doc->setValue(doc->getField("hstringval"), - document::StringFieldValue(val)); - - doPut(doc, document::BucketId(16, 4), framework::MicroSecTime(1000)); - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - file->ensureBodyBlockCached(); - const MemSlot* slot = file->getSlotAtTime(framework::MicroSecTime(1000)); - CPPUNIT_ASSERT(slot != 0); - - vespalib::nbostream serializedHeader; - doc->serializeHeader(serializedHeader); - - vespalib::nbostream serializedBody; - doc->serializeBody(serializedBody); - - CPPUNIT_ASSERT_EQUAL(uint32_t(serializedHeader.size()), - file->getSerializedSize(*slot, HEADER)); - CPPUNIT_ASSERT_EQUAL(uint32_t(serializedBody.size()), - file->getSerializedSize(*slot, BODY)); -} - -void -MemFileTest::testGetBucketInfo() -{ - document::Document::SP doc(createRandomDocumentAtLocation( - 4, 1234, 100, 100)); - doc->setValue(doc->getField("content"), - document::StringFieldValue("foo")); - document::Document::SP doc2(createRandomDocumentAtLocation( - 4, 1235, 100, 100)); - doc2->setValue(doc->getField("content"), - document::StringFieldValue("bar")); - - doPut(doc, document::BucketId(16, 4), framework::MicroSecTime(1000)); - flush(document::BucketId(16, 4)); - - doPut(doc2, document::BucketId(16, 4), framework::MicroSecTime(1001)); - flush(document::BucketId(16, 4)); - - // Do remove which should only add a single meta entry - doRemove(doc->getId(), Timestamp(1002), 0); - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - - CPPUNIT_ASSERT_EQUAL(3u, file->getSlotCount()); - uint32_t maxHeaderExtent = (*file)[1].getLocation(HEADER)._pos - + (*file)[1].getLocation(HEADER)._size; - uint32_t maxBodyExtent = (*file)[1].getLocation(BODY)._pos - + (*file)[1].getLocation(BODY)._size; - - uint32_t wantedUsedSize = 64 + 40*3 + maxHeaderExtent + maxBodyExtent; - BucketInfo info = file->getBucketInfo(); - CPPUNIT_ASSERT_EQUAL(1u, info.getDocumentCount()); - CPPUNIT_ASSERT_EQUAL(3u, info.getEntryCount()); - CPPUNIT_ASSERT_EQUAL(wantedUsedSize, info.getUsedSize()); - uint32_t wantedUniqueSize = (*file)[1].getLocation(HEADER)._size - + (*file)[1].getLocation(BODY)._size; - CPPUNIT_ASSERT_EQUAL(wantedUniqueSize, info.getDocumentSize()); -} - -void -MemFileTest::testCopySlotsPreservesLocationSharing() -{ - document::BucketId bucket(16, 4); - // Feed two puts to same document (identical seed). These should not - // share any blocks. Note: implicit sec -> microsec conversion. - feedDocument(1234, 1000); // slot 0 - auto docId = feedDocument(1234, 1001); // slot 1 - // Update only header of last version of document. This should share - // slot body block 2 with that slot 1. - auto update = createHeaderUpdate(docId, document::IntFieldValue(5678)); - doUpdate(bucket, update, Timestamp(1002 * 1000000), 0); - // Feed a remove for doc in slot 2. This should share the header block of - // slot 3 with the newest document in slot 2. - doRemove(docId, Timestamp(1003 * 1000000), 0); - flush(bucket); - - { - MemFilePtr src(getMemFile(document::BucketId(16, 4))); - MemFilePtr dest(getMemFile(document::BucketId(17, 4))); - std::vector<Timestamp> timestamps { - Timestamp(1000 * 1000000), - Timestamp(1001 * 1000000), - Timestamp(1002 * 1000000), - Timestamp(1003 * 1000000) - }; - std::vector<const MemSlot*> slots { - src->getSlotAtTime(Timestamp(1000 * 1000000)), - src->getSlotAtTime(Timestamp(1001 * 1000000)), - src->getSlotAtTime(Timestamp(1002 * 1000000)), - src->getSlotAtTime(Timestamp(1003 * 1000000)) - }; - dest->copySlotsFrom(*src, slots); - dest->flushToDisk(); - CPPUNIT_ASSERT_EQUAL(uint32_t(4), dest->getSlotCount()); - - DataLocation header[4]; - DataLocation body[4]; - for (int i = 0; i < 4; ++i) { - const MemSlot* slot = dest->getSlotAtTime(timestamps[i]); - header[i] = slot->getLocation(HEADER); - body[i] = slot->getLocation(BODY); - } - CPPUNIT_ASSERT(!(header[0] == header[1])); - - CPPUNIT_ASSERT_EQUAL(body[2], body[1]); - CPPUNIT_ASSERT_EQUAL(header[3], header[2]); - } -} - -void -MemFileTest::testFlushingToNonExistingFileAlwaysRunsCompaction() -{ - document::BucketId bucket(16, 4); - - setMaxDocumentVersionsOption(1); - feedSameDocNTimes(10); - flush(bucket); - - // Max version limit is 1, flushing should have compacted it down. - MemFilePtr file(getMemFile(bucket)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount()); -} - -void -MemFileTest::testOrderDocSchemeDocumentsCanBeAddedToFile() -{ - // Quick explanation of the esoteric and particular values chosen below: - // orderdoc mangles the MSB of the bucket ID based on the document ID's - // ordering parameters and thus its bucket cannot be directly deduced from - // the generated GID. The values given here specify a document whose GID - // bits differ from those generated by the document and where a GID-only - // bucket ownership check would fail (nuking the node with an assertion). - // We have to make sure cases do not trigger false positives. - document::BucketId bucket(0x84000000ee723751); - auto doc = createDocument("the quick red fox trips over a hedge", - "orderdoc(3,1):storage_test:group1:9:9"); - doPut(std::shared_ptr<Document>(std::move(doc)), - bucket, - Timestamp(1000000 * 1234)); - flush(bucket); - - MemFilePtr file(getMemFile(bucket)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount()); - // Ideally we'd test the failure case as well, but that'd require framework - // support for death tests. -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.cpp b/memfilepersistence/src/tests/spi/memfiletestutils.cpp deleted file mode 100644 index 9571d880e9f..00000000000 --- a/memfilepersistence/src/tests/spi/memfiletestutils.cpp +++ /dev/null @@ -1,455 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/document/datatype/documenttype.h> -#include <vespa/memfilepersistence/spi/memfilepersistenceprovider.h> -#include <tests/spi/memfiletestutils.h> -#include <tests/spi/simulatedfailurefile.h> -#include <vespa/memfilepersistence/memfile/memfilecache.h> -#include <vespa/document/update/assignvalueupdate.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/document/test/make_bucket_space.h> -#include <vespa/persistence/spi/test.h> -#include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/util/exceptions.h> -#include <sys/time.h> - -using document::DocumentType; -using document::test::makeBucketSpace; -using storage::spi::test::makeSpiBucket; - -namespace storage { -namespace memfile { - -namespace { - spi::LoadType defaultLoadType(0, "default"); -} - -namespace { - vdstestlib::DirConfig initialize(uint32_t numDisks) { - system(vespalib::make_string("rm -rf vdsroot").c_str()); - for (uint32_t i = 0; i < numDisks; i++) { - system(vespalib::make_string("mkdir -p vdsroot/disks/d%d", i).c_str()); - } - vdstestlib::DirConfig config(getStandardConfig(true)); - return config; - } - - template<typename T> - struct ConfigReader : public T::Subscriber - { - T config; - - ConfigReader(const std::string& configId) { - T::subscribe(configId, *this); - } - void configure(const T& c) { config = c; } - }; -} - -MemFileTestEnvironment::MemFileTestEnvironment( - uint32_t numDisks, - framework::ComponentRegister& reg, - const document::DocumentTypeRepo& repo) - : _config(initialize(numDisks)), - _provider(reg, _config.getConfigId()) -{ - _provider.setDocumentRepo(repo); - _provider.getPartitionStates(); -} - -MemFileTestUtils::MemFileTestUtils() -{ -} - -MemFileTestUtils::~MemFileTestUtils() -{ -} - -void -MemFileTestUtils::setupDisks(uint32_t numDisks) { - tearDown(); - _componentRegister.reset( - new framework::defaultimplementation::ComponentRegisterImpl); - _clock.reset(new FakeClock); - _componentRegister->setClock(*_clock); - _env.reset(new MemFileTestEnvironment(numDisks, - *_componentRegister, - *getTypeRepo())); -} - -Environment& -MemFileTestUtils::env() -{ - return static_cast<MemFilePersistenceProvider&>( - getPersistenceProvider()).getEnvironment(); -} - -MemFilePersistenceProvider& -MemFileTestUtils::getPersistenceProvider() -{ - return _env->_provider; -} - -MemFilePersistenceThreadMetrics& -MemFileTestUtils::getMetrics() -{ - return getPersistenceProvider().getMetrics(); -} - -std::string -MemFileTestUtils::getMemFileStatus(const document::BucketId& id, - uint32_t disk) -{ - MemFilePtr file(getMemFile(id, disk)); - std::ostringstream ost; - ost << id << ": " << file->getSlotCount() << "," << file->getDisk(); - return ost.str(); -} - -std::string -MemFileTestUtils::getModifiedBuckets() -{ - spi::BucketIdListResult result( - getPersistenceProvider().getModifiedBuckets(makeBucketSpace())); - const spi::BucketIdListResult::List& list(result.getList()); - std::ostringstream ss; - for (size_t i = 0; i < list.size(); ++i) { - if (i != 0) { - ss << ","; - } - ss << std::hex << list[i].getId(); - } - return ss.str(); -} - -MemFilePtr -MemFileTestUtils::getMemFile(const document::BucketId& id, uint16_t disk) -{ - return env()._cache.get(id, env(), env().getDirectory(disk)); -} - -spi::Result -MemFileTestUtils::flush(const document::BucketId& id, uint16_t disk) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - return getPersistenceProvider().flush( - makeSpiBucket(id, spi::PartitionId(disk)), context); -} - -document::Document::SP -MemFileTestUtils::doPutOnDisk( - uint16_t disk, - uint32_t location, - Timestamp timestamp, - uint32_t minSize, - uint32_t maxSize) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::Document::SP doc(createRandomDocumentAtLocation( - location, timestamp.getTime(), minSize, maxSize)); - getPersistenceProvider().put( - makeSpiBucket(document::BucketId(16, location), spi::PartitionId(disk)), - spi::Timestamp(timestamp.getTime()), - doc, - context); - return doc; -} - -bool -MemFileTestUtils::doRemoveOnDisk( - uint16_t disk, - const document::BucketId& bucketId, - const document::DocumentId& docId, - Timestamp timestamp, - OperationHandler::RemoveType persistRemove) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - if (persistRemove == OperationHandler::PERSIST_REMOVE_IF_FOUND) { - spi::RemoveResult result = getPersistenceProvider().removeIfFound( - makeSpiBucket(bucketId, spi::PartitionId(disk)), - spi::Timestamp(timestamp.getTime()), - docId, - context); - return result.wasFound(); - } - spi::RemoveResult result = getPersistenceProvider().remove( - makeSpiBucket(bucketId, spi::PartitionId(disk)), - spi::Timestamp(timestamp.getTime()), - docId, - context); - - return result.wasFound(); -} - -bool -MemFileTestUtils::doUnrevertableRemoveOnDisk( - uint16_t disk, - const document::BucketId& bucketId, - const DocumentId& docId, - Timestamp timestamp) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - spi::RemoveResult result = - getPersistenceProvider().remove( - makeSpiBucket(bucketId, spi::PartitionId(disk)), - spi::Timestamp(timestamp.getTime()), - docId, context); - - return result.wasFound(); -} - -spi::GetResult -MemFileTestUtils::doGetOnDisk( - uint16_t disk, - const document::BucketId& bucketId, - const document::DocumentId& docId, - const document::FieldSet& fields) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - return getPersistenceProvider().get( - makeSpiBucket(bucketId, spi::PartitionId(disk)), - fields, docId, context); -} - -document::DocumentUpdate::SP -MemFileTestUtils::createBodyUpdate( - const document::DocumentId& docId, - const document::FieldValue& updateValue) -{ - const DocumentType* - docType(getTypeRepo()->getDocumentType("testdoctype1")); - document::DocumentUpdate::SP update( - new document::DocumentUpdate(*docType, docId)); - std::shared_ptr<document::AssignValueUpdate> assignUpdate( - new document::AssignValueUpdate(updateValue)); - document::FieldUpdate fieldUpdate(docType->getField("content")); - fieldUpdate.addUpdate(*assignUpdate); - update->addUpdate(fieldUpdate); - return update; -} - -document::DocumentUpdate::SP -MemFileTestUtils::createHeaderUpdate( - const document::DocumentId& docId, - const document::FieldValue& updateValue) -{ - const DocumentType* - docType(getTypeRepo()->getDocumentType("testdoctype1")); - document::DocumentUpdate::SP update( - new document::DocumentUpdate(*docType, docId)); - std::shared_ptr<document::AssignValueUpdate> assignUpdate( - new document::AssignValueUpdate(updateValue)); - document::FieldUpdate fieldUpdate(docType->getField("headerval")); - fieldUpdate.addUpdate(*assignUpdate); - update->addUpdate(fieldUpdate); - return update; -} - -void -MemFileTestUtils::doPut(const document::Document::SP& doc, - Timestamp time, - uint16_t disk, - uint16_t usedBits) -{ - document::BucketId bucket( - getBucketIdFactory().getBucketId(doc->getId())); - bucket.setUsedBits(usedBits); - doPut(doc, bucket, time, disk); -} - -void -MemFileTestUtils::doPut(const document::Document::SP& doc, - document::BucketId bid, - Timestamp time, - uint16_t disk) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().put(makeSpiBucket(bid, spi::PartitionId(disk)), - spi::Timestamp(time.getTime()), doc, context); -} - -spi::UpdateResult -MemFileTestUtils::doUpdate(document::BucketId bid, - const document::DocumentUpdate::SP& update, - Timestamp time, - uint16_t disk) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - return getPersistenceProvider().update( - makeSpiBucket(bid, spi::PartitionId(disk)), - spi::Timestamp(time.getTime()), update, context); -} - -void -MemFileTestUtils::doRemove(const document::DocumentId& id, Timestamp time, - uint16_t disk, bool unrevertableRemove, - uint16_t usedBits) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucket(getBucketIdFactory().getBucketId(id)); - bucket.setUsedBits(usedBits); - - if (unrevertableRemove) { - getPersistenceProvider().remove( - makeSpiBucket(bucket, spi::PartitionId(disk)), - spi::Timestamp(time.getTime()), - id, context); - } else { - spi::RemoveResult result = getPersistenceProvider().removeIfFound( - makeSpiBucket(bucket, spi::PartitionId(disk)), - spi::Timestamp(time.getTime()), - id, context); - - if (!result.wasFound()) { - throw vespalib::IllegalStateException( - "Attempted to remove non-existing doc " + id.toString(), - VESPA_STRLOC); - } - } -} - -void -MemFileTestUtils::copyHeader(document::Document& dest, - const document::Document& src) -{ - // FIXME(vekterli): temporary solution while we don't have - // fieldset pruning functionality in Document. - //dest.setHeaderPtr(src.getHeaderPtr()); - vespalib::nbostream originalBodyStream; - dest.serializeBody(originalBodyStream); - - vespalib::nbostream headerStream; - src.serializeHeader(headerStream); - document::ByteBuffer hbuf(headerStream.peek(), headerStream.size()); - dest.deserializeHeader(*getTypeRepo(), hbuf); - // deserializeHeader clears fields struct, so have to re-set body - document::ByteBuffer bbuf(originalBodyStream.peek(), - originalBodyStream.size()); - dest.deserializeBody(*getTypeRepo(), bbuf); -} - -void -MemFileTestUtils::copyBody(document::Document& dest, - const document::Document& src) -{ - // FIXME(vekterli): temporary solution while we don't have - // fieldset pruning functionality in Document. - //dest.setBodyPtr(src.getBodyPtr()); - vespalib::nbostream stream; - src.serializeBody(stream); - document::ByteBuffer buf(stream.peek(), stream.size()); - dest.deserializeBody(*getTypeRepo(), buf); -} - -void -MemFileTestUtils::clearBody(document::Document& doc) -{ - // FIXME(vekterli): temporary solution while we don't have - // fieldset pruning functionality in Document. - //doc->getBody().clear(); - vespalib::nbostream stream; - doc.serializeHeader(stream); - doc.deserialize(*getTypeRepo(), stream); -} - -void -MemFileTestUtils::createTestBucket(const document::BucketId& bucket, - uint16_t disk) -{ - - uint32_t opsPerType = 2; - uint32_t numberOfLocations = 2; - uint32_t minDocSize = 0; - uint32_t maxDocSize = 128; - - for (uint32_t useHeaderOnly = 0; useHeaderOnly < 2; ++useHeaderOnly) { - bool headerOnly = (useHeaderOnly == 1); - for (uint32_t optype=0; optype < 4; ++optype) { - for (uint32_t i=0; i<opsPerType; ++i) { - uint32_t seed = useHeaderOnly * 10000 + optype * 1000 + i + 1; - uint64_t location = (seed % numberOfLocations); - location <<= 32; - location += (bucket.getRawId() & 0xffffffff); - document::Document::SP doc( - createRandomDocumentAtLocation( - location, seed, minDocSize, maxDocSize)); - if (headerOnly) { - clearBody(*doc); - } - doPut(doc, Timestamp(seed), disk, bucket.getUsedBits()); - if (optype == 0) { // Regular put - } else if (optype == 1) { // Overwritten later in time - Document::SP doc2(new Document(*doc)); - doc2->setValue(doc2->getField("content"), - document::StringFieldValue("overwritten")); - doPut(doc2, Timestamp(seed + 500), - disk, bucket.getUsedBits()); - } else if (optype == 2) { // Removed - doRemove(doc->getId(), Timestamp(seed + 500), disk, false, - bucket.getUsedBits()); - } else if (optype == 3) { // Unrevertable removed - doRemove(doc->getId(), Timestamp(seed), disk, true, - bucket.getUsedBits()); - } - } - } - } - flush(bucket, disk); -} - -void -MemFileTestUtils::simulateIoErrorsForSubsequentlyOpenedFiles( - const IoErrors& errs) -{ - std::unique_ptr<SimulatedFailureLazyFile::Factory> factory( - new SimulatedFailureLazyFile::Factory); - factory->setWriteOpsBeforeFailure(errs._afterWrites); - factory->setReadOpsBeforeFailure(errs._afterReads); - env()._lazyFileFactory = std::move(factory); -} - -void -MemFileTestUtils::unSimulateIoErrorsForSubsequentlyOpenedFiles() -{ - env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>( - new DefaultLazyFileFactory(0)); -} - -std::string -MemFileTestUtils::stringifyFields(const document::Document& doc) const -{ - using namespace document; - std::vector<std::string> output; - const StructFieldValue& fields(doc.getFields()); - for (StructFieldValue::const_iterator - it(fields.begin()), e(fields.end()); - it != e; ++it) - { - std::ostringstream ss; - const Field& f(it.field()); - ss << f.getName() << ": "; - FieldValue::UP val(fields.getValue(f)); - if (val.get()) { - ss << val->toString(); - } else { - ss << "(null)"; - } - output.push_back(ss.str()); - } - std::ostringstream ret; - std::sort(output.begin(), output.end()); - std::copy(output.begin(), output.end(), - std::ostream_iterator<std::string>(ret, "\n")); - return ret.str(); -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.h b/memfilepersistence/src/tests/spi/memfiletestutils.h deleted file mode 100644 index 657b116b6e5..00000000000 --- a/memfilepersistence/src/tests/spi/memfiletestutils.h +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::memfile::MemFileTestUtils - * \ingroup memfile - * - * \brief Utilities for unit tests of the MemFile layer. - * - * The memfile layer typically needs a MemFileEnvironment object that must be - * set up. This class creates such an object to be used by unit tests. Other - * utilities useful for only MemFile testing can be added here too. - */ - -#pragma once - -#include <vespa/memfilepersistence/memfile/memfilecache.h> -#include <tests/helper/testhelper.h> -#include <vespa/persistence/spi/persistenceprovider.h> -#include <vespa/memfilepersistence/spi/memfilepersistenceprovider.h> -#include <vespa/document/base/testdocman.h> -#include <vespa/document/update/documentupdate.h> -#include <vespa/storageframework/defaultimplementation/clock/realclock.h> -#include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> - -namespace storage { -namespace memfile { - -struct FakeClock : public framework::Clock { -public: - typedef std::unique_ptr<FakeClock> UP; - - framework::MicroSecTime _absoluteTime; - - FakeClock() {} - - virtual void addSecondsToTime(uint32_t nr) { - _absoluteTime += framework::MicroSecTime(nr * uint64_t(1000000)); - } - - framework::MicroSecTime getTimeInMicros() const override { - return _absoluteTime; - } - framework::MilliSecTime getTimeInMillis() const override { - return getTimeInMicros().getMillis(); - } - framework::SecondTime getTimeInSeconds() const override { - return getTimeInMicros().getSeconds(); - } - framework::MonotonicTimePoint getMonotonicTime() const override { - return framework::MonotonicTimePoint(std::chrono::microseconds( - getTimeInMicros().getTime())); - } -}; - -struct MemFileTestEnvironment { - MemFileTestEnvironment(uint32_t numDisks, - framework::ComponentRegister& reg, - const document::DocumentTypeRepo& repo); - - vdstestlib::DirConfig _config; - MemFilePersistenceProvider _provider; -}; - -class MemFileTestUtils : public Types, public document::TestDocMan, public CppUnit::TestFixture { -private: - // This variables are kept in test class. Instances that needs to be - // unique per test needs to be setup in setupDisks and cleared in - // tearDown - document::BucketIdFactory _bucketIdFactory; - framework::defaultimplementation::ComponentRegisterImpl::UP _componentRegister; - FakeClock::UP _clock; - std::unique_ptr<MemFileTestEnvironment> _env; - -public: - MemFileTestUtils(); - virtual ~MemFileTestUtils(); - - void setupDisks(uint32_t disks); - - void tearDown() override{ - _env.reset(); - _componentRegister.reset(); - _clock.reset(); - } - - std::string getMemFileStatus(const document::BucketId& id, uint32_t disk = 0); - - std::string getModifiedBuckets(); - - /** - Flushes all cached data to disk and updates the bucket database accordingly. - */ - void flush(); - - FakeClock& getFakeClock() { return *_clock; } - - spi::Result flush(const document::BucketId& id, uint16_t disk = 0); - - MemFilePersistenceProvider& getPersistenceProvider(); - - MemFilePtr getMemFile(const document::BucketId& id, uint16_t disk = 0); - - Environment& env(); - - MemFilePersistenceThreadMetrics& getMetrics(); - - MemFileTestEnvironment& getEnv() { return *_env; } - - /** - Performs a put to the given disk. - Returns the document that was inserted. - */ - document::Document::SP doPutOnDisk( - uint16_t disk, - uint32_t location, - Timestamp timestamp, - uint32_t minSize = 0, - uint32_t maxSize = 128); - - document::Document::SP doPut( - uint32_t location, - Timestamp timestamp, - uint32_t minSize = 0, - uint32_t maxSize = 128) - { return doPutOnDisk(0, location, timestamp, minSize, maxSize); } - - /** - Performs a remove to the given disk. - Returns the new doccount if document was removed, or -1 if not found. - */ - bool doRemoveOnDisk( - uint16_t disk, - const document::BucketId& bid, - const document::DocumentId& id, - Timestamp timestamp, - OperationHandler::RemoveType persistRemove); - - bool doRemove( - const document::BucketId& bid, - const document::DocumentId& id, - Timestamp timestamp, - OperationHandler::RemoveType persistRemove) { - return doRemoveOnDisk(0, bid, id, timestamp, persistRemove); - } - - bool doUnrevertableRemoveOnDisk(uint16_t disk, - const document::BucketId& bid, - const DocumentId& id, - Timestamp timestamp); - - bool doUnrevertableRemove(const document::BucketId& bid, - const DocumentId& id, - Timestamp timestamp) - { - return doUnrevertableRemoveOnDisk(0, bid, id, timestamp); - } - - virtual const document::BucketIdFactory& getBucketIdFactory() const - { return _bucketIdFactory; } - - document::BucketIdFactory& getBucketIdFactory() - { return _bucketIdFactory; } - - /** - * Do a remove toward storage set up in test environment. - * - * @id Document to remove. - * @disk If set, use this disk, otherwise lookup in bucket db. - * @unrevertableRemove If set, instead of adding put, turn put to remove. - * @usedBits Generate bucket to use from docid using this amount of bits. - */ - void doRemove(const DocumentId& id, Timestamp, uint16_t disk, - bool unrevertableRemove = false, uint16_t usedBits = 16); - - spi::GetResult doGetOnDisk( - uint16_t disk, - const document::BucketId& bucketId, - const document::DocumentId& docId, - const document::FieldSet& fields); - - spi::GetResult doGet( - const document::BucketId& bucketId, - const document::DocumentId& docId, - const document::FieldSet& fields) - { return doGetOnDisk(0, bucketId, docId, fields); } - - document::DocumentUpdate::SP createBodyUpdate( - const document::DocumentId& id, - const document::FieldValue& updateValue); - - document::DocumentUpdate::SP createHeaderUpdate( - const document::DocumentId& id, - const document::FieldValue& updateValue); - - virtual const std::shared_ptr<const document::DocumentTypeRepo> getTypeRepo() const - { return document::TestDocMan::getTypeRepoSP(); } - - /** - * Do a put toward storage set up in test environment. - * - * @doc Document to put. Use TestDocMan to generate easily. - * @disk If set, use this disk, otherwise lookup in bucket db. - * @usedBits Generate bucket to use from docid using this amount of bits. - */ - void doPut(const Document::SP& doc, Timestamp, - uint16_t disk, uint16_t usedBits = 16); - - void doPut(const document::Document::SP& doc, - document::BucketId bid, - Timestamp time, - uint16_t disk = 0); - - spi::UpdateResult doUpdate(document::BucketId bid, - const document::DocumentUpdate::SP& update, - Timestamp time, - uint16_t disk = 0); - - /** - * Create a test bucket with various content representing most states a - * bucket can represent. (Such that tests have a nice test bucket to use - * that require operations to handle all the various bucket contents. - * - * @disk If set, use this disk, otherwise lookup in bucket db. - */ - void createTestBucket(const BucketId&, uint16_t disk = 0xffff); - - /** - * In-place modify doc so that it has no more body fields. - */ - void clearBody(document::Document& doc); - - /** - * Copy all header data from src into dest, replacing any - * header fields it may already have there. NOTE: this will - * also overwrite document ID, type etc! - */ - void copyHeader(document::Document& dest, - const document::Document& src); - - /** - * Copy all body data from src into dest, replacing any - * body fields it may already have there. - */ - void copyBody(document::Document& dest, - const document::Document& src); - - std::string stringifyFields(const Document& doc) const; - - struct IoErrors { - int _afterReads; - int _afterWrites; - - IoErrors() - : _afterReads(0), - _afterWrites(0) - { - } - - IoErrors& afterReads(int n) { - _afterReads = n; - return *this; - } - - IoErrors& afterWrites(int n) { - _afterWrites = n; - return *this; - } - }; - - /** - * Replaces internal LazyFile factory so that it produces LazyFile - * implementations that trigger I/O exceptions on read/write. Optionally, - * can supply a parameter setting explicit bounds on how many operations - * are allowed on a file before trigging exceptions from there on out. A - * bound of -1 in practice means "don't fail ever" while 0 means "fail the - * next op of that type". - */ - void simulateIoErrorsForSubsequentlyOpenedFiles( - const IoErrors& errs = IoErrors()); - - /** - * Replace internal LazyFile factory with the default, non-failing impl. - */ - void unSimulateIoErrorsForSubsequentlyOpenedFiles(); -}; - -class SingleDiskMemFileTestUtils : public MemFileTestUtils -{ -public: - void setUp() override { - setupDisks(1); - } -}; - -} // memfile -} // storage - diff --git a/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp b/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp deleted file mode 100644 index 9eb2ca00f60..00000000000 --- a/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp +++ /dev/null @@ -1,1100 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/memfilepersistence/mapper/memfilemapper.h> -#include <vespa/memfilepersistence/mapper/memfile_v1_serializer.h> -#include <tests/spi/memfiletestutils.h> -#include <vespa/memfilepersistence/mapper/locationreadplanner.h> -#include <tests/spi/simulatedfailurefile.h> -#include <tests/spi/options_builder.h> - -namespace storage { -namespace memfile { - -struct MemFileV1SerializerTest : public SingleDiskMemFileTestUtils -{ - void tearDown() override; - void setUpPartialWriteEnvironment(); - void resetConfig(uint32_t minimumFileSize, uint32_t minimumFileHeaderBlockSize); - void doTestPartialWriteRemove(bool readAll); - void doTestPartialWriteUpdate(bool readAll); - - void testWriteReadSingleDoc(); - void testWriteReadPartial(); - void testWriteReadPartialRemoved(); - void testPartialWritePutHeaderOnly(); - void testPartialWritePut(); - void testPartialWriteRemoveCached(); - void testPartialWriteRemoveNotCached(); - void testPartialWriteUpdateCached(); - void testPartialWriteUpdateNotCached(); - void testPartialWriteTooMuchFreeSpace(); - void testPartialWriteNotEnoughFreeSpace(); - void testWriteReadSingleRemovedDoc(); - void testLocationDiskIoPlannerSimple(); - void testLocationDiskIoPlannerMergeReads(); - void testLocationDiskIoPlannerAlignReads(); - void testLocationDiskIoPlannerOneDocument(); - void testSeparateReadsForHeaderAndBody(); - void testLocationsRemappedConsistently(); - void testHeaderBufferTooSmall(); - - /*std::unique_ptr<MemFile> createMemFile(FileSpecification& file, - bool callLoadFile) - { - return std::unique_ptr<MemFile>(new MemFile(file, env(), callLoadFile)); - }*/ - - CPPUNIT_TEST_SUITE(MemFileV1SerializerTest); - CPPUNIT_TEST(testWriteReadSingleDoc); - CPPUNIT_TEST(testWriteReadPartial); - CPPUNIT_TEST(testWriteReadPartialRemoved); - CPPUNIT_TEST(testWriteReadSingleRemovedDoc); - CPPUNIT_TEST(testPartialWritePutHeaderOnly); - CPPUNIT_TEST(testPartialWritePut); - CPPUNIT_TEST(testPartialWriteRemoveCached); - CPPUNIT_TEST(testPartialWriteRemoveNotCached); - CPPUNIT_TEST(testPartialWriteUpdateCached); - CPPUNIT_TEST(testPartialWriteUpdateNotCached); - CPPUNIT_TEST(testLocationDiskIoPlannerSimple); - CPPUNIT_TEST(testLocationDiskIoPlannerMergeReads); - CPPUNIT_TEST(testLocationDiskIoPlannerAlignReads); - CPPUNIT_TEST(testLocationDiskIoPlannerOneDocument); - CPPUNIT_TEST(testSeparateReadsForHeaderAndBody); - CPPUNIT_TEST(testPartialWriteTooMuchFreeSpace); - CPPUNIT_TEST(testPartialWriteNotEnoughFreeSpace); - CPPUNIT_TEST(testLocationsRemappedConsistently); - CPPUNIT_TEST(testHeaderBufferTooSmall); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemFileV1SerializerTest); - -namespace { - -const vespalib::LazyFile& -getFileHandle(const MemFile& mf1) -{ - return static_cast<const SimpleMemFileIOBuffer&>( - mf1.getMemFileIO()).getFileHandle(); -} - -const LoggingLazyFile& -getLoggerFile(const MemFile& file) -{ - return static_cast<const LoggingLazyFile&>(getFileHandle(file)); -} - -bool isContentEqual(MemFile& mf1, MemFile& mf2, - bool requireEqualContentCached, std::ostream& error) -{ - MemFile::const_iterator it1( - mf1.begin(Types::ITERATE_GID_UNIQUE | Types::ITERATE_REMOVED)); - MemFile::const_iterator it2( - mf2.begin(Types::ITERATE_GID_UNIQUE | Types::ITERATE_REMOVED)); - while (true) { - if (it1 == mf1.end() && it2 == mf2.end()) { - return true; - } - if (it1 == mf1.end() || it2 == mf2.end()) { - error << "Different amount of GID unique slots"; - return false; - } - if (it1->getTimestamp() != it2->getTimestamp()) { - error << "Different timestamps"; - return false; - } - if (it1->getGlobalId() != it2->getGlobalId()) { - error << "Different gids"; - return false; - } - if (it1->getPersistedFlags() != it2->getPersistedFlags()) { - error << "Different persisted flags"; - return false; - } - if (requireEqualContentCached) { - if (mf1.partAvailable(*it1, Types::BODY) - ^ mf2.partAvailable(*it2, Types::BODY) - || mf1.partAvailable(*it1, Types::HEADER) - ^ mf2.partAvailable(*it2, Types::HEADER)) - { - error << "Difference in cached content: "; - return false; - } - } - - if (mf1.partAvailable(*it1, Types::HEADER) && - mf2.partAvailable(*it2, Types::HEADER)) - { - document::Document::UP doc1 = mf1.getDocument(*it1, Types::ALL); - document::Document::UP doc2 = mf2.getDocument(*it2, Types::ALL); - - CPPUNIT_ASSERT(doc1.get()); - CPPUNIT_ASSERT(doc2.get()); - - if (*doc1 != *doc2) { - error << "Documents different: Expected:\n" - << doc1->toString(true) << "\nActual:\n" - << doc2->toString(true) << "\n"; - return false; - } - } - ++it1; - ++it2; - } -} - -bool -validateMemFileStructure(const MemFile& mf, std::ostream& error) -{ - const SimpleMemFileIOBuffer& ioBuf( - dynamic_cast<const SimpleMemFileIOBuffer&>(mf.getMemFileIO())); - const FileInfo& fileInfo(ioBuf.getFileInfo()); - if (fileInfo.getFileSize() % 512) { - error << "File size is not a multiple of 512 bytes"; - return false; - } - if (fileInfo.getBlockIndex(Types::BODY) % 512) { - error << "Body start index is not a multiple of 512 bytes"; - return false; - } - if (fileInfo.getBlockSize(Types::BODY) % 512) { - error << "Body size is not a multiple of 512 bytes"; - return false; - } - return true; -} - -} - -void -MemFileV1SerializerTest::tearDown() { - //_memFile.reset(); -} - -/** - * Adjust minimum slotfile size values to avoid rewriting file - * when we want to get a partial write - */ -void -MemFileV1SerializerTest::setUpPartialWriteEnvironment() -{ - resetConfig(4096, 2048); -} - -void -MemFileV1SerializerTest::resetConfig(uint32_t minimumFileSize, - uint32_t minimumFileHeaderBlockSize) -{ - using MemFileConfig = vespa::config::storage::StorMemfilepersistenceConfig; - using MemFileConfigBuilder - = vespa::config::storage::StorMemfilepersistenceConfigBuilder; - - MemFileConfigBuilder persistenceConfig( - *env().acquireConfigReadLock().memFilePersistenceConfig()); - persistenceConfig.minimumFileHeaderBlockSize = minimumFileHeaderBlockSize; - persistenceConfig.minimumFileSize = minimumFileSize; - auto newCfg = std::unique_ptr<MemFileConfig>( - new MemFileConfig(persistenceConfig)); - env().acquireConfigWriteLock().setMemFilePersistenceConfig( - std::move(newCfg)); -} - -struct DummyMemFileIOInterface : MemFileIOInterface { - Document::UP getDocumentHeader(const document::DocumentTypeRepo&, - DataLocation) const override - { - return Document::UP(); - } - - document::DocumentId getDocumentId(DataLocation) const override { - return document::DocumentId(""); - } - - void readBody(const document::DocumentTypeRepo&, - DataLocation, - Document&) const override - { - } - DataLocation addDocumentIdOnlyHeader( - const DocumentId&, - const document::DocumentTypeRepo&) override - { - return DataLocation(); - } - DataLocation addHeader(const Document&) override { return DataLocation(); } - DataLocation addBody(const Document&) override { return DataLocation(); } - void clear(DocumentPart) override {} - bool verifyConsistent() const override { return true; } - void move(const FileSpecification&) override {} - DataLocation copyCache(const MemFileIOInterface&, DocumentPart, DataLocation) override { - return DataLocation(); - } - - void close() override {}; - bool isCached(DataLocation, DocumentPart) const override { return false; } - bool isPersisted(DataLocation, DocumentPart) const override { return false; } - uint32_t getSerializedSize(DocumentPart, DataLocation) const override { return 0; } - - void ensureCached(Environment&, DocumentPart, const std::vector<DataLocation>&) override {} - - size_t getCachedSize(DocumentPart) const override { return 0; } -}; - -#define VESPA_MEMFILEV1_SETUP_SOURCE \ - system("rm -f testfile.0"); \ - document::Document::SP doc(createRandomDocumentAtLocation(4)); \ - FileSpecification file(document::BucketId(16, 4), env().getDirectory(0), "testfile.0"); \ - MemFile source(file, env()); - -#define VESPA_MEMFILEV1_DIFF(source, target) \ - "\nSource:\n" + source.toString(true) \ - + "\nTarget:\n" + target.toString(true) - -#define VESPA_MEMFILEV1_VALIDATE_STRUCTURE(mfile) \ -{ \ - std::ostringstream validateErr; \ - if (!validateMemFileStructure(mfile, validateErr)) { \ - CPPUNIT_FAIL(validateErr.str()); \ - } \ -} - -#define VESPA_MEMFILEV1_ASSERT_SERIALIZATION(sourceMemFile) \ -env()._memFileMapper.flush(sourceMemFile, env()); \ -VESPA_MEMFILEV1_VALIDATE_STRUCTURE(sourceMemFile) \ -MemFile target(file, env()); \ -VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target) \ -{ \ - target.ensureBodyBlockCached(); \ - target.getBucketInfo(); \ - std::ostringstream diff; \ - if (!isContentEqual(sourceMemFile, target, true, diff)) { \ - std::string msg = "MemFiles not content equal: " + diff.str() \ - + VESPA_MEMFILEV1_DIFF(sourceMemFile, target); \ - CPPUNIT_FAIL(msg); \ - } \ -} - -void -MemFileV1SerializerTest::testWriteReadSingleDoc() -{ - VESPA_MEMFILEV1_SETUP_SOURCE; - source.addPutSlot(*doc, Timestamp(1001)); - std::string foo(VESPA_MEMFILEV1_DIFF(source, source)); - VESPA_MEMFILEV1_ASSERT_SERIALIZATION(source); -} - -void -MemFileV1SerializerTest::testWriteReadPartial() -{ - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - std::map<Timestamp, Document::SP> docs; - { - MemFile source(file, env()); - - for (int i = 0; i < 50; ++i) { - Document::SP doc(createRandomDocumentAtLocation(4, i, 1000, 2000)); - source.addPutSlot(*doc, Timestamp(1001 + i)); - docs[Timestamp(1001 + i)] = doc; - } - - env()._memFileMapper.flush(source, env()); - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(source); - } - - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options).maximumReadThroughGap(1024).build()); - env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>( - new LoggingLazyFile::Factory()); - - MemFile target(file, env()); - - std::vector<Timestamp> timestamps; - - for (int i = 0; i < 50; i+=4) { - timestamps.push_back(Timestamp(1001 + i)); - } - CPPUNIT_ASSERT_EQUAL(size_t(13), timestamps.size()); - - getLoggerFile(target).operations.clear(); - target.ensureDocumentCached(timestamps, false); - // Headers are small enough that they get read in 1 op + 13 body reads - CPPUNIT_ASSERT_EQUAL(14, (int)getLoggerFile(target).operations.size()); - - for (std::size_t i = 0; i < timestamps.size(); ++i) { - const MemSlot* slot = target.getSlotAtTime(timestamps[i]); - CPPUNIT_ASSERT(slot); - CPPUNIT_ASSERT(target.partAvailable(*slot, HEADER)); - CPPUNIT_ASSERT(target.partAvailable(*slot, BODY)); - CPPUNIT_ASSERT_EQUAL(*docs[timestamps[i]], *target.getDocument(*slot, ALL)); - } - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target); -} - -void -MemFileV1SerializerTest::testWriteReadPartialRemoved() -{ - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - MemFile source(file, env()); - - for (int i = 0; i < 50; ++i) { - Document::SP doc(createRandomDocumentAtLocation(4, i, 1000, 2000)); - source.addPutSlot(*doc, Timestamp(1001 + i)); - source.addRemoveSlot(*source.getSlotAtTime(Timestamp(1001 + i)), - Timestamp(2001 + i)); - } - - env()._memFileMapper.flush(source, env()); - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(source); - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options).maximumReadThroughGap(1024).build()); - env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>( - new LoggingLazyFile::Factory); - - MemFile target(file, env()); - - std::vector<Timestamp> timestamps; - - for (int i = 0; i < 50; i+=4) { - timestamps.push_back(Timestamp(2001 + i)); - } - - getLoggerFile(target).operations.clear(); - target.ensureDocumentCached(timestamps, false); - // All removed; should only read header locations - CPPUNIT_ASSERT_EQUAL(1, (int)getLoggerFile(target).operations.size()); - - for (std::size_t i = 0; i < timestamps.size(); ++i) { - const MemSlot* slot = target.getSlotAtTime(timestamps[i]); - const MemSlot* removedPut( - target.getSlotAtTime(timestamps[i] - Timestamp(1000))); - CPPUNIT_ASSERT(slot); - CPPUNIT_ASSERT(removedPut); - CPPUNIT_ASSERT(target.partAvailable(*slot, HEADER)); - CPPUNIT_ASSERT_EQUAL(removedPut->getLocation(HEADER), - slot->getLocation(HEADER)); - CPPUNIT_ASSERT_EQUAL(DataLocation(0, 0), slot->getLocation(BODY)); - } - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target); -} - -void MemFileV1SerializerTest::testWriteReadSingleRemovedDoc() -{ - VESPA_MEMFILEV1_SETUP_SOURCE; - source.addPutSlot(*doc, Timestamp(1001)); - source.addRemoveSlot( - *source.getSlotAtTime(Timestamp(1001)), Timestamp(2001)); - VESPA_MEMFILEV1_ASSERT_SERIALIZATION(source); -} - -/** - * Write a single put with no body to the memfile and ensure it is - * persisted properly without a body block - */ -void -MemFileV1SerializerTest::testPartialWritePutHeaderOnly() -{ - setUpPartialWriteEnvironment(); - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - document::Document::SP doc(createRandomDocumentAtLocation(4)); - { - MemFile source(file, env()); - source.addPutSlot(*doc, Timestamp(1001)); - env()._memFileMapper.flush(source, env()); - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(source); - } - { - // Have to put a second time since the first one will always - // rewrite the entire file - MemFile target(file, env()); - Document::SP doc2(createRandomDocumentAtLocation(4)); - clearBody(*doc2); - target.addPutSlot(*doc2, Timestamp(1003)); - env()._memFileMapper.flush(target, env()); - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target); - } - { - MemFile target(file, env()); - target.ensureBodyBlockCached(); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getSlotCount()); - - const MemSlot& slot = *target.getSlotAtTime(Timestamp(1003)); - CPPUNIT_ASSERT(slot.getLocation(HEADER)._pos > 0); - CPPUNIT_ASSERT(slot.getLocation(HEADER)._size > 0); - CPPUNIT_ASSERT_EQUAL( - DataLocation(0, 0), slot.getLocation(BODY)); - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target); - } -} - - - - -void -MemFileV1SerializerTest::testLocationDiskIoPlannerSimple() -{ - std::vector<MemSlot> slots; - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1001), - DataLocation(0, 1024), - DataLocation(4096, 512), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1003), - DataLocation(1024, 1024), - DataLocation(8192, 512), 0, 0)); - } - - std::vector<DataLocation> headers; - std::vector<DataLocation> bodies; - headers.push_back(slots[0].getLocation(HEADER)); - bodies.push_back(slots[0].getLocation(BODY)); - - DummyMemFileIOInterface dummyIo; - { - LocationDiskIoPlanner planner(dummyIo, HEADER, headers, 100, 0); - - CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size()); - CPPUNIT_ASSERT_EQUAL( - DataLocation(0, 1024), - planner.getIoOperations()[0]); - } - { - LocationDiskIoPlanner planner(dummyIo, BODY, bodies, 100, 4096); - - CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size()); - CPPUNIT_ASSERT_EQUAL( - DataLocation(8192, 512), // + block index - planner.getIoOperations()[0]); - } -} - -void -MemFileV1SerializerTest::testLocationDiskIoPlannerMergeReads() -{ - std::vector<MemSlot> slots; - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1001), - DataLocation(0, 1024), - DataLocation(5120, 512), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1002), - DataLocation(2048, 1024), - DataLocation(7168, 512), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1003), - DataLocation(1024, 1024), - DataLocation(9216, 512), 0, 0)); - } - - std::vector<DataLocation> headers; - std::vector<DataLocation> bodies; - for (int i = 0; i < 2; ++i) { - headers.push_back(slots[i].getLocation(HEADER)); - bodies.push_back(slots[i].getLocation(BODY)); - } - - DummyMemFileIOInterface dummyIo; - { - LocationDiskIoPlanner planner(dummyIo, HEADER, headers, 1025, 0); - - CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size()); - CPPUNIT_ASSERT_EQUAL( - DataLocation(0, 3072), - planner.getIoOperations()[0]); - } - - { - LocationDiskIoPlanner planner(dummyIo, BODY, bodies, 1025, 0); - - CPPUNIT_ASSERT_EQUAL(2, (int)planner.getIoOperations().size()); - CPPUNIT_ASSERT_EQUAL( - DataLocation(5120, 512), - planner.getIoOperations()[0]); - CPPUNIT_ASSERT_EQUAL( - DataLocation(7168, 512), - planner.getIoOperations()[1]); - } -} - -void -MemFileV1SerializerTest::testLocationDiskIoPlannerOneDocument() -{ - std::vector<MemSlot> slots; - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1001), - DataLocation(0, 1024), - DataLocation(5120, 512), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1002), - DataLocation(2048, 1024), - DataLocation(7168, 512), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1003), - DataLocation(1024, 1024), - DataLocation(9216, 512), 0, 0)); - } - - std::vector<DataLocation> headers; - std::vector<DataLocation> bodies; - headers.push_back(slots[1].getLocation(HEADER)); - bodies.push_back(slots[1].getLocation(BODY)); - - DummyMemFileIOInterface dummyIo; - { - LocationDiskIoPlanner planner(dummyIo, HEADER, headers, 1000, 0); - CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size()); - CPPUNIT_ASSERT_EQUAL( - DataLocation(2048, 1024), - planner.getIoOperations()[0]); - } - - { - LocationDiskIoPlanner planner(dummyIo, BODY, bodies, 1000, 0); - CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size()); - CPPUNIT_ASSERT_EQUAL( - DataLocation(7168, 512), - planner.getIoOperations()[0]); - } -} - -void -MemFileV1SerializerTest::testLocationDiskIoPlannerAlignReads() -{ - std::vector<MemSlot> slots; - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1001), - DataLocation(7, 100), - DataLocation(5000, 500), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1002), - DataLocation(2000, 100), - DataLocation(7000, 500), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1003), - DataLocation(110, 200), - DataLocation(9000, 500), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1004), - DataLocation(3000, 100), - DataLocation(11000, 500), 0, 0)); - } - - std::vector<DataLocation> headers; - std::vector<DataLocation> bodies; - for (int i = 0; i < 2; ++i) { - headers.push_back(slots[i].getLocation(HEADER)); - bodies.push_back(slots[i].getLocation(BODY)); - } - - DummyMemFileIOInterface dummyIo; - { - LocationDiskIoPlanner planner(dummyIo, HEADER, headers, 512, 0); - std::vector<DataLocation> expected; - expected.push_back(DataLocation(0, 512)); - expected.push_back(DataLocation(1536, 1024)); - - CPPUNIT_ASSERT_EQUAL(expected, planner.getIoOperations()); - } - { - LocationDiskIoPlanner planner(dummyIo, BODY, bodies, 512, 0); - std::vector<DataLocation> expected; - expected.push_back(DataLocation(4608, 1024)); - expected.push_back(DataLocation(6656, 1024)); - - CPPUNIT_ASSERT_EQUAL(expected, planner.getIoOperations()); - } -} - -// TODO(vekterli): add read planner test with a location cached - -void -MemFileV1SerializerTest::testSeparateReadsForHeaderAndBody() -{ - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - Document::SP doc(createRandomDocumentAtLocation(4, 0, 1000, 2000)); - { - MemFile source(file, env()); - source.addPutSlot(*doc, Timestamp(1001)); - - env()._memFileMapper.flush(source, env()); - } - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options) - .maximumReadThroughGap(1024*1024*100) - .build()); - env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>( - new LoggingLazyFile::Factory()); - - MemFile target(file, env()); - - std::vector<Timestamp> timestamps; - timestamps.push_back(Timestamp(1001)); - - getLoggerFile(target).operations.clear(); - target.ensureDocumentCached(timestamps, false); - - CPPUNIT_ASSERT_EQUAL(2, (int)getLoggerFile(target).operations.size()); - const MemSlot* slot = target.getSlotAtTime(Timestamp(1001)); - CPPUNIT_ASSERT(slot); - CPPUNIT_ASSERT(target.partAvailable(*slot, HEADER)); - CPPUNIT_ASSERT(target.partAvailable(*slot, BODY)); - CPPUNIT_ASSERT_EQUAL(*doc, *target.getDocument(*slot, ALL)); - - CPPUNIT_ASSERT(getMetrics().serialization.headerReadSize.getLast() > 0); - CPPUNIT_ASSERT(getMetrics().serialization.bodyReadSize.getLast() > 0); -} - -/** - * Write a single put with body to the memfile and ensure it is - * persisted properly with both header and body blocks - */ -void -MemFileV1SerializerTest::testPartialWritePut() -{ - setUpPartialWriteEnvironment(); - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - Document::SP doc(createRandomDocumentAtLocation(4)); - { - MemFile source(file, env()); - source.addPutSlot(*doc, Timestamp(1001)); - - env()._memFileMapper.flush(source, env()); - } - - { - // Have to put a second time since the first one will always - // rewrite the entire file - MemFile target(file, env()); - Document::SP doc2(createRandomDocumentAtLocation(4)); - target.addPutSlot(*doc2, Timestamp(1003)); - env()._memFileMapper.flush(target, env()); - } - { - MemFile target(file, env()); - target.ensureBodyBlockCached(); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getSlotCount()); - - const MemSlot& slot = *target.getSlotAtTime(Timestamp(1003)); - CPPUNIT_ASSERT(slot.getLocation(HEADER)._pos > 0); - CPPUNIT_ASSERT(slot.getLocation(HEADER)._size > 0); - - CPPUNIT_ASSERT(slot.getLocation(BODY)._size > 0); - CPPUNIT_ASSERT(slot.getLocation(BODY)._pos > 0); - } -} - -void -MemFileV1SerializerTest::doTestPartialWriteRemove(bool readAll) -{ - setUpPartialWriteEnvironment(); - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - Document::SP doc(createRandomDocumentAtLocation(4)); - { - MemFile source(file, env()); - source.addPutSlot(*doc, Timestamp(1001)); - env()._memFileMapper.flush(source, env()); - } - { - MemFile target(file, env()); - // Only populate cache before removing if explicitly told so - if (readAll) { - target.ensureBodyBlockCached(); - } - CPPUNIT_ASSERT_EQUAL(uint32_t(1), target.getSlotCount()); - target.addRemoveSlot(target[0], Timestamp(1003)); - - env()._memFileMapper.flush(target, env()); - } - { - MemFile target(file, env()); - target.ensureBodyBlockCached(); - - CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getSlotCount()); - - const MemSlot& originalSlot = target[0]; - const MemSlot& removeSlot = target[1]; - CPPUNIT_ASSERT(originalSlot.getLocation(HEADER)._size > 0); - CPPUNIT_ASSERT(originalSlot.getLocation(BODY)._size > 0); - CPPUNIT_ASSERT_EQUAL( - originalSlot.getLocation(HEADER), - removeSlot.getLocation(HEADER)); - CPPUNIT_ASSERT_EQUAL( - DataLocation(0, 0), removeSlot.getLocation(BODY)); - } -} - -/** - * Ensure that removes get the same header location as the Put - * they're removing, and that they get a zero body location - */ -void -MemFileV1SerializerTest::testPartialWriteRemoveCached() -{ - doTestPartialWriteRemove(true); -} - -void -MemFileV1SerializerTest::testPartialWriteRemoveNotCached() -{ - doTestPartialWriteRemove(false); -} - -void -MemFileV1SerializerTest::doTestPartialWriteUpdate(bool readAll) -{ - setUpPartialWriteEnvironment(); - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - Document::SP doc(createRandomDocumentAtLocation(4)); - { - MemFile source(file, env()); - source.addPutSlot(*doc, Timestamp(1001)); - env()._memFileMapper.flush(source, env()); - } - - Document::SP doc2; - { - MemFile target(file, env()); - if (readAll) { - target.ensureBodyBlockCached(); - } - - doc2.reset(new Document(*doc->getDataType(), doc->getId())); - clearBody(*doc2); - doc2->setValue(doc->getField("hstringval"), - document::StringFieldValue("Some updated content")); - - target.addUpdateSlot(*doc2, *target.getSlotAtTime(Timestamp(1001)), - Timestamp(1003)); - env()._memFileMapper.flush(target, env()); - } - - { - MemFile target(file, env()); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getSlotCount()); - const MemSlot& originalSlot = target[0]; - const MemSlot& updateSlot = target[1]; - CPPUNIT_ASSERT(originalSlot.getLocation(HEADER)._size > 0); - CPPUNIT_ASSERT(originalSlot.getLocation(BODY)._size > 0); - CPPUNIT_ASSERT_EQUAL( - originalSlot.getLocation(BODY), - updateSlot.getLocation(BODY)); - CPPUNIT_ASSERT( - updateSlot.getLocation(HEADER) - != originalSlot.getLocation(HEADER)); - - CPPUNIT_ASSERT_EQUAL(*doc, *target.getDocument(target[0], ALL)); - copyHeader(*doc, *doc2); - CPPUNIT_ASSERT_EQUAL(*doc, *target.getDocument(target[1], ALL)); - } -} - -/** - * Ensure that header updates keep the same body block - */ -void -MemFileV1SerializerTest::testPartialWriteUpdateCached() -{ - doTestPartialWriteUpdate(true); -} - -void -MemFileV1SerializerTest::testPartialWriteUpdateNotCached() -{ - doTestPartialWriteUpdate(false); -} - -void -MemFileV1SerializerTest::testPartialWriteTooMuchFreeSpace() -{ - setUpPartialWriteEnvironment(); - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - { - MemFile source(file, env()); - Document::SP doc(createRandomDocumentAtLocation(4)); - source.addPutSlot(*doc, Timestamp(1001)); - env()._memFileMapper.flush(source, env()); - } - int64_t sizeBefore; - // Append filler to slotfile to make it too big for comfort, - // forcing a rewrite to shrink it down - { - vespalib::File slotfile(file.getPath()); - slotfile.open(0); - CPPUNIT_ASSERT(slotfile.isOpen()); - sizeBefore = slotfile.getFileSize(); - slotfile.resize(sizeBefore * 20); // Well over min fill rate of 10% - } - // Write new slot to file; it should now be rewritten with the - // same file size as originally - { - MemFile source(file, env()); - Document::SP doc(createRandomDocumentAtLocation(4)); - source.addPutSlot(*doc, Timestamp(1003)); - env()._memFileMapper.flush(source, env()); - } - { - vespalib::File slotfile(file.getPath()); - slotfile.open(0); - CPPUNIT_ASSERT(slotfile.isOpen()); - CPPUNIT_ASSERT_EQUAL( - sizeBefore, - slotfile.getFileSize()); - } - CPPUNIT_ASSERT_EQUAL(uint64_t(1), getMetrics().serialization - .fullRewritesDueToDownsizingFile.getValue()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), getMetrics().serialization - .fullRewritesDueToTooSmallFile.getValue()); -} - -void -MemFileV1SerializerTest::testPartialWriteNotEnoughFreeSpace() -{ - setUpPartialWriteEnvironment(); - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - // Write file initially - MemFile source(file, env()); - { - Document::SP doc(createRandomDocumentAtLocation(4)); - source.addPutSlot(*doc, Timestamp(1001)); - env()._memFileMapper.flush(source, env()); - } - - uint32_t minFile = 1024 * 512; - auto memFileCfg = env().acquireConfigReadLock().memFilePersistenceConfig(); - resetConfig(minFile, memFileCfg->minimumFileHeaderBlockSize); - - // Create doc bigger than initial minimum filesize, - // prompting a full rewrite - Document::SP doc( - createRandomDocumentAtLocation(4, 0, 4096, 4096)); - source.addPutSlot(*doc, Timestamp(1003)); - - env()._memFileMapper.flush(source, env()); - - CPPUNIT_ASSERT_EQUAL( - minFile, - uint32_t(getFileHandle(source).getFileSize())); - - CPPUNIT_ASSERT_EQUAL(uint64_t(0), getMetrics().serialization - .fullRewritesDueToDownsizingFile.getValue()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), getMetrics().serialization - .fullRewritesDueToTooSmallFile.getValue()); - - // Now, ensure we respect minimum file size and don't try to - // "helpfully" rewrite the file again (try to detect full - // file rewrite with help from the fact we don't currently - // check whether or not the file is < the minimum filesize. - // If that changes, so must this) - memFileCfg = env().acquireConfigReadLock().memFilePersistenceConfig(); - resetConfig(2 * minFile, memFileCfg->minimumFileHeaderBlockSize); - - source.addRemoveSlot(*source.getSlotAtTime(Timestamp(1003)), - Timestamp(1005)); - env()._memFileMapper.flush(source, env()); - - CPPUNIT_ASSERT_EQUAL( - minFile, - uint32_t(getFileHandle(source).getFileSize())); - - CPPUNIT_ASSERT_EQUAL(uint64_t(1), getMetrics().serialization - .fullRewritesDueToTooSmallFile.getValue()); -} - -// Test that we don't mess up when remapping locations that -// have already been written during the same operation. That is: -// part A is remapped (P1, S1) -> (P2, S2) -// part B is remapped (P2, S2) -> (P3, S3) -// Obviously, part B should not overwrite the location of part A, -// but this will happen if we don't do the updating in one batch. -void -MemFileV1SerializerTest::testLocationsRemappedConsistently() -{ - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - - std::map<Timestamp, Document::SP> docs; - { - MemFile mf(file, env()); - Document::SP tmpDoc( - createRandomDocumentAtLocation(4, 0, 100, 100)); - - // Create docs identical in size but differing only in doc ids - // By keeping same size but inserting with _lower_ timestamps - // for docs that get higher location positions, we ensure that - // when the file is rewritten, the lower timestamp slots will - // get remapped to locations that match existing locations for - // higher timestamp slots. - for (int i = 0; i < 2; ++i) { - std::ostringstream ss; - ss << "doc" << i; - DocumentId id(document::UserDocIdString("userdoc:foo:4:" + ss.str())); - Document::SP doc(new Document(*tmpDoc->getDataType(), id)); - doc->getFields() = tmpDoc->getFields(); - mf.addPutSlot(*doc, Timestamp(1000 - i)); - docs[Timestamp(1000 - i)] = doc; - } - - env()._memFileMapper.flush(mf, env()); - // Dirty the cache for rewrite - { - DocumentId id2(document::UserDocIdString("userdoc:foo:4:doc9")); - Document::UP doc2(new Document(*tmpDoc->getDataType(), id2)); - doc2->getFields() = tmpDoc->getFields(); - mf.addPutSlot(*doc2, Timestamp(2000)); - docs[Timestamp(2000)] = std::move(doc2); - } - - // Force rewrite - auto memFileCfg = env().acquireConfigReadLock() - .memFilePersistenceConfig(); - resetConfig(1024*512, memFileCfg ->minimumFileHeaderBlockSize); - env()._memFileMapper.flush(mf, env()); - } - - MemFile target(file, env()); - target.ensureBodyBlockCached(); - - std::ostringstream err; - if (!env()._memFileMapper.verify(target, env(), err)) { - std::cerr << err.str() << "\n"; - CPPUNIT_FAIL("MemFile verification failed"); - } - - typedef std::map<Timestamp, Document::SP>::iterator Iter; - for (Iter it(docs.begin()); it != docs.end(); ++it) { - const MemSlot* slot = target.getSlotAtTime(it->first); - CPPUNIT_ASSERT(slot); - CPPUNIT_ASSERT(target.partAvailable(*slot, HEADER)); - CPPUNIT_ASSERT(target.partAvailable(*slot, BODY)); - CPPUNIT_ASSERT_EQUAL(*it->second, *target.getDocument(*slot, ALL)); - } -} - -/** - * Test that we read in the correct header information when we have to read - * in two passes to get it in its entirety. - */ -void -MemFileV1SerializerTest::testHeaderBufferTooSmall() -{ - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - FileInfo wantedInfo; - { - MemFile f(file, env()); - // 50*40 bytes of meta list data should be more than sufficient - for (size_t i = 0; i < 50; ++i) { - Document::SP doc(createRandomDocumentAtLocation(4, i)); - f.addPutSlot(*doc, Timestamp(1001 + i)); - env()._memFileMapper.flush(f, env()); - } - SimpleMemFileIOBuffer& io( - dynamic_cast<SimpleMemFileIOBuffer&>(f.getMemFileIO())); - wantedInfo = io.getFileInfo(); - } - - // Force initial index read to be too small to contain all metadata, - // triggering buffer resize and secondary read. - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options).initialIndexRead(512).build()); - { - MemFile f(file, env()); - CPPUNIT_ASSERT_EQUAL(uint32_t(50), f.getSlotCount()); - // Ensure we've read correct file info - SimpleMemFileIOBuffer& io( - dynamic_cast<SimpleMemFileIOBuffer&>(f.getMemFileIO())); - const FileInfo& info(io.getFileInfo()); - CPPUNIT_ASSERT_EQUAL(wantedInfo.getFileSize(), info.getFileSize()); - CPPUNIT_ASSERT_EQUAL(wantedInfo.getHeaderBlockStartIndex(), - info.getHeaderBlockStartIndex()); - CPPUNIT_ASSERT_EQUAL(wantedInfo.getBodyBlockStartIndex(), - info.getBodyBlockStartIndex()); - CPPUNIT_ASSERT_EQUAL(wantedInfo.getBlockSize(HEADER), - info.getBlockSize(HEADER)); - CPPUNIT_ASSERT_EQUAL(wantedInfo.getBlockSize(BODY), - info.getBlockSize(BODY)); - } -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp b/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp deleted file mode 100644 index c38842bfeb0..00000000000 --- a/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp +++ /dev/null @@ -1,496 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/memfilepersistence/mapper/memfilemapper.h> -#include <vespa/memfilepersistence/mapper/memfile_v1_serializer.h> -#include <vespa/memfilepersistence/mapper/memfile_v1_verifier.h> -#include <tests/spi/memfiletestutils.h> - -namespace storage { -namespace memfile { - -struct MemFileV1VerifierTest : public SingleDiskMemFileTestUtils -{ - void testVerify(); - - void tearDown() override; - - std::unique_ptr<MemFile> createMemFile(FileSpecification& file, - bool callLoadFile) - { - return std::unique_ptr<MemFile>(new MemFile(file, env(), callLoadFile)); - } - - CPPUNIT_TEST_SUITE(MemFileV1VerifierTest); - CPPUNIT_TEST_IGNORED(testVerify); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemFileV1VerifierTest); - -namespace { - // A totall uncached memfile with content to use for verify testing - std::unique_ptr<MemFile> _memFile; - - // Clear old content. Create new file. Make sure nothing is cached. - void prepareBucket(SingleDiskMemFileTestUtils& util, - const FileSpecification& file) { - _memFile.reset(); - util.env()._cache.clear(); - vespalib::unlink(file.getPath()); - util.createTestBucket(file.getBucketId(), 0); - util.env()._cache.clear(); - _memFile.reset(new MemFile(file, util.env())); - _memFile->getMemFileIO().close(); - - } - - // Get copy of header of memfile created - Header getHeader() { - assert(_memFile.get()); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - Header result; - file.read(&result, sizeof(Header), 0); - return result; - } - - MetaSlot getSlot(uint32_t index) { - assert(_memFile.get()); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - MetaSlot result; - file.read(&result, sizeof(MetaSlot), - sizeof(Header) + sizeof(MetaSlot) * index); - return result; - } - - void setSlot(uint32_t index, MetaSlot slot, - bool updateFileChecksum = true) - { - (void)updateFileChecksum; - assert(_memFile.get()); - //if (updateFileChecksum) slot.updateFileChecksum(); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - file.write(&slot, sizeof(MetaSlot), - sizeof(Header) + sizeof(MetaSlot) * index); - } - - void setHeader(const Header& header) { - assert(_memFile.get()); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - file.write(&header, sizeof(Header), 0); - } - - void verifySlotFile(MemFileV1VerifierTest& util, - const std::string& expectedError, - const std::string& message, - int32_t remainingEntries, - bool includeContent = true, - bool includeHeader = true) - { - assert(_memFile.get()); - FileSpecification file(_memFile->getFile()); - _memFile.reset(); - _memFile = util.createMemFile(file, false); - std::ostringstream before; - try{ - util.env()._memFileMapper.loadFile(*_memFile, util.env(), false); - _memFile->print(before, true, ""); - } catch (vespalib::Exception& e) { - before << "Unknown. Exception during loadFile\n"; - } - std::ostringstream errors; - uint32_t flags = (includeContent ? 0 : Types::DONT_VERIFY_BODY) - | (includeHeader ? 0 : Types::DONT_VERIFY_HEADER); - if (util.env()._memFileMapper.verify( - *_memFile, util.env(), errors, flags)) - { - _memFile->print(std::cerr, true, ""); - std::cerr << errors.str() << "\n"; - CPPUNIT_FAIL("verify() failed to detect: " + message); - } - CPPUNIT_ASSERT_CONTAIN_MESSAGE(message + "\nBefore: " + before.str(), - expectedError, errors.str()); - errors.str(""); - if (util.env()._memFileMapper.repair( - *_memFile, util.env(), errors, flags)) - { - CPPUNIT_FAIL("repair() failed to detect: " + message - + ": " + errors.str()); - } - CPPUNIT_ASSERT_CONTAIN_MESSAGE(message + "\nBefore: " + before.str(), - expectedError, errors.str()); - std::ostringstream remainingErrors; - if (!util.env()._memFileMapper.verify( - *_memFile, util.env(), remainingErrors, flags)) - { - CPPUNIT_FAIL("verify() returns issue after repair of: " - + message + ": " + remainingErrors.str()); - } - CPPUNIT_ASSERT_MESSAGE(remainingErrors.str(), - remainingErrors.str().size() == 0); - if (remainingEntries < 0) { - if (_memFile->fileExists()) { - CPPUNIT_FAIL(message + ": Expected file to not exist anymore"); - } - } else if (dynamic_cast<SimpleMemFileIOBuffer&>(_memFile->getMemFileIO()) - .getFileHandle().getFileSize() == 0) - { - std::ostringstream ost; - ost << "Expected " << remainingEntries << " to remain in file, " - << "but file does not exist\n"; - CPPUNIT_FAIL(message + ": " + ost.str()); - } else { - if (int64_t(_memFile->getSlotCount()) != remainingEntries) { - std::ostringstream ost; - ost << "Expected " << remainingEntries << " to remain in file, " - << "but found " << _memFile->getSlotCount() << "\n"; - ost << errors.str() << "\n"; - ost << "Before: " << before.str() << "\nAfter: "; - _memFile->print(ost, true, ""); - CPPUNIT_FAIL(message + ": " + ost.str()); - } - } - } -} - -void -MemFileV1VerifierTest::tearDown() -{ - _memFile.reset(0); - SingleDiskMemFileTestUtils::tearDown(); -}; - -void -MemFileV1VerifierTest::testVerify() -{ - BucketId bucket(16, 0xa); - std::unique_ptr<FileSpecification> file; - createTestBucket(bucket, 0); - - { - MemFilePtr memFilePtr(env()._cache.get(bucket, env(), env().getDirectory())); - file.reset(new FileSpecification(memFilePtr->getFile())); - env()._cache.clear(); - } - { // Ensure buildTestFile builds a valid file - // Initial file should be fine. - MemFile memFile(*file, env()); - std::ostringstream errors; - if (!env()._memFileMapper.verify(memFile, env(), errors)) { - memFile.print(std::cerr, false, ""); - CPPUNIT_FAIL("Slotfile failed verification: " + errors.str()); - } - } - // Header tests - prepareBucket(*this, *file); - Header orgheader(getHeader()); - { // Test wrong version - Header header(orgheader); - header.setVersion(0xc0edbabe); - header.updateChecksum(); - setHeader(header); - verifySlotFile(*this, - "400000000000000a.0 is of wrong version", - "Faulty version", - -1); - } - { // Test meta data list size bigger than file - prepareBucket(*this, *file); - Header header(orgheader); - header.setMetaDataListSize(0xFFFF); - header.updateChecksum(); - setHeader(header); - verifySlotFile(*this, - "indicates file is bigger than it physically is", - "Too big meta data list size", - -1); - } - { // Test header block size bigger than file - prepareBucket(*this, *file); - Header header(orgheader); - header.setHeaderBlockSize(0xFFFF); - header.updateChecksum(); - setHeader(header); - verifySlotFile(*this, - "Header indicates file is bigger than it physically is", - "Too big header block size", - -1); - } - { // Test wrong header crc - prepareBucket(*this, *file); - Header header(orgheader); - header.setMetaDataListSize(4); - setHeader(header); - verifySlotFile(*this, - "Header checksum mismatch", - "Wrong header checksum", - -1); - } - // Meta data tests - prepareBucket(*this, *file); - MetaSlot slot6(getSlot(6)); - { // Test extra removes - currently allowed - MetaSlot slot7(getSlot(7)); - MetaSlot s(slot7); - s.setTimestamp(Timestamp(s._timestamp.getTime() - 1)); - s.updateChecksum(); - setSlot(6, s); - s.setTimestamp(Timestamp(s._timestamp.getTime() + 1)); - s.updateChecksum(); - setSlot(7, s); - std::ostringstream errors; - if (!env()._memFileMapper.verify(*_memFile, env(), errors)) { - _memFile->print(std::cerr, false, ""); - std::cerr << errors.str() << "\n"; - CPPUNIT_FAIL("Supposed to be legal with multiple remove values"); - } - setSlot(7, slot7); - } - { - // Test metadata crc mismatch with "used" flag being accidentally - // flipped. Should not inhibit adding of subsequent slots. - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setUseFlag(false); - setSlot(6, s); - verifySlotFile(*this, - "Slot 6 at timestamp 2001 failed checksum verification", - "Crc failure with use flag", 23, false); - } - { // Test overlapping documents - MetaSlot s(slot6); - // Direct overlapping header - prepareBucket(*this, *file); - s.setHeaderPos(0); - s.setHeaderSize(51); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "overlaps with slot", - "Direct overlapping header", 6, false, false); - // Contained header - // (contained bit not valid header so fails on other error now) - prepareBucket(*this, *file); - s.setHeaderPos(176); - s.setHeaderSize(80); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "not big enough to contain a document id", - "Contained header", 7, false); - // Partly overlapping header - // (contained bit not valid header so fails on other error now) - prepareBucket(*this, *file); - s.setHeaderPos(191); - s.setHeaderSize(35); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "not big enough to contain a document id", - "Partly overlapping header", 7, false); - prepareBucket(*this, *file); - s.setHeaderPos(185); - s.setHeaderSize(33); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "not big enough to contain a document id", - "Partly overlapping header (2)", 7, false); - // Direct overlapping body - prepareBucket(*this, *file); - s = slot6; - s.setBodyPos(0); - s.setBodySize(136); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Multiple slots with different gids use same body position", - "Directly overlapping body", 6, false); - // Contained body - prepareBucket(*this, *file); - s.setBodyPos(10); - s.setBodySize(50); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "overlaps with slot", - "Contained body", 6, false); - CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(1)) == 0); - // Overlapping body - prepareBucket(*this, *file); - s.setBodyPos(160); - s.setBodySize(40); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "overlaps with slot", - "Overlapping body", 5, false); - CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(2)) == 0); - CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(1501)) == 0); - // Overlapping body, verifying bodies - // (Bad body bit should be removed first, so only one slot needs - // removing) - prepareBucket(*this, *file); - setSlot(6, s); - verifySlotFile(*this, - "Body checksum mismatch", - "Overlapping body(2)", 7, true); - } - { // Test out of bounds - MetaSlot s(slot6); - - // Header out of bounds - prepareBucket(*this, *file); - s.setHeaderPos(500); - s.setHeaderSize(5000); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "goes out of bounds", - "Header out of bounds", 7, false, false); - // Body out of bounds - prepareBucket(*this, *file); - s = slot6; - s.setBodyPos(2400); - s.setBodySize(6000); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "goes out of bounds", - "Body out of bounds", 7, false); - } - { // Test timestamp collision - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setTimestamp(Timestamp(10002)); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "has same timestamp as slot 5", - "Timestamp collision", 6, false); - } - { // Test timestamp out of order - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setTimestamp(Timestamp(38)); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Slot 6 is out of timestamp order", - "Timestamp out of order", 8, false); - } - { // Test metadata crc mismatch - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setTimestamp(Timestamp(40)); - setSlot(6, s); - verifySlotFile(*this, - "Slot 6 at timestamp 40 failed checksum verification", - "Crc failure", 7, false); - } - { // Test used after unused - // This might actually lose documents after the unused entries. - // The memfile will not know about the documents after unused entry. - // If the memfile contains changes and writes metadata back due to this, - // the following entries will be missing. - // (To prevent this repair would have to add metadata entries, but that - // may be problems if repair happens at a time where all header or body - // data in the file needs to be cached.) - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setUseFlag(false); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Slot 7 found after unused entries", - "Used after unused", 6, false); - } - { // Test header blob corrupt - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setHeaderPos(519); - s.setHeaderSize(86); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Header checksum mismatch", - "Corrupt header blob.", 7); - } - { // Test body blob corrupt - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setBodyPos(52); - s.setBodySize(18); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Body checksum mismatch", - "Corrupt body blob.", 7); - } - { // Test too long name for header chunk - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setHeaderPos(160); - s.setHeaderSize(33); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "header is not big enough to contain a document", - "Too long name in header.", 7); - } - { // Test wrong file checksum -// Currently disabled. Currently only possible to calculate file checksum from -// memfile now, and memfile object wont be valid. -/* - // First test if we actually have less entries at all.. - prepareBucket(*this, *file); - MetaSlot s(getSlot(7)); - s.setUseFlag(false); - s.updateChecksum(); - setSlot(7, s, false); - s = getSlot(8); - s.setUseFlag(false); - s.updateChecksum(); - setSlot(8, s, false); - verifySlotFile(*this, - "File checksum should have been", - "Wrong file checksum in file.", 7, false); -std::cerr << "U\n"; - // Then test with different timestamp in remaining document - prepareBucket(*this, *file); - s = getSlot(6); - s.setTimestamp(s._timestamp + 1); - s.updateChecksum(); - setSlot(6, s, false); - verifySlotFile(*this, - "File checksum should have been", - "Wrong file checksum in file.", 9, false); -std::cerr << "V\n"; - // Then check with different gid - prepareBucket(*this, *file); - s = getSlot(6); - s._gid = GlobalId("sdfsdfsedsdfsdfsd"); - s.updateChecksum(); - setSlot(6, s, false); - verifySlotFile(*this, - "File checksum should have been", - "Wrong file checksum in file.", 9, false, false); -*/ - } - { // Test that documents not belonging in a bucket is removed -// Currently disabled. Hard to test. Needs total rewrite -/* - prepareBucket(*this, *file); - Blob b(createBlob(43u, "userdoc::0:315", "header", "body")); - _memFile->write(b, 80); - CPPUNIT_ASSERT_EQUAL(4u, _memFile->getBlobCount()); - CPPUNIT_ASSERT(_memFile->read(b)); - verifySlotFile(*this, - "belongs in bucket", - "Document not belonging there", 9); - CPPUNIT_ASSERT_EQUAL(3u, _memFile->getBlobCount()); -*/ - } -} - -} -} diff --git a/memfilepersistence/src/tests/spi/options_builder.h b/memfilepersistence/src/tests/spi/options_builder.h deleted file mode 100644 index 7f04a02086c..00000000000 --- a/memfilepersistence/src/tests/spi/options_builder.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/memfilepersistence/common/environment.h> -#include <vespa/vespalib/stllike/string.h> -#include <memory> - -namespace storage { -namespace memfile { - -class OptionsBuilder -{ - Options _newOptions; -public: - OptionsBuilder(const Options& opts) - : _newOptions(opts) - { - } - - OptionsBuilder& maximumReadThroughGap(uint32_t readThroughGap) { - _newOptions._maximumGapToReadThrough = readThroughGap; - return *this; - } - - OptionsBuilder& initialIndexRead(uint32_t bytesToRead) { - _newOptions._initialIndexRead = bytesToRead; - return *this; - } - - OptionsBuilder& revertTimePeriod(framework::MicroSecTime revertTime) { - _newOptions._revertTimePeriod = revertTime; - return *this; - } - - OptionsBuilder& defaultRemoveDocType(vespalib::stringref typeName) { - _newOptions._defaultRemoveDocType = typeName; - return *this; - } - - OptionsBuilder& maxDocumentVersions(uint32_t maxVersions) { - _newOptions._maxDocumentVersions = maxVersions; - return *this; - } - - std::unique_ptr<Options> build() const { - return std::unique_ptr<Options>(new Options(_newOptions)); - } -}; - -} // memfile -} // storage - diff --git a/memfilepersistence/src/tests/spi/providerconformancetest.cpp b/memfilepersistence/src/tests/spi/providerconformancetest.cpp deleted file mode 100644 index 7ba91bde619..00000000000 --- a/memfilepersistence/src/tests/spi/providerconformancetest.cpp +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memfiletestutils.h" -#include <vespa/persistence/conformancetest/conformancetest.h> - -namespace storage { -namespace memfile { - -struct ProviderConformanceTest : public spi::ConformanceTest { - struct Factory : public PersistenceFactory { - framework::defaultimplementation::ComponentRegisterImpl _compRegister; - framework::defaultimplementation::RealClock _clock; - std::unique_ptr<MemFileCache> cache; - - Factory() - : _compRegister(), - _clock() - { - _compRegister.setClock(_clock); - } - - spi::PersistenceProvider::UP - getPersistenceImplementation(const std::shared_ptr<const document::DocumentTypeRepo>& repo, - const document::DocumenttypesConfig&) override - { - system("rm -rf vdsroot"); - system("mkdir -p vdsroot/disks/d0"); - vdstestlib::DirConfig config(getStandardConfig(true)); - - MemFilePersistenceProvider::UP result( - new MemFilePersistenceProvider( - _compRegister, - config.getConfigId())); - result->setDocumentRepo(*repo); - return spi::PersistenceProvider::UP(result.release()); - } - - bool - supportsRevert() const - { - return true; - } - }; - - ProviderConformanceTest() - : spi::ConformanceTest(PersistenceFactory::UP(new Factory)) {} - - CPPUNIT_TEST_SUITE(ProviderConformanceTest); - DEFINE_CONFORMANCE_TESTS(); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(ProviderConformanceTest); - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/spi/shared_data_location_tracker_test.cpp b/memfilepersistence/src/tests/spi/shared_data_location_tracker_test.cpp deleted file mode 100644 index ee95cc0026d..00000000000 --- a/memfilepersistence/src/tests/spi/shared_data_location_tracker_test.cpp +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/memfilepersistence/memfile/shared_data_location_tracker.h> - -namespace storage { -namespace memfile { - -class SharedDataLocationTrackerTest : public CppUnit::TestFixture -{ -public: - void headerIsPassedDownToCacheAccessor(); - void bodyIsPassedDownToCacheAccessor(); - void firstInvocationReturnsNewLocation(); - void multipleInvocationsForSharedSlotReturnSameLocation(); - - CPPUNIT_TEST_SUITE(SharedDataLocationTrackerTest); - CPPUNIT_TEST(headerIsPassedDownToCacheAccessor); - CPPUNIT_TEST(bodyIsPassedDownToCacheAccessor); - CPPUNIT_TEST(firstInvocationReturnsNewLocation); - CPPUNIT_TEST(multipleInvocationsForSharedSlotReturnSameLocation); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(SharedDataLocationTrackerTest); - -namespace { - -using Params = std::pair<Types::DocumentPart, DataLocation>; -constexpr auto HEADER = Types::HEADER; -constexpr auto BODY = Types::BODY; - -/** - * A simple mock of a buffer cache which records all invocations - * and returns a location increasing by 100 for each invocation. - */ -struct MockBufferCacheCopier : BufferCacheCopier -{ - // This is practically _screaming_ for GoogleMock. - std::vector<Params> invocations; - - DataLocation doCopyFromSourceToLocal( - Types::DocumentPart part, - DataLocation sourceLocation) override - { - Params params(part, sourceLocation); - const size_t invocationsBefore = invocations.size(); - invocations.push_back(params); - return DataLocation(invocationsBefore * 100, - invocationsBefore * 100 + 100); - } -}; - -} - -void -SharedDataLocationTrackerTest::headerIsPassedDownToCacheAccessor() -{ - MockBufferCacheCopier cache; - SharedDataLocationTracker tracker(cache, HEADER); - tracker.getOrCreateSharedLocation({0, 100}); - CPPUNIT_ASSERT_EQUAL(size_t(1), cache.invocations.size()); - CPPUNIT_ASSERT_EQUAL(Params(HEADER, {0, 100}), cache.invocations[0]); -} - -void -SharedDataLocationTrackerTest::bodyIsPassedDownToCacheAccessor() -{ - MockBufferCacheCopier cache; - SharedDataLocationTracker tracker(cache, BODY); - tracker.getOrCreateSharedLocation({0, 100}); - CPPUNIT_ASSERT_EQUAL(size_t(1), cache.invocations.size()); - CPPUNIT_ASSERT_EQUAL(Params(BODY, {0, 100}), cache.invocations[0]); -} - -void -SharedDataLocationTrackerTest::firstInvocationReturnsNewLocation() -{ - MockBufferCacheCopier cache; - SharedDataLocationTracker tracker(cache, HEADER); - // Auto-incrementing per cache copy invocation. - CPPUNIT_ASSERT_EQUAL(DataLocation(0, 100), - tracker.getOrCreateSharedLocation({500, 600})); - CPPUNIT_ASSERT_EQUAL(DataLocation(100, 200), - tracker.getOrCreateSharedLocation({700, 800})); - - CPPUNIT_ASSERT_EQUAL(size_t(2), cache.invocations.size()); - CPPUNIT_ASSERT_EQUAL(Params(HEADER, {500, 600}), cache.invocations[0]); - CPPUNIT_ASSERT_EQUAL(Params(HEADER, {700, 800}), cache.invocations[1]); -} - -void -SharedDataLocationTrackerTest - ::multipleInvocationsForSharedSlotReturnSameLocation() -{ - MockBufferCacheCopier cache; - SharedDataLocationTracker tracker(cache, HEADER); - CPPUNIT_ASSERT_EQUAL(DataLocation(0, 100), - tracker.getOrCreateSharedLocation({500, 600})); - // Same source location, thus we can reuse the same destination location - // as well. - CPPUNIT_ASSERT_EQUAL(DataLocation(0, 100), - tracker.getOrCreateSharedLocation({500, 600})); - - CPPUNIT_ASSERT_EQUAL(size_t(1), cache.invocations.size()); - CPPUNIT_ASSERT_EQUAL(Params(HEADER, {500, 600}), cache.invocations[0]); -} - -} // memfile -} // storage - diff --git a/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp b/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp deleted file mode 100644 index 7701df98661..00000000000 --- a/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp +++ /dev/null @@ -1,656 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <tests/spi/memfiletestutils.h> -#include <tests/spi/options_builder.h> - -namespace storage { -namespace memfile { - -class SimpleMemFileIOBufferTest : public SingleDiskMemFileTestUtils -{ - CPPUNIT_TEST_SUITE(SimpleMemFileIOBufferTest); - CPPUNIT_TEST(testAddAndReadDocument); - CPPUNIT_TEST(testNonExistingLocation); - CPPUNIT_TEST(testCopy); - CPPUNIT_TEST(testCacheLocation); - CPPUNIT_TEST(testPersist); - CPPUNIT_TEST(testGetSerializedSize); - CPPUNIT_TEST(testRemapLocations); - CPPUNIT_TEST(testAlignmentUtilFunctions); - CPPUNIT_TEST(testCalculatedCacheSize); - CPPUNIT_TEST(testSharedBuffer); - CPPUNIT_TEST(testSharedBufferUsage); - CPPUNIT_TEST(testHeaderChunkEncoderComputesSizesCorrectly); - CPPUNIT_TEST(testHeaderChunkEncoderSerializesIdCorrectly); - CPPUNIT_TEST(testHeaderChunkEncoderSerializesHeaderCorrectly); - CPPUNIT_TEST(testRemovesCanBeWrittenWithBlankDefaultDocument); - CPPUNIT_TEST(testRemovesCanBeWrittenWithIdInferredDoctype); - CPPUNIT_TEST(testRemovesWithInvalidDocTypeThrowsException); - CPPUNIT_TEST_SUITE_END(); - - using BufferType = SimpleMemFileIOBuffer::BufferType; - using BufferSP = BufferType::SP; - using BufferAllocation = SimpleMemFileIOBuffer::BufferAllocation; - using HeaderChunkEncoder = SimpleMemFileIOBuffer::HeaderChunkEncoder; - using SimpleMemFileIOBufferUP = std::unique_ptr<SimpleMemFileIOBuffer>; - - BufferAllocation allocateBuffer(size_t sz) { - return BufferAllocation(BufferSP(new BufferType(sz)), 0, sz); - } - - /** - * Create an I/O buffer instance with for a dummy bucket. If removeDocType - * is non-empty, remove entries will be written in backwards compatible - * mode. - */ - SimpleMemFileIOBufferUP createIoBufferWithDummySpec( - vespalib::stringref removeDocType = ""); - -public: - class DummyFileReader : public VersionSerializer { - public: - FileVersion getFileVersion() override { return FileVersion(); } - void loadFile(MemFile&, Environment&, Buffer&, uint64_t ) override {} - FlushResult flushUpdatesToFile(MemFile&, Environment&) override { - return FlushResult::TooSmall; - } - void rewriteFile(MemFile&, Environment&) override {} - bool verify(MemFile&, Environment&, std::ostream&, bool, uint16_t) override { return false; }; - void cacheLocations(MemFileIOInterface&, Environment&, const Options&, - DocumentPart, const std::vector<DataLocation>&) override {} - }; - - DummyFileReader dfr; - - void testAddAndReadDocument(); - void testNonExistingLocation(); - void testCopy(); - void testCacheLocation(); - void testPersist(); - void testGetSerializedSize(); - void testRemapLocations(); - void testAlignmentUtilFunctions(); - void testCalculatedCacheSize(); - void testSharedBuffer(); - void testSharedBufferUsage(); - void testHeaderChunkEncoderComputesSizesCorrectly(); - void testHeaderChunkEncoderSerializesIdCorrectly(); - void testHeaderChunkEncoderSerializesHeaderCorrectly(); - void testRemovesCanBeWrittenWithBlankDefaultDocument(); - void testRemovesCanBeWrittenWithIdInferredDoctype(); - void testRemovesWithInvalidDocTypeThrowsException(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(SimpleMemFileIOBufferTest); - - -void -SimpleMemFileIOBufferTest::testAddAndReadDocument() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 456, - 789, - 1234)); - - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr<FileInfo>(new FileInfo), - fileSpec, - env()); - - DataLocation h = buffer.addHeader(*doc); - DataLocation b = buffer.addBody(*doc); - - Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), h); - buffer.readBody(*getTypeRepo(), b, *newDoc); - - CPPUNIT_ASSERT_EQUAL(*doc, *newDoc); - CPPUNIT_ASSERT_EQUAL(true, buffer.isCached(h, HEADER)); - CPPUNIT_ASSERT_EQUAL(true, buffer.isCached(b, BODY)); - CPPUNIT_ASSERT_EQUAL(false, buffer.isCached(h, BODY)); - CPPUNIT_ASSERT_EQUAL(false, buffer.isCached(b, HEADER)); - CPPUNIT_ASSERT_EQUAL(doc->getId(), buffer.getDocumentId(h)); -} - -void -SimpleMemFileIOBufferTest::testPersist() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 456, - 789, - 1234)); - - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr<FileInfo>(new FileInfo), - fileSpec, - env()); - - DataLocation h = buffer.addHeader(*doc); - DataLocation b = buffer.addBody(*doc); - - CPPUNIT_ASSERT(!buffer.isPersisted(h, HEADER)); - CPPUNIT_ASSERT(!buffer.isPersisted(b, BODY)); - - buffer.persist(HEADER, h, DataLocation(1000, h.size())); - buffer.persist(BODY, b, DataLocation(5000, b.size())); - - Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), DataLocation(1000, h.size())); - buffer.readBody(*getTypeRepo(), DataLocation(5000, b.size()), *newDoc); - - CPPUNIT_ASSERT(buffer.isPersisted(DataLocation(1000, h.size()), HEADER)); - CPPUNIT_ASSERT(buffer.isPersisted(DataLocation(5000, b.size()), BODY)); - - CPPUNIT_ASSERT_EQUAL(*doc, *newDoc); -} - -void -SimpleMemFileIOBufferTest::testCopy() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr<FileInfo>(new FileInfo), - fileSpec, - env()); - - for (uint32_t i = 0; i < 10; ++i) { - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 456, - 789, - 1234)); - - DataLocation h = buffer.addHeader(*doc); - DataLocation b = buffer.addBody(*doc); - - SimpleMemFileIOBuffer buffer2(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr<FileInfo>(new FileInfo), - fileSpec, - env()); - - DataLocation h2 = buffer2.copyCache(buffer, HEADER, h); - DataLocation b2 = buffer2.copyCache(buffer, BODY, b); - - Document::UP newDoc = buffer2.getDocumentHeader(*getTypeRepo(), h2); - buffer2.readBody(*getTypeRepo(), b2, *newDoc); - - CPPUNIT_ASSERT_EQUAL(*doc, *newDoc); - } -} - -void -SimpleMemFileIOBufferTest::testNonExistingLocation() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 456, - 789, - 1234)); - - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr<FileInfo>(new FileInfo), - fileSpec, - env()); - - DataLocation h = buffer.addHeader(*doc); - DataLocation b = buffer.addBody(*doc); - - buffer.clear(HEADER); - - try { - Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), h); - CPPUNIT_ASSERT(false); - } catch (SimpleMemFileIOBuffer::PartNotCachedException& e) { - } - - buffer.clear(BODY); - - try { - document::Document newDoc; - buffer.readBody(*getTypeRepo(), b, newDoc); - CPPUNIT_ASSERT(false); - } catch (SimpleMemFileIOBuffer::PartNotCachedException& e) { - } -} - -void -SimpleMemFileIOBufferTest::testCacheLocation() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - FileInfo::UP(new FileInfo(100, 10000, 50000)), - fileSpec, - env()); - - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 456, - 789, - 1234)); - - BufferAllocation headerBuf = buffer.serializeHeader(*doc); - BufferAllocation bodyBuf = buffer.serializeBody(*doc); - - DataLocation hloc(1234, headerBuf.getSize()); - DataLocation bloc(5678, bodyBuf.getSize()); - - buffer.cacheLocation(HEADER, hloc, headerBuf.getSharedBuffer(), 0); - buffer.cacheLocation(BODY, bloc, bodyBuf.getSharedBuffer(), 0); - - Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), hloc); - buffer.readBody(*getTypeRepo(), bloc, *newDoc); - - CPPUNIT_ASSERT_EQUAL(*doc, *newDoc); -} - -void -SimpleMemFileIOBufferTest::testGetSerializedSize() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - FileInfo::UP(new FileInfo(100, 10000, 50000)), - fileSpec, - env()); - - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 456, - 789, - 1234)); - - BufferAllocation headerBuf = buffer.serializeHeader(*doc); - BufferAllocation bodyBuf = buffer.serializeBody(*doc); - - DataLocation hloc(1234, headerBuf.getSize()); - DataLocation bloc(5678, bodyBuf.getSize()); - - buffer.cacheLocation(HEADER, hloc, headerBuf.getSharedBuffer(), 0); - buffer.cacheLocation(BODY, bloc, bodyBuf.getSharedBuffer(), 0); - - vespalib::nbostream serializedHeader; - doc->serializeHeader(serializedHeader); - - vespalib::nbostream serializedBody; - doc->serializeBody(serializedBody); - - CPPUNIT_ASSERT_EQUAL(uint32_t(serializedHeader.size()), - buffer.getSerializedSize(HEADER, hloc)); - CPPUNIT_ASSERT_EQUAL(uint32_t(serializedBody.size()), - buffer.getSerializedSize(BODY, bloc)); -} - -// Test that remapping does not overwrite datalocations that it has -// already updated -void -SimpleMemFileIOBufferTest::testRemapLocations() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - FileInfo::UP(new FileInfo(100, 10000, 50000)), - fileSpec, - env()); - - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 100, - 100)); - BufferAllocation headerBuf = buffer.serializeHeader(*doc); - BufferAllocation bodyBuf = buffer.serializeBody(*doc); - - document::Document::SP doc2(createRandomDocumentAtLocation( - 123, - 100, - 100)); - - BufferAllocation headerBuf2 = buffer.serializeHeader(*doc2); - BufferAllocation bodyBuf2 = buffer.serializeBody(*doc2); - - DataLocation hloc(30000, headerBuf.getSize()); - DataLocation hloc2(0, headerBuf2.getSize()); - DataLocation hloc3(10000, hloc2._size); - - buffer.cacheLocation(HEADER, hloc, headerBuf.getSharedBuffer(), 0); - buffer.cacheLocation(HEADER, hloc2, headerBuf2.getSharedBuffer(), 0); - - std::map<DataLocation, DataLocation> remapping; - remapping[hloc2] = hloc; - remapping[hloc] = hloc3; - - buffer.remapAndPersistAllLocations(HEADER, remapping); - - Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), hloc3); - document::ByteBuffer bbuf(bodyBuf.getBuffer(), bodyBuf.getSize()); - newDoc->deserializeBody(*getTypeRepo(), bbuf); - - CPPUNIT_ASSERT_EQUAL(*doc, *newDoc); - - Document::UP newDoc2 = buffer.getDocumentHeader(*getTypeRepo(), hloc); - document::ByteBuffer bbuf2(bodyBuf.getBuffer(), bodyBuf.getSize()); - newDoc2->deserializeBody(*getTypeRepo(), bbuf2); - CPPUNIT_ASSERT_EQUAL(*doc2, *newDoc2); -} - -/** - * Not technically a part of SimpleMemFileIOBuffer, but used by it and - * currently contained within its header file. Move test somewhere else - * if the code itself is moved. - */ -void -SimpleMemFileIOBufferTest::testAlignmentUtilFunctions() -{ - using namespace util; - CPPUNIT_ASSERT_EQUAL(size_t(0), alignUpPow2<4096>(0)); - CPPUNIT_ASSERT_EQUAL(size_t(4096), alignUpPow2<4096>(1)); - CPPUNIT_ASSERT_EQUAL(size_t(4096), alignUpPow2<4096>(512)); - CPPUNIT_ASSERT_EQUAL(size_t(4096), alignUpPow2<4096>(4096)); - CPPUNIT_ASSERT_EQUAL(size_t(8192), alignUpPow2<4096>(4097)); - CPPUNIT_ASSERT_EQUAL(size_t(32), alignUpPow2<16>(20)); - CPPUNIT_ASSERT_EQUAL(size_t(32), alignUpPow2<32>(20)); - CPPUNIT_ASSERT_EQUAL(size_t(64), alignUpPow2<64>(20)); - CPPUNIT_ASSERT_EQUAL(size_t(128), alignUpPow2<128>(20)); - - CPPUNIT_ASSERT_EQUAL(uint32_t(0), nextPow2(0)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), nextPow2(1)); - CPPUNIT_ASSERT_EQUAL(uint32_t(4), nextPow2(3)); - CPPUNIT_ASSERT_EQUAL(uint32_t(16), nextPow2(15)); - CPPUNIT_ASSERT_EQUAL(uint32_t(64), nextPow2(40)); - CPPUNIT_ASSERT_EQUAL(uint32_t(64), nextPow2(64)); -} - -/** - * Test that allocated buffers are correctly reported with their sizes - * rounded up to account for mmap overhead. - */ -void -SimpleMemFileIOBufferTest::testCalculatedCacheSize() -{ - FileSpecification fileSpec(BucketId(16, 123), - env().getDirectory(), "testfile.0"); - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr<FileInfo>(new FileInfo), - fileSpec, - env()); - - CPPUNIT_ASSERT_EQUAL(size_t(0), buffer.getCachedSize(HEADER)); - CPPUNIT_ASSERT_EQUAL(size_t(0), buffer.getCachedSize(BODY)); - - // All buffers are on a 4k page granularity. - BufferAllocation sharedHeaderBuffer(allocateBuffer(1500)); // -> 4096 - buffer.cacheLocation(HEADER, DataLocation(0, 85), - sharedHeaderBuffer.getSharedBuffer(), 0); - CPPUNIT_ASSERT_EQUAL(size_t(4096), buffer.getCachedSize(HEADER)); - - buffer.cacheLocation(HEADER, DataLocation(200, 100), - sharedHeaderBuffer.getSharedBuffer(), 85); - CPPUNIT_ASSERT_EQUAL(size_t(4096), buffer.getCachedSize(HEADER)); - - BufferAllocation singleHeaderBuffer(allocateBuffer(200)); // -> 4096 - buffer.cacheLocation(HEADER, DataLocation(0, 100), - singleHeaderBuffer.getSharedBuffer(), 0); - CPPUNIT_ASSERT_EQUAL(size_t(8192), buffer.getCachedSize(HEADER)); - - BufferAllocation singleBodyBuffer(allocateBuffer(300)); // -> 4096 - buffer.cacheLocation(BODY, DataLocation(0, 100), - singleBodyBuffer.getSharedBuffer(), 0); - CPPUNIT_ASSERT_EQUAL(size_t(4096), buffer.getCachedSize(BODY)); - - buffer.clear(HEADER); - CPPUNIT_ASSERT_EQUAL(size_t(0), buffer.getCachedSize(HEADER)); - - buffer.clear(BODY); - CPPUNIT_ASSERT_EQUAL(size_t(0), buffer.getCachedSize(BODY)); -} - -void -SimpleMemFileIOBufferTest::testSharedBuffer() -{ - typedef SimpleMemFileIOBuffer::SharedBuffer SharedBuffer; - - { - SharedBuffer buf(1024); - CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getSize()); - CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.getUsedSize()); - CPPUNIT_ASSERT(buf.hasRoomFor(1024)); - CPPUNIT_ASSERT(!buf.hasRoomFor(1025)); - - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.allocate(13)); - // Allocation should be rounded up to nearest alignment. - // TODO: is this even necessary? - CPPUNIT_ASSERT_EQUAL(size_t(16), buf.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(size_t(1008), buf.getFreeSize()); - CPPUNIT_ASSERT(buf.hasRoomFor(1008)); - CPPUNIT_ASSERT(!buf.hasRoomFor(1009)); - CPPUNIT_ASSERT_EQUAL(size_t(16), buf.allocate(1)); - CPPUNIT_ASSERT_EQUAL(size_t(24), buf.getUsedSize()); - - CPPUNIT_ASSERT_EQUAL(size_t(24), buf.allocate(999)); - CPPUNIT_ASSERT(!buf.hasRoomFor(1)); - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getUsedSize()); - } - // Test exact fit. - { - SharedBuffer buf(1024); - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.allocate(1024)); - CPPUNIT_ASSERT(!buf.hasRoomFor(1)); - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getUsedSize()); - } - // Test 512-byte alignment. - { - SharedBuffer buf(1024); - CPPUNIT_ASSERT(buf.hasRoomFor(1000, SharedBuffer::ALIGN_512_BYTES)); - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.allocate(10)); - CPPUNIT_ASSERT(!buf.hasRoomFor(1000, SharedBuffer::ALIGN_512_BYTES)); - CPPUNIT_ASSERT(!buf.hasRoomFor(513, SharedBuffer::ALIGN_512_BYTES)); - CPPUNIT_ASSERT(buf.hasRoomFor(512, SharedBuffer::ALIGN_512_BYTES)); - CPPUNIT_ASSERT_EQUAL(size_t(512), buf.allocate(512, SharedBuffer::ALIGN_512_BYTES)); - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getUsedSize()); - } -} - -void -SimpleMemFileIOBufferTest::testSharedBufferUsage() -{ - FileSpecification fileSpec(BucketId(16, 123), - env().getDirectory(), "testfile.0"); - SimpleMemFileIOBuffer ioBuf(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr<FileInfo>(new FileInfo), - fileSpec, - env()); - - const size_t threshold = SimpleMemFileIOBuffer::WORKING_BUFFER_SIZE; - - // Brand new allocation - BufferAllocation ba(ioBuf.allocateBuffer(HEADER, 1)); - CPPUNIT_ASSERT(ba.buf.get()); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), ba.pos); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), ba.size); - // Should reuse buffer, but get other offset - BufferAllocation ba2(ioBuf.allocateBuffer(HEADER, 500)); - CPPUNIT_ASSERT_EQUAL(ba.buf.get(), ba2.buf.get()); - CPPUNIT_ASSERT_EQUAL(uint32_t(8), ba2.pos); - CPPUNIT_ASSERT_EQUAL(uint32_t(500), ba2.size); - CPPUNIT_ASSERT_EQUAL(size_t(512), ba2.buf->getUsedSize()); - - // Allocate a buffer so big that it should get its own buffer instance - BufferAllocation ba3(ioBuf.allocateBuffer(HEADER, threshold)); - CPPUNIT_ASSERT(ba3.buf.get() != ba2.buf.get()); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), ba3.pos); - CPPUNIT_ASSERT_EQUAL(uint32_t(threshold), ba3.size); - - // But smaller allocs should still be done from working buffer - BufferAllocation ba4(ioBuf.allocateBuffer(HEADER, 512)); - CPPUNIT_ASSERT_EQUAL(ba.buf.get(), ba4.buf.get()); - CPPUNIT_ASSERT_EQUAL(uint32_t(512), ba4.pos); - CPPUNIT_ASSERT_EQUAL(uint32_t(512), ba4.size); - CPPUNIT_ASSERT_EQUAL(size_t(1024), ba4.buf->getUsedSize()); - - // Allocate lots of smaller buffers from the same buffer until we run out. - while (true) { - BufferAllocation tmp(ioBuf.allocateBuffer(HEADER, 1024)); - CPPUNIT_ASSERT_EQUAL(ba.buf.get(), tmp.buf.get()); - if (!tmp.buf->hasRoomFor(2048)) { - break; - } - } - BufferAllocation ba5(ioBuf.allocateBuffer(HEADER, 2048)); - CPPUNIT_ASSERT(ba5.buf.get() != ba.buf.get()); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), ba5.pos); - CPPUNIT_ASSERT_EQUAL(uint32_t(2048), ba5.size); - - // Allocating for different part should get different buffer. - BufferAllocation ba6(ioBuf.allocateBuffer(BODY, 128)); - CPPUNIT_ASSERT(ba6.buf.get() != ba5.buf.get()); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), ba6.pos); - CPPUNIT_ASSERT_EQUAL(uint32_t(128), ba6.size); -} - -void -SimpleMemFileIOBufferTest::testHeaderChunkEncoderComputesSizesCorrectly() -{ - document::Document::SP doc(createRandomDocumentAtLocation(123, 100, 100)); - - std::string idString = doc->getId().toString(); - HeaderChunkEncoder encoder(doc->getId()); - // Without document, payload is: 3x u32 + doc id string (no zero term). - CPPUNIT_ASSERT_EQUAL(sizeof(uint32_t)*3 + idString.size(), - static_cast<size_t>(encoder.encodedSize())); - - encoder.bufferDocument(*doc); - vespalib::nbostream stream; - doc->serializeHeader(stream); - // With document, add size of serialized document to the mix. - CPPUNIT_ASSERT_EQUAL(sizeof(uint32_t)*3 + idString.size() + stream.size(), - static_cast<size_t>(encoder.encodedSize())); -} - -SimpleMemFileIOBufferTest::SimpleMemFileIOBufferUP -SimpleMemFileIOBufferTest::createIoBufferWithDummySpec( - vespalib::stringref removeDocType) -{ - FileSpecification fileSpec(BucketId(16, 123), - env().getDirectory(), "testfile.0"); - // Override config. - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options) - .defaultRemoveDocType(removeDocType) - .build()); - - SimpleMemFileIOBufferUP ioBuf( - new SimpleMemFileIOBuffer( - dfr, - vespalib::LazyFile::UP(), - std::unique_ptr<FileInfo>(new FileInfo), - fileSpec, - env())); - return ioBuf; -} - -void -SimpleMemFileIOBufferTest::testHeaderChunkEncoderSerializesIdCorrectly() -{ - document::Document::SP doc(createRandomDocumentAtLocation(123, 100, 100)); - HeaderChunkEncoder encoder(doc->getId()); - - SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec()); - - BufferAllocation buf(ioBuf->allocateBuffer(HEADER, encoder.encodedSize())); - encoder.writeTo(buf); - DataLocation newLoc = ioBuf->addLocation(HEADER, buf); - document::DocumentId checkId = ioBuf->getDocumentId(newLoc); - - CPPUNIT_ASSERT_EQUAL(doc->getId(), checkId); -} - -void -SimpleMemFileIOBufferTest::testHeaderChunkEncoderSerializesHeaderCorrectly() -{ - document::Document::SP doc(createRandomDocumentAtLocation(123, 100, 100)); - HeaderChunkEncoder encoder(doc->getId()); - encoder.bufferDocument(*doc); - - SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec()); - BufferAllocation buf(ioBuf->allocateBuffer(HEADER, encoder.encodedSize())); - encoder.writeTo(buf); - DataLocation newLoc = ioBuf->addLocation(HEADER, buf); - Document::UP checkDoc = ioBuf->getDocumentHeader(*getTypeRepo(), newLoc); - - CPPUNIT_ASSERT_EQUAL(doc->getId(), checkDoc->getId()); - CPPUNIT_ASSERT_EQUAL(doc->getType(), checkDoc->getType()); -} - -void -SimpleMemFileIOBufferTest::testRemovesCanBeWrittenWithBlankDefaultDocument() -{ - SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec("testdoctype1")); - - document::DocumentId id("userdoc:yarn:12345:fluff"); - DataLocation loc(ioBuf->addDocumentIdOnlyHeader(id, *getTypeRepo())); - // Despite adding with document id only, we should now actually have a - // valid document header. Will fail with a DeserializeException if no - // header has been written. - Document::UP removeWithHeader( - ioBuf->getDocumentHeader(*getTypeRepo(), loc)); - CPPUNIT_ASSERT_EQUAL(removeWithHeader->getId(), id); - CPPUNIT_ASSERT_EQUAL(removeWithHeader->getType(), - *getTypeRepo()->getDocumentType("testdoctype1")); -} - -void -SimpleMemFileIOBufferTest::testRemovesCanBeWrittenWithIdInferredDoctype() -{ - SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec("testdoctype1")); - - document::DocumentId id("id:yarn:testdoctype2:n=12345:fluff"); - DataLocation loc(ioBuf->addDocumentIdOnlyHeader(id, *getTypeRepo())); - // Since document id contains an explicit document type, the blank remove - // document header should be written with that type instead of the one - // provided as default via config. - Document::UP removeWithHeader( - ioBuf->getDocumentHeader(*getTypeRepo(), loc)); - CPPUNIT_ASSERT_EQUAL(removeWithHeader->getId(), id); - CPPUNIT_ASSERT_EQUAL(removeWithHeader->getType(), - *getTypeRepo()->getDocumentType("testdoctype2")); -} - -void -SimpleMemFileIOBufferTest::testRemovesWithInvalidDocTypeThrowsException() -{ - SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec("testdoctype1")); - - document::DocumentId id("id:yarn:nosuchtype:n=12345:fluff"); - try { - DataLocation loc(ioBuf->addDocumentIdOnlyHeader(id, *getTypeRepo())); - CPPUNIT_FAIL("No exception thrown on bad doctype"); - } catch (const vespalib::Exception& e) { - CPPUNIT_ASSERT(e.getMessage().find("Could not serialize document " - "for remove with unknown doctype " - "'nosuchtype'") - != std::string::npos); - } -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/spi/simulatedfailurefile.cpp b/memfilepersistence/src/tests/spi/simulatedfailurefile.cpp deleted file mode 100644 index b7da294f8eb..00000000000 --- a/memfilepersistence/src/tests/spi/simulatedfailurefile.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simulatedfailurefile.h" -#include <vespa/vespalib/util/exceptions.h> - -namespace storage { -namespace memfile { - -vespalib::LazyFile::UP -SimulatedFailureLazyFile::Factory::createFile(const std::string& fileName) const { - return vespalib::LazyFile::UP( - new SimulatedFailureLazyFile(fileName, - vespalib::File::DIRECTIO, - _readOpsBeforeFailure, - _writeOpsBeforeFailure)); -} - -SimulatedFailureLazyFile::SimulatedFailureLazyFile( - const std::string& filename, - int flags, - int readOpsBeforeFailure, - int writeOpsBeforeFailure) - : LazyFile(filename, flags), - _readOpsBeforeFailure(readOpsBeforeFailure), - _writeOpsBeforeFailure(writeOpsBeforeFailure) -{ -} - -off_t -SimulatedFailureLazyFile::write(const void *buf, size_t bufsize, off_t offset) -{ - if (_writeOpsBeforeFailure == 0) { - throw vespalib::IoException( - "A simulated I/O write exception was triggered", - vespalib::IoException::CORRUPT_DATA, VESPA_STRLOC); - } - --_writeOpsBeforeFailure; - return vespalib::LazyFile::write(buf, bufsize, offset); -} - -size_t -SimulatedFailureLazyFile::read(void *buf, size_t bufsize, off_t offset) const -{ - if (_readOpsBeforeFailure == 0) { - throw vespalib::IoException( - "A simulated I/O read exception was triggered", - vespalib::IoException::CORRUPT_DATA, VESPA_STRLOC); - } - --_readOpsBeforeFailure; - return vespalib::LazyFile::read(buf, bufsize, offset); -} - -} // ns memfile -} // ns storage - diff --git a/memfilepersistence/src/tests/spi/simulatedfailurefile.h b/memfilepersistence/src/tests/spi/simulatedfailurefile.h deleted file mode 100644 index e3dbd5e13e2..00000000000 --- a/memfilepersistence/src/tests/spi/simulatedfailurefile.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <tests/spi/memfiletestutils.h> -#include <tests/spi/logginglazyfile.h> - -namespace storage { -namespace memfile { - -class SimulatedFailureLazyFile : public vespalib::LazyFile -{ - mutable int _readOpsBeforeFailure; - mutable int _writeOpsBeforeFailure; -public: - class Factory : public Environment::LazyFileFactory { - public: - Factory() - : _readOpsBeforeFailure(-1), - _writeOpsBeforeFailure(0) - { } - vespalib::LazyFile::UP createFile(const std::string& fileName) const override; - - void setReadOpsBeforeFailure(int ops) { - _readOpsBeforeFailure = ops; - } - - void setWriteOpsBeforeFailure(int ops) { - _writeOpsBeforeFailure = ops; - } - private: - int _readOpsBeforeFailure; - int _writeOpsBeforeFailure; - }; - - SimulatedFailureLazyFile( - const std::string& filename, - int flags, - int readOpsBeforeFailure, - int writeOpsBeforeFailure); - - off_t write(const void *buf, size_t bufsize, off_t offset) override; - size_t read(void *buf, size_t bufsize, off_t offset) const override; -}; - -} // ns memfile -} // ns storage - diff --git a/memfilepersistence/src/tests/spi/splitoperationhandlertest.cpp b/memfilepersistence/src/tests/spi/splitoperationhandlertest.cpp deleted file mode 100644 index 89a7d0f6e03..00000000000 --- a/memfilepersistence/src/tests/spi/splitoperationhandlertest.cpp +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memfiletestutils.h" -#include <vespa/document/datatype/documenttype.h> -#include <vespa/persistence/spi/test.h> - -using document::DocumentType; -using storage::spi::test::makeSpiBucket; - -namespace storage { -namespace memfile { -namespace { - spi::LoadType defaultLoadType(0, "default"); -} - -class SplitOperationHandlerTest : public SingleDiskMemFileTestUtils -{ - - void doTestMultiDisk(uint16_t sourceDisk, - uint16_t targetDisk0, - uint16_t targetDisk1); - - - CPPUNIT_TEST_SUITE(SplitOperationHandlerTest); - CPPUNIT_TEST(testSimple); - CPPUNIT_TEST(testMultiDisk); - CPPUNIT_TEST(testMultiDiskNonZeroSourceIndex); - CPPUNIT_TEST(testExceptionDuringSplittingEvictsAllBuckets); - CPPUNIT_TEST_SUITE_END(); - -public: - void testSimple(); - void testMultiDisk(); - void testMultiDiskNonZeroSourceIndex(); - void testExceptionDuringSplittingEvictsAllBuckets(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(SplitOperationHandlerTest); - -void -SplitOperationHandlerTest::testSimple() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(1); - - for (uint32_t i = 0; i < 100; i++) { - uint32_t location = 4; - if (i % 2 == 0) { - location |= (1 << 16); - } - - doPut(location, Timestamp(1000 + i)); - } - flush(document::BucketId(16, 4)); - - env()._cache.clear(); - - document::BucketId sourceBucket = document::BucketId(16, 4); - document::BucketId target1 = document::BucketId(17, 4); - document::BucketId target2 = document::BucketId(17, 4 | (1 << 16)); - - SplitOperationHandler handler(env()); - spi::Result result = getPersistenceProvider().split( - makeSpiBucket(sourceBucket), - makeSpiBucket(target1), - makeSpiBucket(target2), - context); - - env()._cache.clear(); - - { - MemFilePtr file(handler.getMemFile(sourceBucket, 0)); - CPPUNIT_ASSERT_EQUAL(0, (int)file->getSlotCount()); - } - - { - MemFilePtr file(handler.getMemFile(target1, 0)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - for (uint32_t i = 0; i < file->getSlotCount(); ++i) { - file->getDocument((*file)[i], ALL); - } - } - - { - MemFilePtr file(handler.getMemFile(target2, 0)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - for (uint32_t i = 0; i < file->getSlotCount(); ++i) { - file->getDocument((*file)[i], ALL); - } - } -} - -void -SplitOperationHandlerTest::doTestMultiDisk(uint16_t sourceDisk, - uint16_t targetDisk0, - uint16_t targetDisk1) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(3); - - for (uint32_t i = 0; i < 100; i++) { - uint32_t location = 4; - if (i % 2 == 0) { - location |= (1 << 16); - } - - doPutOnDisk(sourceDisk, location, Timestamp(1000 + i)); - } - flush(document::BucketId(16, 4)); - - env()._cache.clear(); - - document::BucketId sourceBucket = document::BucketId(16, 4); - document::BucketId target1 = document::BucketId(17, 4); - document::BucketId target2 = document::BucketId(17, 4 | (1 << 16)); - - SplitOperationHandler handler(env()); - spi::Result result = getPersistenceProvider().split( - makeSpiBucket(sourceBucket, spi::PartitionId(sourceDisk)), - makeSpiBucket(target1, spi::PartitionId(targetDisk0)), - makeSpiBucket(target2, spi::PartitionId(targetDisk1)), - context); - - env()._cache.clear(); - - { - MemFilePtr file(handler.getMemFile(sourceBucket, sourceDisk)); - CPPUNIT_ASSERT_EQUAL(0, (int)file->getSlotCount()); - } - - { - MemFilePtr file(handler.getMemFile(target1, targetDisk0)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - for (uint32_t i = 0; i < file->getSlotCount(); ++i) { - file->getDocument((*file)[i], ALL); - } - } - - { - MemFilePtr file(handler.getMemFile(target2, targetDisk1)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - for (uint32_t i = 0; i < file->getSlotCount(); ++i) { - file->getDocument((*file)[i], ALL); - } - } -} - -void -SplitOperationHandlerTest::testMultiDisk() -{ - doTestMultiDisk(0, 1, 2); -} - -void -SplitOperationHandlerTest::testMultiDiskNonZeroSourceIndex() -{ - doTestMultiDisk(1, 2, 0); -} - -void -SplitOperationHandlerTest::testExceptionDuringSplittingEvictsAllBuckets() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(1); - - for (uint32_t i = 0; i < 100; i++) { - uint32_t location = 4; - if (i % 2 == 0) { - location |= (1 << 16); - } - - doPut(location, Timestamp(1000 + i)); - } - flush(document::BucketId(16, 4)); - - simulateIoErrorsForSubsequentlyOpenedFiles(); - - document::BucketId sourceBucket(16, 4); - document::BucketId target1(17, 4); - document::BucketId target2(17, 4 | (1 << 16)); - - try { - SplitOperationHandler handler(env()); - spi::Result result = getPersistenceProvider().split( - makeSpiBucket(sourceBucket), - makeSpiBucket(target1), - makeSpiBucket(target2), - context); - CPPUNIT_FAIL("Exception not thrown on flush failure"); - } catch (std::exception&) { - } - - CPPUNIT_ASSERT(!env()._cache.contains(sourceBucket)); - CPPUNIT_ASSERT(!env()._cache.contains(target1)); - CPPUNIT_ASSERT(!env()._cache.contains(target2)); - - unSimulateIoErrorsForSubsequentlyOpenedFiles(); - - // Source must not have been deleted - { - SplitOperationHandler handler(env()); - MemFilePtr file(handler.getMemFile(sourceBucket, 0)); - CPPUNIT_ASSERT_EQUAL(100, (int)file->getSlotCount()); - } -} - -} - -} |