aboutsummaryrefslogtreecommitdiffstats
path: root/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp')
-rw-r--r--memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp496
1 files changed, 0 insertions, 496 deletions
diff --git a/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp b/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp
deleted file mode 100644
index c38842bfeb0..00000000000
--- a/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp
+++ /dev/null
@@ -1,496 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/memfilepersistence/mapper/memfilemapper.h>
-#include <vespa/memfilepersistence/mapper/memfile_v1_serializer.h>
-#include <vespa/memfilepersistence/mapper/memfile_v1_verifier.h>
-#include <tests/spi/memfiletestutils.h>
-
-namespace storage {
-namespace memfile {
-
-struct MemFileV1VerifierTest : public SingleDiskMemFileTestUtils
-{
- void testVerify();
-
- void tearDown() override;
-
- std::unique_ptr<MemFile> createMemFile(FileSpecification& file,
- bool callLoadFile)
- {
- return std::unique_ptr<MemFile>(new MemFile(file, env(), callLoadFile));
- }
-
- CPPUNIT_TEST_SUITE(MemFileV1VerifierTest);
- CPPUNIT_TEST_IGNORED(testVerify);
- CPPUNIT_TEST_SUITE_END();
-};
-
-CPPUNIT_TEST_SUITE_REGISTRATION(MemFileV1VerifierTest);
-
-namespace {
- // A totall uncached memfile with content to use for verify testing
- std::unique_ptr<MemFile> _memFile;
-
- // Clear old content. Create new file. Make sure nothing is cached.
- void prepareBucket(SingleDiskMemFileTestUtils& util,
- const FileSpecification& file) {
- _memFile.reset();
- util.env()._cache.clear();
- vespalib::unlink(file.getPath());
- util.createTestBucket(file.getBucketId(), 0);
- util.env()._cache.clear();
- _memFile.reset(new MemFile(file, util.env()));
- _memFile->getMemFileIO().close();
-
- }
-
- // Get copy of header of memfile created
- Header getHeader() {
- assert(_memFile.get());
- vespalib::LazyFile file(_memFile->getFile().getPath(), 0);
- Header result;
- file.read(&result, sizeof(Header), 0);
- return result;
- }
-
- MetaSlot getSlot(uint32_t index) {
- assert(_memFile.get());
- vespalib::LazyFile file(_memFile->getFile().getPath(), 0);
- MetaSlot result;
- file.read(&result, sizeof(MetaSlot),
- sizeof(Header) + sizeof(MetaSlot) * index);
- return result;
- }
-
- void setSlot(uint32_t index, MetaSlot slot,
- bool updateFileChecksum = true)
- {
- (void)updateFileChecksum;
- assert(_memFile.get());
- //if (updateFileChecksum) slot.updateFileChecksum();
- vespalib::LazyFile file(_memFile->getFile().getPath(), 0);
- file.write(&slot, sizeof(MetaSlot),
- sizeof(Header) + sizeof(MetaSlot) * index);
- }
-
- void setHeader(const Header& header) {
- assert(_memFile.get());
- vespalib::LazyFile file(_memFile->getFile().getPath(), 0);
- file.write(&header, sizeof(Header), 0);
- }
-
- void verifySlotFile(MemFileV1VerifierTest& util,
- const std::string& expectedError,
- const std::string& message,
- int32_t remainingEntries,
- bool includeContent = true,
- bool includeHeader = true)
- {
- assert(_memFile.get());
- FileSpecification file(_memFile->getFile());
- _memFile.reset();
- _memFile = util.createMemFile(file, false);
- std::ostringstream before;
- try{
- util.env()._memFileMapper.loadFile(*_memFile, util.env(), false);
- _memFile->print(before, true, "");
- } catch (vespalib::Exception& e) {
- before << "Unknown. Exception during loadFile\n";
- }
- std::ostringstream errors;
- uint32_t flags = (includeContent ? 0 : Types::DONT_VERIFY_BODY)
- | (includeHeader ? 0 : Types::DONT_VERIFY_HEADER);
- if (util.env()._memFileMapper.verify(
- *_memFile, util.env(), errors, flags))
- {
- _memFile->print(std::cerr, true, "");
- std::cerr << errors.str() << "\n";
- CPPUNIT_FAIL("verify() failed to detect: " + message);
- }
- CPPUNIT_ASSERT_CONTAIN_MESSAGE(message + "\nBefore: " + before.str(),
- expectedError, errors.str());
- errors.str("");
- if (util.env()._memFileMapper.repair(
- *_memFile, util.env(), errors, flags))
- {
- CPPUNIT_FAIL("repair() failed to detect: " + message
- + ": " + errors.str());
- }
- CPPUNIT_ASSERT_CONTAIN_MESSAGE(message + "\nBefore: " + before.str(),
- expectedError, errors.str());
- std::ostringstream remainingErrors;
- if (!util.env()._memFileMapper.verify(
- *_memFile, util.env(), remainingErrors, flags))
- {
- CPPUNIT_FAIL("verify() returns issue after repair of: "
- + message + ": " + remainingErrors.str());
- }
- CPPUNIT_ASSERT_MESSAGE(remainingErrors.str(),
- remainingErrors.str().size() == 0);
- if (remainingEntries < 0) {
- if (_memFile->fileExists()) {
- CPPUNIT_FAIL(message + ": Expected file to not exist anymore");
- }
- } else if (dynamic_cast<SimpleMemFileIOBuffer&>(_memFile->getMemFileIO())
- .getFileHandle().getFileSize() == 0)
- {
- std::ostringstream ost;
- ost << "Expected " << remainingEntries << " to remain in file, "
- << "but file does not exist\n";
- CPPUNIT_FAIL(message + ": " + ost.str());
- } else {
- if (int64_t(_memFile->getSlotCount()) != remainingEntries) {
- std::ostringstream ost;
- ost << "Expected " << remainingEntries << " to remain in file, "
- << "but found " << _memFile->getSlotCount() << "\n";
- ost << errors.str() << "\n";
- ost << "Before: " << before.str() << "\nAfter: ";
- _memFile->print(ost, true, "");
- CPPUNIT_FAIL(message + ": " + ost.str());
- }
- }
- }
-}
-
-void
-MemFileV1VerifierTest::tearDown()
-{
- _memFile.reset(0);
- SingleDiskMemFileTestUtils::tearDown();
-};
-
-void
-MemFileV1VerifierTest::testVerify()
-{
- BucketId bucket(16, 0xa);
- std::unique_ptr<FileSpecification> file;
- createTestBucket(bucket, 0);
-
- {
- MemFilePtr memFilePtr(env()._cache.get(bucket, env(), env().getDirectory()));
- file.reset(new FileSpecification(memFilePtr->getFile()));
- env()._cache.clear();
- }
- { // Ensure buildTestFile builds a valid file
- // Initial file should be fine.
- MemFile memFile(*file, env());
- std::ostringstream errors;
- if (!env()._memFileMapper.verify(memFile, env(), errors)) {
- memFile.print(std::cerr, false, "");
- CPPUNIT_FAIL("Slotfile failed verification: " + errors.str());
- }
- }
- // Header tests
- prepareBucket(*this, *file);
- Header orgheader(getHeader());
- { // Test wrong version
- Header header(orgheader);
- header.setVersion(0xc0edbabe);
- header.updateChecksum();
- setHeader(header);
- verifySlotFile(*this,
- "400000000000000a.0 is of wrong version",
- "Faulty version",
- -1);
- }
- { // Test meta data list size bigger than file
- prepareBucket(*this, *file);
- Header header(orgheader);
- header.setMetaDataListSize(0xFFFF);
- header.updateChecksum();
- setHeader(header);
- verifySlotFile(*this,
- "indicates file is bigger than it physically is",
- "Too big meta data list size",
- -1);
- }
- { // Test header block size bigger than file
- prepareBucket(*this, *file);
- Header header(orgheader);
- header.setHeaderBlockSize(0xFFFF);
- header.updateChecksum();
- setHeader(header);
- verifySlotFile(*this,
- "Header indicates file is bigger than it physically is",
- "Too big header block size",
- -1);
- }
- { // Test wrong header crc
- prepareBucket(*this, *file);
- Header header(orgheader);
- header.setMetaDataListSize(4);
- setHeader(header);
- verifySlotFile(*this,
- "Header checksum mismatch",
- "Wrong header checksum",
- -1);
- }
- // Meta data tests
- prepareBucket(*this, *file);
- MetaSlot slot6(getSlot(6));
- { // Test extra removes - currently allowed
- MetaSlot slot7(getSlot(7));
- MetaSlot s(slot7);
- s.setTimestamp(Timestamp(s._timestamp.getTime() - 1));
- s.updateChecksum();
- setSlot(6, s);
- s.setTimestamp(Timestamp(s._timestamp.getTime() + 1));
- s.updateChecksum();
- setSlot(7, s);
- std::ostringstream errors;
- if (!env()._memFileMapper.verify(*_memFile, env(), errors)) {
- _memFile->print(std::cerr, false, "");
- std::cerr << errors.str() << "\n";
- CPPUNIT_FAIL("Supposed to be legal with multiple remove values");
- }
- setSlot(7, slot7);
- }
- {
- // Test metadata crc mismatch with "used" flag being accidentally
- // flipped. Should not inhibit adding of subsequent slots.
- prepareBucket(*this, *file);
- MetaSlot s(slot6);
- s.setUseFlag(false);
- setSlot(6, s);
- verifySlotFile(*this,
- "Slot 6 at timestamp 2001 failed checksum verification",
- "Crc failure with use flag", 23, false);
- }
- { // Test overlapping documents
- MetaSlot s(slot6);
- // Direct overlapping header
- prepareBucket(*this, *file);
- s.setHeaderPos(0);
- s.setHeaderSize(51);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "overlaps with slot",
- "Direct overlapping header", 6, false, false);
- // Contained header
- // (contained bit not valid header so fails on other error now)
- prepareBucket(*this, *file);
- s.setHeaderPos(176);
- s.setHeaderSize(80);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "not big enough to contain a document id",
- "Contained header", 7, false);
- // Partly overlapping header
- // (contained bit not valid header so fails on other error now)
- prepareBucket(*this, *file);
- s.setHeaderPos(191);
- s.setHeaderSize(35);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "not big enough to contain a document id",
- "Partly overlapping header", 7, false);
- prepareBucket(*this, *file);
- s.setHeaderPos(185);
- s.setHeaderSize(33);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "not big enough to contain a document id",
- "Partly overlapping header (2)", 7, false);
- // Direct overlapping body
- prepareBucket(*this, *file);
- s = slot6;
- s.setBodyPos(0);
- s.setBodySize(136);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "Multiple slots with different gids use same body position",
- "Directly overlapping body", 6, false);
- // Contained body
- prepareBucket(*this, *file);
- s.setBodyPos(10);
- s.setBodySize(50);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "overlaps with slot",
- "Contained body", 6, false);
- CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(1)) == 0);
- // Overlapping body
- prepareBucket(*this, *file);
- s.setBodyPos(160);
- s.setBodySize(40);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "overlaps with slot",
- "Overlapping body", 5, false);
- CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(2)) == 0);
- CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(1501)) == 0);
- // Overlapping body, verifying bodies
- // (Bad body bit should be removed first, so only one slot needs
- // removing)
- prepareBucket(*this, *file);
- setSlot(6, s);
- verifySlotFile(*this,
- "Body checksum mismatch",
- "Overlapping body(2)", 7, true);
- }
- { // Test out of bounds
- MetaSlot s(slot6);
-
- // Header out of bounds
- prepareBucket(*this, *file);
- s.setHeaderPos(500);
- s.setHeaderSize(5000);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "goes out of bounds",
- "Header out of bounds", 7, false, false);
- // Body out of bounds
- prepareBucket(*this, *file);
- s = slot6;
- s.setBodyPos(2400);
- s.setBodySize(6000);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "goes out of bounds",
- "Body out of bounds", 7, false);
- }
- { // Test timestamp collision
- prepareBucket(*this, *file);
- MetaSlot s(slot6);
- s.setTimestamp(Timestamp(10002));
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "has same timestamp as slot 5",
- "Timestamp collision", 6, false);
- }
- { // Test timestamp out of order
- prepareBucket(*this, *file);
- MetaSlot s(slot6);
- s.setTimestamp(Timestamp(38));
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "Slot 6 is out of timestamp order",
- "Timestamp out of order", 8, false);
- }
- { // Test metadata crc mismatch
- prepareBucket(*this, *file);
- MetaSlot s(slot6);
- s.setTimestamp(Timestamp(40));
- setSlot(6, s);
- verifySlotFile(*this,
- "Slot 6 at timestamp 40 failed checksum verification",
- "Crc failure", 7, false);
- }
- { // Test used after unused
- // This might actually lose documents after the unused entries.
- // The memfile will not know about the documents after unused entry.
- // If the memfile contains changes and writes metadata back due to this,
- // the following entries will be missing.
- // (To prevent this repair would have to add metadata entries, but that
- // may be problems if repair happens at a time where all header or body
- // data in the file needs to be cached.)
- prepareBucket(*this, *file);
- MetaSlot s(slot6);
- s.setUseFlag(false);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "Slot 7 found after unused entries",
- "Used after unused", 6, false);
- }
- { // Test header blob corrupt
- prepareBucket(*this, *file);
- MetaSlot s(slot6);
- s.setHeaderPos(519);
- s.setHeaderSize(86);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "Header checksum mismatch",
- "Corrupt header blob.", 7);
- }
- { // Test body blob corrupt
- prepareBucket(*this, *file);
- MetaSlot s(slot6);
- s.setBodyPos(52);
- s.setBodySize(18);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "Body checksum mismatch",
- "Corrupt body blob.", 7);
- }
- { // Test too long name for header chunk
- prepareBucket(*this, *file);
- MetaSlot s(slot6);
- s.setHeaderPos(160);
- s.setHeaderSize(33);
- s.updateChecksum();
- setSlot(6, s);
- verifySlotFile(*this,
- "header is not big enough to contain a document",
- "Too long name in header.", 7);
- }
- { // Test wrong file checksum
-// Currently disabled. Currently only possible to calculate file checksum from
-// memfile now, and memfile object wont be valid.
-/*
- // First test if we actually have less entries at all..
- prepareBucket(*this, *file);
- MetaSlot s(getSlot(7));
- s.setUseFlag(false);
- s.updateChecksum();
- setSlot(7, s, false);
- s = getSlot(8);
- s.setUseFlag(false);
- s.updateChecksum();
- setSlot(8, s, false);
- verifySlotFile(*this,
- "File checksum should have been",
- "Wrong file checksum in file.", 7, false);
-std::cerr << "U\n";
- // Then test with different timestamp in remaining document
- prepareBucket(*this, *file);
- s = getSlot(6);
- s.setTimestamp(s._timestamp + 1);
- s.updateChecksum();
- setSlot(6, s, false);
- verifySlotFile(*this,
- "File checksum should have been",
- "Wrong file checksum in file.", 9, false);
-std::cerr << "V\n";
- // Then check with different gid
- prepareBucket(*this, *file);
- s = getSlot(6);
- s._gid = GlobalId("sdfsdfsedsdfsdfsd");
- s.updateChecksum();
- setSlot(6, s, false);
- verifySlotFile(*this,
- "File checksum should have been",
- "Wrong file checksum in file.", 9, false, false);
-*/
- }
- { // Test that documents not belonging in a bucket is removed
-// Currently disabled. Hard to test. Needs total rewrite
-/*
- prepareBucket(*this, *file);
- Blob b(createBlob(43u, "userdoc::0:315", "header", "body"));
- _memFile->write(b, 80);
- CPPUNIT_ASSERT_EQUAL(4u, _memFile->getBlobCount());
- CPPUNIT_ASSERT(_memFile->read(b));
- verifySlotFile(*this,
- "belongs in bucket",
- "Document not belonging there", 9);
- CPPUNIT_ASSERT_EQUAL(3u, _memFile->getBlobCount());
-*/
- }
-}
-
-}
-}