From acd4af1d2c19119132c6803f15fff70cb65b7d4f Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 6 Apr 2018 00:21:15 +0200 Subject: Remove memfile implementation. --- .../src/tests/spi/memfilev1verifiertest.cpp | 496 --------------------- 1 file changed, 496 deletions(-) delete mode 100644 memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp (limited to 'memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp') diff --git a/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp b/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp deleted file mode 100644 index c38842bfeb0..00000000000 --- a/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp +++ /dev/null @@ -1,496 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include - -namespace storage { -namespace memfile { - -struct MemFileV1VerifierTest : public SingleDiskMemFileTestUtils -{ - void testVerify(); - - void tearDown() override; - - std::unique_ptr createMemFile(FileSpecification& file, - bool callLoadFile) - { - return std::unique_ptr(new MemFile(file, env(), callLoadFile)); - } - - CPPUNIT_TEST_SUITE(MemFileV1VerifierTest); - CPPUNIT_TEST_IGNORED(testVerify); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemFileV1VerifierTest); - -namespace { - // A totall uncached memfile with content to use for verify testing - std::unique_ptr _memFile; - - // Clear old content. Create new file. Make sure nothing is cached. - void prepareBucket(SingleDiskMemFileTestUtils& util, - const FileSpecification& file) { - _memFile.reset(); - util.env()._cache.clear(); - vespalib::unlink(file.getPath()); - util.createTestBucket(file.getBucketId(), 0); - util.env()._cache.clear(); - _memFile.reset(new MemFile(file, util.env())); - _memFile->getMemFileIO().close(); - - } - - // Get copy of header of memfile created - Header getHeader() { - assert(_memFile.get()); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - Header result; - file.read(&result, sizeof(Header), 0); - return result; - } - - MetaSlot getSlot(uint32_t index) { - assert(_memFile.get()); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - MetaSlot result; - file.read(&result, sizeof(MetaSlot), - sizeof(Header) + sizeof(MetaSlot) * index); - return result; - } - - void setSlot(uint32_t index, MetaSlot slot, - bool updateFileChecksum = true) - { - (void)updateFileChecksum; - assert(_memFile.get()); - //if (updateFileChecksum) slot.updateFileChecksum(); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - file.write(&slot, sizeof(MetaSlot), - sizeof(Header) + sizeof(MetaSlot) * index); - } - - void setHeader(const Header& header) { - assert(_memFile.get()); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - file.write(&header, sizeof(Header), 0); - } - - void verifySlotFile(MemFileV1VerifierTest& util, - const std::string& expectedError, - const std::string& message, - int32_t remainingEntries, - bool includeContent = true, - bool includeHeader = true) - { - assert(_memFile.get()); - FileSpecification file(_memFile->getFile()); - _memFile.reset(); - _memFile = util.createMemFile(file, false); - std::ostringstream before; - try{ - util.env()._memFileMapper.loadFile(*_memFile, util.env(), false); - _memFile->print(before, true, ""); - } catch (vespalib::Exception& e) { - before << "Unknown. Exception during loadFile\n"; - } - std::ostringstream errors; - uint32_t flags = (includeContent ? 0 : Types::DONT_VERIFY_BODY) - | (includeHeader ? 0 : Types::DONT_VERIFY_HEADER); - if (util.env()._memFileMapper.verify( - *_memFile, util.env(), errors, flags)) - { - _memFile->print(std::cerr, true, ""); - std::cerr << errors.str() << "\n"; - CPPUNIT_FAIL("verify() failed to detect: " + message); - } - CPPUNIT_ASSERT_CONTAIN_MESSAGE(message + "\nBefore: " + before.str(), - expectedError, errors.str()); - errors.str(""); - if (util.env()._memFileMapper.repair( - *_memFile, util.env(), errors, flags)) - { - CPPUNIT_FAIL("repair() failed to detect: " + message - + ": " + errors.str()); - } - CPPUNIT_ASSERT_CONTAIN_MESSAGE(message + "\nBefore: " + before.str(), - expectedError, errors.str()); - std::ostringstream remainingErrors; - if (!util.env()._memFileMapper.verify( - *_memFile, util.env(), remainingErrors, flags)) - { - CPPUNIT_FAIL("verify() returns issue after repair of: " - + message + ": " + remainingErrors.str()); - } - CPPUNIT_ASSERT_MESSAGE(remainingErrors.str(), - remainingErrors.str().size() == 0); - if (remainingEntries < 0) { - if (_memFile->fileExists()) { - CPPUNIT_FAIL(message + ": Expected file to not exist anymore"); - } - } else if (dynamic_cast(_memFile->getMemFileIO()) - .getFileHandle().getFileSize() == 0) - { - std::ostringstream ost; - ost << "Expected " << remainingEntries << " to remain in file, " - << "but file does not exist\n"; - CPPUNIT_FAIL(message + ": " + ost.str()); - } else { - if (int64_t(_memFile->getSlotCount()) != remainingEntries) { - std::ostringstream ost; - ost << "Expected " << remainingEntries << " to remain in file, " - << "but found " << _memFile->getSlotCount() << "\n"; - ost << errors.str() << "\n"; - ost << "Before: " << before.str() << "\nAfter: "; - _memFile->print(ost, true, ""); - CPPUNIT_FAIL(message + ": " + ost.str()); - } - } - } -} - -void -MemFileV1VerifierTest::tearDown() -{ - _memFile.reset(0); - SingleDiskMemFileTestUtils::tearDown(); -}; - -void -MemFileV1VerifierTest::testVerify() -{ - BucketId bucket(16, 0xa); - std::unique_ptr file; - createTestBucket(bucket, 0); - - { - MemFilePtr memFilePtr(env()._cache.get(bucket, env(), env().getDirectory())); - file.reset(new FileSpecification(memFilePtr->getFile())); - env()._cache.clear(); - } - { // Ensure buildTestFile builds a valid file - // Initial file should be fine. - MemFile memFile(*file, env()); - std::ostringstream errors; - if (!env()._memFileMapper.verify(memFile, env(), errors)) { - memFile.print(std::cerr, false, ""); - CPPUNIT_FAIL("Slotfile failed verification: " + errors.str()); - } - } - // Header tests - prepareBucket(*this, *file); - Header orgheader(getHeader()); - { // Test wrong version - Header header(orgheader); - header.setVersion(0xc0edbabe); - header.updateChecksum(); - setHeader(header); - verifySlotFile(*this, - "400000000000000a.0 is of wrong version", - "Faulty version", - -1); - } - { // Test meta data list size bigger than file - prepareBucket(*this, *file); - Header header(orgheader); - header.setMetaDataListSize(0xFFFF); - header.updateChecksum(); - setHeader(header); - verifySlotFile(*this, - "indicates file is bigger than it physically is", - "Too big meta data list size", - -1); - } - { // Test header block size bigger than file - prepareBucket(*this, *file); - Header header(orgheader); - header.setHeaderBlockSize(0xFFFF); - header.updateChecksum(); - setHeader(header); - verifySlotFile(*this, - "Header indicates file is bigger than it physically is", - "Too big header block size", - -1); - } - { // Test wrong header crc - prepareBucket(*this, *file); - Header header(orgheader); - header.setMetaDataListSize(4); - setHeader(header); - verifySlotFile(*this, - "Header checksum mismatch", - "Wrong header checksum", - -1); - } - // Meta data tests - prepareBucket(*this, *file); - MetaSlot slot6(getSlot(6)); - { // Test extra removes - currently allowed - MetaSlot slot7(getSlot(7)); - MetaSlot s(slot7); - s.setTimestamp(Timestamp(s._timestamp.getTime() - 1)); - s.updateChecksum(); - setSlot(6, s); - s.setTimestamp(Timestamp(s._timestamp.getTime() + 1)); - s.updateChecksum(); - setSlot(7, s); - std::ostringstream errors; - if (!env()._memFileMapper.verify(*_memFile, env(), errors)) { - _memFile->print(std::cerr, false, ""); - std::cerr << errors.str() << "\n"; - CPPUNIT_FAIL("Supposed to be legal with multiple remove values"); - } - setSlot(7, slot7); - } - { - // Test metadata crc mismatch with "used" flag being accidentally - // flipped. Should not inhibit adding of subsequent slots. - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setUseFlag(false); - setSlot(6, s); - verifySlotFile(*this, - "Slot 6 at timestamp 2001 failed checksum verification", - "Crc failure with use flag", 23, false); - } - { // Test overlapping documents - MetaSlot s(slot6); - // Direct overlapping header - prepareBucket(*this, *file); - s.setHeaderPos(0); - s.setHeaderSize(51); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "overlaps with slot", - "Direct overlapping header", 6, false, false); - // Contained header - // (contained bit not valid header so fails on other error now) - prepareBucket(*this, *file); - s.setHeaderPos(176); - s.setHeaderSize(80); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "not big enough to contain a document id", - "Contained header", 7, false); - // Partly overlapping header - // (contained bit not valid header so fails on other error now) - prepareBucket(*this, *file); - s.setHeaderPos(191); - s.setHeaderSize(35); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "not big enough to contain a document id", - "Partly overlapping header", 7, false); - prepareBucket(*this, *file); - s.setHeaderPos(185); - s.setHeaderSize(33); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "not big enough to contain a document id", - "Partly overlapping header (2)", 7, false); - // Direct overlapping body - prepareBucket(*this, *file); - s = slot6; - s.setBodyPos(0); - s.setBodySize(136); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Multiple slots with different gids use same body position", - "Directly overlapping body", 6, false); - // Contained body - prepareBucket(*this, *file); - s.setBodyPos(10); - s.setBodySize(50); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "overlaps with slot", - "Contained body", 6, false); - CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(1)) == 0); - // Overlapping body - prepareBucket(*this, *file); - s.setBodyPos(160); - s.setBodySize(40); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "overlaps with slot", - "Overlapping body", 5, false); - CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(2)) == 0); - CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(1501)) == 0); - // Overlapping body, verifying bodies - // (Bad body bit should be removed first, so only one slot needs - // removing) - prepareBucket(*this, *file); - setSlot(6, s); - verifySlotFile(*this, - "Body checksum mismatch", - "Overlapping body(2)", 7, true); - } - { // Test out of bounds - MetaSlot s(slot6); - - // Header out of bounds - prepareBucket(*this, *file); - s.setHeaderPos(500); - s.setHeaderSize(5000); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "goes out of bounds", - "Header out of bounds", 7, false, false); - // Body out of bounds - prepareBucket(*this, *file); - s = slot6; - s.setBodyPos(2400); - s.setBodySize(6000); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "goes out of bounds", - "Body out of bounds", 7, false); - } - { // Test timestamp collision - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setTimestamp(Timestamp(10002)); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "has same timestamp as slot 5", - "Timestamp collision", 6, false); - } - { // Test timestamp out of order - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setTimestamp(Timestamp(38)); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Slot 6 is out of timestamp order", - "Timestamp out of order", 8, false); - } - { // Test metadata crc mismatch - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setTimestamp(Timestamp(40)); - setSlot(6, s); - verifySlotFile(*this, - "Slot 6 at timestamp 40 failed checksum verification", - "Crc failure", 7, false); - } - { // Test used after unused - // This might actually lose documents after the unused entries. - // The memfile will not know about the documents after unused entry. - // If the memfile contains changes and writes metadata back due to this, - // the following entries will be missing. - // (To prevent this repair would have to add metadata entries, but that - // may be problems if repair happens at a time where all header or body - // data in the file needs to be cached.) - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setUseFlag(false); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Slot 7 found after unused entries", - "Used after unused", 6, false); - } - { // Test header blob corrupt - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setHeaderPos(519); - s.setHeaderSize(86); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Header checksum mismatch", - "Corrupt header blob.", 7); - } - { // Test body blob corrupt - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setBodyPos(52); - s.setBodySize(18); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Body checksum mismatch", - "Corrupt body blob.", 7); - } - { // Test too long name for header chunk - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setHeaderPos(160); - s.setHeaderSize(33); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "header is not big enough to contain a document", - "Too long name in header.", 7); - } - { // Test wrong file checksum -// Currently disabled. Currently only possible to calculate file checksum from -// memfile now, and memfile object wont be valid. -/* - // First test if we actually have less entries at all.. - prepareBucket(*this, *file); - MetaSlot s(getSlot(7)); - s.setUseFlag(false); - s.updateChecksum(); - setSlot(7, s, false); - s = getSlot(8); - s.setUseFlag(false); - s.updateChecksum(); - setSlot(8, s, false); - verifySlotFile(*this, - "File checksum should have been", - "Wrong file checksum in file.", 7, false); -std::cerr << "U\n"; - // Then test with different timestamp in remaining document - prepareBucket(*this, *file); - s = getSlot(6); - s.setTimestamp(s._timestamp + 1); - s.updateChecksum(); - setSlot(6, s, false); - verifySlotFile(*this, - "File checksum should have been", - "Wrong file checksum in file.", 9, false); -std::cerr << "V\n"; - // Then check with different gid - prepareBucket(*this, *file); - s = getSlot(6); - s._gid = GlobalId("sdfsdfsedsdfsdfsd"); - s.updateChecksum(); - setSlot(6, s, false); - verifySlotFile(*this, - "File checksum should have been", - "Wrong file checksum in file.", 9, false, false); -*/ - } - { // Test that documents not belonging in a bucket is removed -// Currently disabled. Hard to test. Needs total rewrite -/* - prepareBucket(*this, *file); - Blob b(createBlob(43u, "userdoc::0:315", "header", "body")); - _memFile->write(b, 80); - CPPUNIT_ASSERT_EQUAL(4u, _memFile->getBlobCount()); - CPPUNIT_ASSERT(_memFile->read(b)); - verifySlotFile(*this, - "belongs in bucket", - "Document not belonging there", 9); - CPPUNIT_ASSERT_EQUAL(3u, _memFile->getBlobCount()); -*/ - } -} - -} -} -- cgit v1.2.3