diff options
Diffstat (limited to 'memfilepersistence')
22 files changed, 193 insertions, 85 deletions
diff --git a/memfilepersistence/src/tests/device/devicemanagertest.cpp b/memfilepersistence/src/tests/device/devicemanagertest.cpp index f5e16032361..15882663383 100644 --- a/memfilepersistence/src/tests/device/devicemanagertest.cpp +++ b/memfilepersistence/src/tests/device/devicemanagertest.cpp @@ -1,8 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/memfilepersistence/device/devicemanager.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/util/exception.h> +#include <sys/errno.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> namespace storage { @@ -68,17 +70,19 @@ namespace { std::ostringstream ost; Listener() : ost() { ost << "\n"; } - ~Listener() {} + virtual ~Listener() {} - void handleDirectoryEvent(Directory& dir, const IOEvent& e) override { + virtual void handleDirectoryEvent(Directory& dir, const IOEvent& e) override { ost << "Dir " << dir.getPath() << ": " << e.toString(true) << "\n"; } - void handlePartitionEvent(Partition& part, const IOEvent& e) override { - ost << "Partition " << part.getMountPoint() << ": " << e.toString(true) << "\n"; + virtual void handlePartitionEvent(Partition& part, const IOEvent& e) override { + ost << "Partition " << part.getMountPoint() << ": " + << e.toString(true) << "\n"; } - void handleDiskEvent(Disk& disk, const IOEvent& e) override { + virtual void handleDiskEvent(Disk& disk, const IOEvent& e) override { ost << "Disk " << disk.getId() << ": " << e.toString(true) << "\n"; } + }; } diff --git a/memfilepersistence/src/tests/device/partitionmonitortest.cpp b/memfilepersistence/src/tests/device/partitionmonitortest.cpp index d1f40e07320..44f3752d840 100644 --- a/memfilepersistence/src/tests/device/partitionmonitortest.cpp +++ b/memfilepersistence/src/tests/device/partitionmonitortest.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/memfilepersistence/device/partitionmonitor.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vdstestlib/cppunit/macros.h> @@ -55,7 +56,7 @@ struct FakeStatter : public PartitionMonitor::Statter { _info.f_bfree -= (size / _info.f_bsize); } - void statFileSystem(const std::string&, struct statvfs& info) override { + virtual void statFileSystem(const std::string&, struct statvfs& info) override { info = _info; } }; diff --git a/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp b/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp index 4372a8a45a7..673a72068b7 100644 --- a/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp +++ b/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp @@ -1,9 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <set> +#include <vector> +#include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h> #include <tests/spi/memfiletestutils.h> #include <tests/spi/simulatedfailurefile.h> #include <tests/spi/options_builder.h> +#include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/select/parser.h> namespace storage { diff --git a/memfilepersistence/src/tests/spi/memcachetest.cpp b/memfilepersistence/src/tests/spi/memcachetest.cpp index 7eece580b0a..07697b79cac 100644 --- a/memfilepersistence/src/tests/spi/memcachetest.cpp +++ b/memfilepersistence/src/tests/spi/memcachetest.cpp @@ -1,8 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/memfilepersistence/memfile/memfilecache.h> #include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h> #include <tests/spi/memfiletestutils.h> +#include <vespa/vdstestlib/cppunit/macros.h> namespace storage { diff --git a/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp b/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp index 265054a02ba..eefe962979d 100644 --- a/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp +++ b/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp @@ -1,9 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/memfilepersistence/mapper/memfilemapper.h> #include <vespa/memfilepersistence/mapper/memfile_v1_serializer.h> #include <vespa/memfilepersistence/mapper/memfile_v1_verifier.h> +#include <vespa/memfilepersistence/mapper/fileinfo.h> +#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h> #include <tests/spi/memfiletestutils.h> +#include <vespa/vdstestlib/cppunit/macros.h> namespace storage { namespace memfile { diff --git a/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp b/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp index f612bec1189..ad20a28ed0c 100644 --- a/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp +++ b/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp @@ -1,8 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/memfilepersistence/mapper/memfilemapper.h> #include <vespa/memfilepersistence/mapper/memfile_v1_serializer.h> +#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h> #include <tests/spi/memfiletestutils.h> +#include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/memfilepersistence/mapper/locationreadplanner.h> #include <tests/spi/simulatedfailurefile.h> #include <tests/spi/options_builder.h> @@ -226,16 +229,23 @@ struct DummyMemFileIOInterface : MemFileIOInterface { void clear(DocumentPart) override {} bool verifyConsistent() const override { return true; } void move(const FileSpecification&) override {} - DataLocation copyCache(const MemFileIOInterface&, DocumentPart, DataLocation) override { + DataLocation copyCache(const MemFileIOInterface&, + DocumentPart, + DataLocation) override + { return DataLocation(); } - void close() override {}; + void close() override {} bool isCached(DataLocation, DocumentPart) const override { return false; } bool isPersisted(DataLocation, DocumentPart) const override { return false; } - uint32_t getSerializedSize(DocumentPart, DataLocation) const override { return 0; } + uint32_t getSerializedSize(DocumentPart, + DataLocation) const override { return 0; } - void ensureCached(Environment&, DocumentPart, const std::vector<DataLocation>&) override {} + void ensureCached(Environment&, + DocumentPart, + const std::vector<DataLocation>&) override + {} size_t getCachedSize(DocumentPart) const override { return 0; } }; diff --git a/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp b/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp index 632bded6c6c..51951e22aea 100644 --- a/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp +++ b/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp @@ -1,9 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/memfilepersistence/mapper/memfilemapper.h> #include <vespa/memfilepersistence/mapper/memfile_v1_serializer.h> #include <vespa/memfilepersistence/mapper/memfile_v1_verifier.h> +#include <vespa/memfilepersistence/mapper/fileinfo.h> +#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h> #include <tests/spi/memfiletestutils.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <tests/spi/simulatedfailurefile.h> namespace storage { namespace memfile { diff --git a/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp b/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp index cb1bc057705..ffe7972552f 100644 --- a/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp +++ b/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h> +#include <vespa/vdstestlib/cppunit/macros.h> #include <tests/spi/memfiletestutils.h> #include <tests/spi/options_builder.h> @@ -50,15 +51,19 @@ class SimpleMemFileIOBufferTest : public SingleDiskMemFileTestUtils public: class DummyFileReader : public VersionSerializer { public: - FileVersion getFileVersion() override { return FileVersion(); } - void loadFile(MemFile&, Environment&, Buffer&, uint64_t ) override {} - FlushResult flushUpdatesToFile(MemFile&, Environment&) override { + virtual FileVersion getFileVersion() override { return FileVersion(); } + virtual void loadFile(MemFile&, Environment&, Buffer&, uint64_t ) override {} + virtual FlushResult flushUpdatesToFile(MemFile&, Environment&) override { return FlushResult::TooSmall; } - void rewriteFile(MemFile&, Environment&) override {} - bool verify(MemFile&, Environment&, std::ostream&, bool, uint16_t) override { return false; }; - void cacheLocations(MemFileIOInterface&, Environment&, const Options&, - DocumentPart, const std::vector<DataLocation>&) override {} + virtual void rewriteFile(MemFile&, Environment&) override {} + virtual bool verify(MemFile&, Environment&, + std::ostream&, bool, uint16_t) override { return false; }; + virtual void cacheLocations(MemFileIOInterface&, + Environment&, + const Options&, + DocumentPart, + const std::vector<DataLocation>&) override {} }; DummyFileReader dfr; diff --git a/memfilepersistence/src/tests/testrunner.cpp b/memfilepersistence/src/tests/testrunner.cpp index 5baa0f60b6e..16027870c47 100644 --- a/memfilepersistence/src/tests/testrunner.cpp +++ b/memfilepersistence/src/tests/testrunner.cpp @@ -1,12 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <iostream> +#include <vespa/log/log.h> #include <vespa/vdstestlib/cppunit/cppunittestrunner.h> -#include <vespa/log/log.h> LOG_SETUP("persistencecppunittests"); int -main(int argc, const char *argv[]) +main(int argc, char **argv) { vdstestlib::CppUnitTestRunner testRunner; return testRunner.run(argc, argv); diff --git a/memfilepersistence/src/tests/tools/vdsdisktooltest.cpp b/memfilepersistence/src/tests/tools/vdsdisktooltest.cpp index 5de40311b2f..0f36f0a0571 100644 --- a/memfilepersistence/src/tests/tools/vdsdisktooltest.cpp +++ b/memfilepersistence/src/tests/tools/vdsdisktooltest.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/config/subscription/configuri.h> #include <vespa/memfilepersistence/tools/vdsdisktool.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/filespecification.h b/memfilepersistence/src/vespa/memfilepersistence/common/filespecification.h index cc2145444a5..23e4857fdb3 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/common/filespecification.h +++ b/memfilepersistence/src/vespa/memfilepersistence/common/filespecification.h @@ -37,7 +37,8 @@ public: const String& getPath() const { return _path; } FileVersion getWantedFileVersion() const { return _wantedVersion; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const override; bool operator==(const FileSpecification& o) const { return (_bucketId == o._bucketId && _dir == o._dir @@ -47,3 +48,4 @@ public: } // storage } // memfile + diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp index b0fa6cf6667..b483d4691c4 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp @@ -26,7 +26,8 @@ namespace { } struct RealStatter : public PartitionMonitor::Statter { - void statFileSystem(const std::string& file, struct statvfs& info) override { + virtual void statFileSystem(const std::string& file, struct statvfs& info) override + { if (statvfs(file.c_str(), &info) != 0) { vespalib::asciistream ost; ost << "Failed to run statvfs to find data on disk containing " diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp index d9cc0c35cc4..742a37fcf36 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp @@ -3,13 +3,16 @@ #include "memfile_v1_serializer.h" #include "memfile_v1_verifier.h" +#include "memfilemapper.h" #include "locationreadplanner.h" #include "uniqueslotgenerator.h" +#include "simplememfileiobuffer.h" #include <vespa/memfilepersistence/common/exceptions.h> #include <vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h> +#include <vespa/vespalib/util/crc.h> #include <vespa/vespalib/stllike/asciistream.h> - #include <vespa/log/log.h> + LOG_SETUP(".persistence.memfilev1"); namespace storage { @@ -739,16 +742,19 @@ namespace { _buffer.resize(firstAligned - _headerBlockIndex); } - uint32_t getCachedAmount() const override { return _buffer.size() + _headerBlockIndex; } - char* getCache(uint32_t pos) override { + virtual uint32_t getCachedAmount() const override + { return _buffer.size() + _headerBlockIndex; } + + virtual char* getCache(uint32_t pos) override { // We should never get requests to write prior to header block // index. assert(pos >= _headerBlockIndex); return (&_buffer[0] + (pos - _headerBlockIndex)); } - bool duplicateCacheWrite() const override { return true; } - void setData(const char* data, size_t len, uint64_t pos) override { + virtual bool duplicateCacheWrite() const override { return true; } + + virtual void setData(const char* data, size_t len, uint64_t pos) override { if (pos < _headerBlockIndex) { if (len <= _headerBlockIndex - pos) return; uint32_t diff = (_headerBlockIndex - pos); diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.h b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.h index 500c47e94e4..283ab174895 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.h +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.h @@ -1,10 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "bufferedfilewriter.h" -#include "versionserializer.h" -#include "fileinfo.h" -#include "simplememfileiobuffer.h" +#include <vespa/memfilepersistence/mapper/bufferedfilewriter.h> +#include <vespa/memfilepersistence/mapper/versionserializer.h> +#include <vespa/memfilepersistence/mapper/fileinfo.h> +#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h> #include <vespa/memfilepersistence/common/environment.h> #include <vespa/memfilepersistence/spi/threadmetricprovider.h> @@ -22,25 +22,40 @@ public: MemFileV1Serializer(ThreadMetricProvider&); - FileVersion getFileVersion() override { return TRADITIONAL_SLOTFILE; } - void loadFile(MemFile& file, Environment&, Buffer& buffer, uint64_t bytesRead) override; + virtual FileVersion getFileVersion() override { return TRADITIONAL_SLOTFILE; } - void cacheLocationsForPart(SimpleMemFileIOBuffer& cache, DocumentPart part, uint32_t blockIndex, + virtual void loadFile(MemFile& file, Environment&, + Buffer& buffer, uint64_t bytesRead) override; + + void cacheLocationsForPart(SimpleMemFileIOBuffer& cache, + DocumentPart part, + uint32_t blockIndex, const std::vector<DataLocation>& locationsToCache, const std::vector<DataLocation>& locationsRead, SimpleMemFileIOBuffer::BufferAllocation& buf); - void cacheLocations(MemFileIOInterface& cache, Environment& env, const Options& options, - DocumentPart part, const std::vector<DataLocation>& locations) override; + virtual void cacheLocations(MemFileIOInterface& cache, + Environment& env, + const Options& options, + DocumentPart part, + const std::vector<DataLocation>& locations) override; + + virtual FlushResult flushUpdatesToFile(MemFile&, Environment&) override; + + virtual void rewriteFile(MemFile&, Environment&) override; - FlushResult flushUpdatesToFile(MemFile&, Environment&) override; - void rewriteFile(MemFile&, Environment&) override; - bool verify(MemFile&, Environment&, std::ostream& errorReport, - bool repairErrors, uint16_t fileVerifyFlags) override; + virtual bool verify(MemFile&, Environment&, + std::ostream& errorReport, bool repairErrors, + uint16_t fileVerifyFlags) override; + + uint64_t read(vespalib::LazyFile& file, + char* buf, + const std::vector<DataLocation>& readOps); - uint64_t read(vespalib::LazyFile& file, char* buf, const std::vector<DataLocation>& readOps); void ensureFormatSpecificDataSet(const MemFile& file); - uint32_t writeMetaData(BufferedFileWriter& writer, const MemFile& file); + + uint32_t writeMetaData(BufferedFileWriter& writer, + const MemFile& file); uint32_t writeAndUpdateLocations( MemFile& file, @@ -53,3 +68,4 @@ public: } // memfile } // storage + diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h index 74875d3ee79..6ed529e6c80 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h @@ -199,7 +199,8 @@ public: class PartNotCachedException : public vespalib::Exception { public: - PartNotCachedException(const std::string& msg, const std::string& location) + PartNotCachedException(const std::string& msg, + const std::string& location) : vespalib::Exception(msg, location) {}; }; @@ -210,39 +211,64 @@ public: const FileSpecification& fileSpec, const Environment& env); - Document::UP getDocumentHeader(const document::DocumentTypeRepo& repo, DataLocation loc) const override; - document::DocumentId getDocumentId(DataLocation loc) const override; - void readBody(const document::DocumentTypeRepo& repo, DataLocation loc, Document& doc) const override; - DataLocation addDocumentIdOnlyHeader(const DocumentId& id, const document::DocumentTypeRepo& repo) override; - DataLocation addHeader(const Document& doc) override; - DataLocation addBody(const Document& doc) override; - void clear(DocumentPart type) override; - bool verifyConsistent() const override; + virtual Document::UP getDocumentHeader( + const document::DocumentTypeRepo& repo, + DataLocation loc) const override; + + virtual document::DocumentId getDocumentId(DataLocation loc) const override; + + virtual void readBody( + const document::DocumentTypeRepo& repo, + DataLocation loc, + Document& doc) const override; + + virtual DataLocation addDocumentIdOnlyHeader( + const DocumentId& id, + const document::DocumentTypeRepo& repo) override; + + virtual DataLocation addHeader(const Document& doc) override; + + virtual DataLocation addBody(const Document& doc) override; + + virtual void clear(DocumentPart type) override; + + virtual bool verifyConsistent() const override; /** * Moves the underlying file to another location. */ - void move(const FileSpecification& target) override; - void close() override; - DataLocation copyCache(const MemFileIOInterface& source, DocumentPart part, DataLocation loc) override; + virtual void move(const FileSpecification& target) override; + + virtual void close() override; + + virtual DataLocation copyCache(const MemFileIOInterface& source, + DocumentPart part, + DataLocation loc) override; /** * Add a location -> buffer mapping */ - void cacheLocation(DocumentPart part, DataLocation loc, BufferType::SP buf, uint32_t bufferPos); + void cacheLocation(DocumentPart part, + DataLocation loc, + BufferType::SP buf, + uint32_t bufferPos); /** * @return Returns true if the given location is cached. */ - bool isCached(DataLocation loc, DocumentPart type) const override; + virtual bool isCached(DataLocation loc, DocumentPart type) const override; /** * @return Returns true if the given location has been persisted to disk. */ + virtual bool isPersisted(DataLocation loc, DocumentPart type) const override; - bool isPersisted(DataLocation loc, DocumentPart type) const override; - uint32_t getSerializedSize(DocumentPart part, DataLocation loc) const override; - void ensureCached(Environment& env, DocumentPart part, const std::vector<DataLocation>& locations) override; + virtual uint32_t getSerializedSize(DocumentPart part, + DataLocation loc) const override; + + virtual void ensureCached(Environment& env, + DocumentPart part, + const std::vector<DataLocation>& locations) override; /** * Moves the given location into the persisted data area. @@ -316,9 +342,14 @@ private: // the reasoning about a given bucket in the face of such actions. std::shared_ptr<const Options> _options; - DataLocation addLocation(DocumentPart part, BufferAllocation newData); + DataLocation addLocation(DocumentPart part, + BufferAllocation newData); + const Data& getData(DocumentPart part, DataLocation loc) const; - BufferAllocation serializeDocumentIdOnlyHeader(const DocumentId& id, const document::DocumentTypeRepo&); + + BufferAllocation serializeDocumentIdOnlyHeader( + const DocumentId& id, + const document::DocumentTypeRepo&); BufferAllocation serializeHeader(const Document& doc); BufferAllocation serializeBody(const Document& doc); @@ -327,3 +358,5 @@ private: } } + + diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.h b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.h index f22035faf04..b3ac15ed376 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.h +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.h @@ -27,12 +27,12 @@ #pragma once -#include "memslot.h" -#include "slotiterator.h" -#include "memfileiointerface.h" #include <vespa/memfilepersistence/common/filespecification.h> #include <vespa/memfilepersistence/common/types.h> +#include <vespa/memfilepersistence/memfile/memslot.h> +#include <vespa/memfilepersistence/memfile/slotiterator.h> #include <vespa/vespalib/io/fileutil.h> +#include <vespa/memfilepersistence/memfile/memfileiointerface.h> namespace storage { namespace memfile { diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp index e8fff0facd0..8ea60d57f80 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp @@ -1,16 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "memfilecache.h" +#include <vespa/fastos/fastos.h> #include <vespa/memfilepersistence/common/environment.h> #include <vespa/memfilepersistence/mapper/memfilemapper.h> +#include <vespa/memfilepersistence/memfile/memfilecache.h> +#include <vespa/log/log.h> +#include <vespa/vespalib/util/vstringfmt.h> #include <vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h> -#include <vespa/log/log.h> LOG_SETUP(".persistence.memfile.cache"); namespace storage { namespace memfile { + void MemFileCache::Entry::setInUse(bool inUse) { LOG(debug, "Setting in use to %d for file %s", inUse, _file.toString().c_str()); @@ -108,7 +111,7 @@ struct MemFileCache::CacheEntryGuard : public MemFilePtr::EntryGuard { _entry(&entry) { } - ~CacheEntryGuard() { + virtual ~CacheEntryGuard() { if (_entry) { _cache.done(*_entry); } @@ -118,20 +121,20 @@ struct MemFileCache::CacheEntryGuard : public MemFilePtr::EntryGuard { return _entry->_file; } - void deleteFile() override { + virtual void deleteFile() override { LOG(debug, "Cache entry guard deleting %s", _file->toString().c_str()); _env._memFileMapper.deleteFile(*_file, _env); erase(); } - void erase() override { + virtual void erase() override { LOG(debug, "Cache entry guard erasing %s from cache", _file->toString().c_str()); _cache.erase(document::BucketId(_entry->_file.getFile().getBucketId())); _entry = 0; } - void move(EntryGuard& target) override { + virtual void move(EntryGuard& target) override { LOG(debug, "Cache entry guard moving %s", _file->toString().c_str()); _cache.move(*this, static_cast<CacheEntryGuard&>(target)); } @@ -172,7 +175,10 @@ MemFileCache::setCacheSize(MemoryUsage cacheSize) vespalib::LockGuard lock(_cacheLock); _cacheLimit = cacheSize; - _memoryToken->resize(std::min(_memoryToken->getSize(), _cacheLimit.sum()), _cacheLimit.sum()); + + _memoryToken->resize(std::min(_memoryToken->getSize(), _cacheLimit.sum()), + _cacheLimit.sum()); + evictWhileFull(); } diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/slotiterator.h b/memfilepersistence/src/vespa/memfilepersistence/memfile/slotiterator.h index 91106cd765b..b1d68591b92 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/slotiterator.h +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/slotiterator.h @@ -65,8 +65,8 @@ public: Timestamp fromTimestamp, Timestamp toTimestamp); - void iterate() const override; - SlotIterator* clone() const override; + virtual void iterate() const override; + virtual SlotIterator* clone() const override; }; class AllSlotsIterator : public SlotIterator { @@ -82,8 +82,8 @@ public: Timestamp fromTimestamp, Timestamp toTimestamp); - void iterate() const override; - SlotIterator* clone() const override; + virtual void iterate() const override; + virtual SlotIterator* clone() const override; }; /** @@ -127,3 +127,4 @@ public: } // memfile } // storage + diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/splitoperationhandler.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/splitoperationhandler.cpp index 9e75e4fa182..247722588f5 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/splitoperationhandler.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/splitoperationhandler.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "splitoperationhandler.h" -#include "cacheevictionguard.h" - +#include <vespa/fastos/fastos.h> +#include <vespa/memfilepersistence/spi/splitoperationhandler.h> +#include <vespa/memfilepersistence/spi/cacheevictionguard.h> #include <vespa/log/log.h> + LOG_SETUP(".persistence.memfile.handler.split"); namespace storage { @@ -25,7 +25,7 @@ struct BucketMatcher : public SlotMatcher { _factory(factory), _bid(bid) {} - bool match(const Slot& slot) override { + virtual bool match(const Slot& slot) override { document::DocumentId id(slot.getDocumentId()); document::BucketId bucket = _factory.getBucketId(id); bucket.setUsedBits(_bid.getUsedBits()); @@ -69,7 +69,7 @@ SplitOperationHandler::splitIntoFile(MemFile& source, MemFileCacheEvictionGuard targetFile(getMemFile(target, false)); LOG(debug, - "Found %zu slots to move from file %s to file %s", + "Found %" PRIu64 " slots to move from file %s to file %s", ts.size(), source.getFile().toString().c_str(), targetFile->getFile().toString().c_str()); diff --git a/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp b/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp index 072b3e9fae0..c794d611767 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "dumpslotfile.h" + #include <vespa/config/helper/configgetter.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/repo/documenttyperepo.h> @@ -9,6 +9,7 @@ #include <vespa/memfilepersistence/mapper/memfilemapper.h> #include <vespa/memfilepersistence/memfile/memfilecache.h> #include <vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h> +#include <vespa/memfilepersistence/tools/dumpslotfile.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> #include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h> diff --git a/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfileapp.cpp b/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfileapp.cpp index 544be9c5eac..f8d13714ae9 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfileapp.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfileapp.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "dumpslotfile.h" +#include <vespa/fastos/fastos.h> #include <vespa/config/subscription/configuri.h> -#include <vespa/fastos/app.h> +#include <vespa/memfilepersistence/tools/dumpslotfile.h> #include <iostream> namespace { diff --git a/memfilepersistence/src/vespa/memfilepersistence/tools/vdsdiskapp.cpp b/memfilepersistence/src/vespa/memfilepersistence/tools/vdsdiskapp.cpp index 34855aeabc1..7127c75cb1b 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/tools/vdsdiskapp.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/tools/vdsdiskapp.cpp @@ -1,10 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "vdsdisktool.h" +#include <vespa/fastos/fastos.h> #include <vespa/defaults.h> -#include <vespa/fastos/app.h> +#include <vespa/log/log.h> +#include <vespa/memfilepersistence/tools/vdsdisktool.h> #include <iostream> +LOG_SETUP(".vdsdisktool"); + namespace { struct DiskApp : public FastOS_Application { int Main() override { |