diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-03-14 17:19:26 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-03-14 17:19:26 +0100 |
commit | da55d54e49de80fcdb73ad32c021c28e0e0a856e (patch) | |
tree | 08a0d796466250bb8cebb33c604adb75a0e19fe2 | |
parent | 5ef72e6f6280190eb98f4e53607e8240ce73a6e7 (diff) | |
parent | d3774910a5ba8f2b324bd55fb6d392cb92fd3bd6 (diff) |
Merge pull request #26436 from vespa-engine/balder/gc-dangerous-docstore-options-with-no-right-to-live
summary.cache.initialentries and summary.log.chunk.skipcrconread does…
24 files changed, 41 insertions, 104 deletions
diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 5829666c41b..375ff0f2012 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -212,11 +212,13 @@ summary.cache.maxbytes long default=-4 ## Include visits in the cache, if the visitoperation allows it. ## This will enable another separate cache of summary.cache.maxbytes size. +## IGNORED and DEPRECATED Will go away soon summary.cache.allowvisitcaching bool default=true ## Control number of cache entries preallocated. ## Default is no preallocation. ## Can be set to a higher number to avoid resizing. +## IGNORED and DEPRECATED Will go away soon summary.cache.initialentries long default=0 restart ## Control compression type of the summary while in the cache. @@ -252,6 +254,7 @@ summary.log.chunk.compression.level int default=9 summary.log.chunk.maxbytes int default=65536 ## Skip crc32 check on read. +## IGNORED and DEPRECATED Will go away soon summary.log.chunk.skipcrconread bool default=false ## Max size per summary file. diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 3b3c043d578..2c75f00a672 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -174,7 +174,7 @@ getStoreConfig(const ProtonConfig::Summary::Cache & cache, const HwInfo & hwInfo size_t maxBytes = (cache.maxbytes < 0) ? (hwInfo.memory().sizeBytes()*std::min(INT64_C(50), -cache.maxbytes))/100l : cache.maxbytes; - return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries) + return DocumentStore::Config(deriveCompression(cache.compression), maxBytes) .updateStrategy(derive(cache.updateStrategy)); } @@ -189,7 +189,7 @@ deriveConfig(const ProtonConfig::Summary & summary, const HwInfo & hwInfo) { .setMaxNumLids(log.maxnumlids) .setMaxBucketSpread(log.maxbucketspread).setMinFileSizeFactor(log.minfilesizefactor) .compactCompression(deriveCompression(log.compact.compression)) - .setFileConfig(fileConfig).disableCrcOnRead(chunk.skipcrconread); + .setFileConfig(fileConfig); return {config, logConfig}; } diff --git a/searchlib/src/apps/docstore/create-idx-from-dat.cpp b/searchlib/src/apps/docstore/create-idx-from-dat.cpp index 7ac6b37473c..d5c23384d35 100644 --- a/searchlib/src/apps/docstore/create-idx-from-dat.cpp +++ b/searchlib/src/apps/docstore/create-idx-from-dat.cpp @@ -32,7 +32,7 @@ bool tryDecode(size_t chunks, size_t offset, const char * p, size_t sz, size_t n bool success(false); for (size_t lengthError(0); !success && (sz + lengthError <= nextSync); lengthError++) { try { - Chunk chunk(chunks, p, sz + lengthError, false); + Chunk chunk(chunks, p, sz + lengthError); success = true; } catch (const vespalib::Exception & e) { fprintf(stdout, "Chunk %ld, with size=%ld failed with lengthError %ld due to '%s'\n", offset, sz, lengthError, e.what()); @@ -62,7 +62,7 @@ generate(uint64_t serialNum, size_t chunks, FastOS_FileInterface & idxFile, size vespalib::nbostream os; for (size_t lengthError(0); int64_t(sz+lengthError) <= nextStart-start; lengthError++) { try { - Chunk chunk(chunks, current, sz + lengthError, false); + Chunk chunk(chunks, current, sz + lengthError); fprintf(stdout, "id=%d lastSerial=%" PRIu64 " count=%ld\n", chunk.getId(), chunk.getLastSerial(), chunk.count()); const Chunk::LidList & lidlist = chunk.getLids(); if (chunk.getLastSerial() < serialNum) { diff --git a/searchlib/src/tests/docstore/chunk/chunk_test.cpp b/searchlib/src/tests/docstore/chunk/chunk_test.cpp index 574e04d4900..28bd5208b73 100644 --- a/searchlib/src/tests/docstore/chunk/chunk_test.cpp +++ b/searchlib/src/tests/docstore/chunk/chunk_test.cpp @@ -84,7 +84,7 @@ void verifyChunkCompression(CompressionConfig::Type cfgType, const void * buf, s chunk.pack(7, buffer, cfg); EXPECT_EQUAL(expectedLen, buffer.getDataLen()); vespalib::nbostream is(buffer.getData(), buffer.getDataLen()); - ChunkFormat::UP deserialized = ChunkFormat::deserialize(buffer.getData(), buffer.getDataLen(), false); + ChunkFormat::UP deserialized = ChunkFormat::deserialize(buffer.getData(), buffer.getDataLen()); uint64_t magic(0); deserialized->getBuffer() >> magic; EXPECT_EQUAL(MAGIC_CONTENT, magic); diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp index 9fef6f57e85..99a9cdec17e 100644 --- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp +++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp @@ -50,7 +50,7 @@ struct NullDataStore : IDataStore { NullDataStore::~NullDataStore() = default; TEST_FFF("require that uncache docstore lookups are counted", - DocumentStore::Config(CompressionConfig::NONE, 0, 0), + DocumentStore::Config(CompressionConfig::NONE, 0), NullDataStore(), DocumentStore(f1, f2)) { EXPECT_EQUAL(0u, f3.getCacheStats().misses); @@ -59,7 +59,7 @@ TEST_FFF("require that uncache docstore lookups are counted", } TEST_FFF("require that cached docstore lookups are counted", - DocumentStore::Config(CompressionConfig::NONE, 100000, 100), + DocumentStore::Config(CompressionConfig::NONE, 100000), NullDataStore(), DocumentStore(f1, f2)) { EXPECT_EQUAL(0u, f3.getCacheStats().misses); @@ -70,10 +70,9 @@ TEST_FFF("require that cached docstore lookups are counted", TEST("require that DocumentStore::Config equality operator detects inequality") { using C = DocumentStore::Config; EXPECT_TRUE(C() == C()); - EXPECT_TRUE(C(CompressionConfig::NONE, 100000, 100) == C(CompressionConfig::NONE, 100000, 100)); - EXPECT_FALSE(C(CompressionConfig::NONE, 100000, 100) == C(CompressionConfig::NONE, 100000, 99)); - EXPECT_FALSE(C(CompressionConfig::NONE, 100000, 100) == C(CompressionConfig::NONE, 100001, 100)); - EXPECT_FALSE(C(CompressionConfig::NONE, 100000, 100) == C(CompressionConfig::LZ4, 100000, 100)); + EXPECT_TRUE(C(CompressionConfig::NONE, 100000) == C(CompressionConfig::NONE, 100000)); + EXPECT_FALSE(C(CompressionConfig::NONE, 100000) == C(CompressionConfig::NONE, 100001)); + EXPECT_FALSE(C(CompressionConfig::NONE, 100000) == C(CompressionConfig::LZ4, 100000)); } TEST("require that LogDocumentStore::Config equality operator detects inequality") { @@ -82,7 +81,7 @@ TEST("require that LogDocumentStore::Config equality operator detects inequality using DC = DocumentStore::Config; EXPECT_TRUE(C() == C()); EXPECT_FALSE(C() != C()); - EXPECT_FALSE(C(DC(CompressionConfig::NONE, 100000, 100), LC()) == C()); + EXPECT_FALSE(C(DC(CompressionConfig::NONE, 100000), LC()) == C()); EXPECT_FALSE(C(DC(), LC().setMaxBucketSpread(7)) == C()); } diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index e841e334215..9bc649149d0 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -237,7 +237,7 @@ struct Fixture Fixture::Fixture() : _baseDir("visitor"), _repo(makeDocTypeRepoConfig()), - _storeConfig(DocumentStore::Config(CompressionConfig::NONE, 0, 0), + _storeConfig(DocumentStore::Config(CompressionConfig::NONE, 0), LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0) .setFileConfig(WriteableFileChunk::Config(CompressionConfig(), 16_Ki))), _executor(1), diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp index 6e69e5092bc..64ae4dfd2f2 100644 --- a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -100,12 +100,7 @@ struct ReadFixture : public FixtureBase { explicit ReadFixture(const vespalib::string &baseName, bool dirCleanup = true) : FixtureBase(baseName, dirCleanup), - chunk(FileChunk::FileId(0), - FileChunk::NameId(1234), - baseName, - tuneFile, - &bucketizer, - false) + chunk(FileChunk::FileId(0), FileChunk::NameId(1234), baseName, tuneFile, &bucketizer) { dir.cleanup(dirCleanup); } @@ -123,17 +118,8 @@ struct WriteFixture : public FixtureBase { uint32_t docIdLimit, bool dirCleanup = true) : FixtureBase(baseName, dirCleanup), - chunk(executor, - FileChunk::FileId(0), - FileChunk::NameId(1234), - baseName, - serialNum, - docIdLimit, - WriteableFileChunk::Config(CompressionConfig(), 0x1000), - tuneFile, - fileHeaderCtx, - &bucketizer, - false) + chunk(executor, FileChunk::FileId(0), FileChunk::NameId(1234), baseName, serialNum, docIdLimit, + {CompressionConfig(), 0x1000}, tuneFile, fileHeaderCtx, &bucketizer) { dir.cleanup(dirCleanup); } diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index eec49043282..9043782e639 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -528,7 +528,7 @@ VisitCacheStore::VerifyVisitor::~VerifyVisitor() { VisitCacheStore::VisitCacheStore(UpdateStrategy strategy) : _myDir("visitcache"), _repo(makeDocTypeRepoConfig()), - _config(DocumentStore::Config(CompressionConfig::LZ4, 1000000, 0).updateStrategy(strategy), + _config(DocumentStore::Config(CompressionConfig::LZ4, 1000000).updateStrategy(strategy), LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0) .setFileConfig(WriteableFileChunk::Config(CompressionConfig(), 16_Ki))), _fileHeaderContext(), @@ -565,8 +565,8 @@ TEST("Control static memory usage") { IDocumentStore &ds = vcs.getStore(); vespalib::MemoryUsage usage = ds.getMemoryUsage(); constexpr size_t mutex_size = sizeof(std::mutex) * 2 * (113 + 1); // sizeof(std::mutex) is platform dependent - EXPECT_EQUAL(74580 + mutex_size, usage.allocatedBytes()); - EXPECT_EQUAL(952u + mutex_size, usage.usedBytes()); + EXPECT_EQUAL(74572 + mutex_size, usage.allocatedBytes()); + EXPECT_EQUAL(944u + mutex_size, usage.usedBytes()); } TEST("test the update cache strategy") { @@ -1023,8 +1023,8 @@ TEST_F("require that there is control of static memory usage", Fixture) { vespalib::MemoryUsage usage = f.store.getMemoryUsage(); EXPECT_EQUAL(584u + sizeof(std::mutex), sizeof(LogDataStore)); - EXPECT_EQUAL(74116u, usage.allocatedBytes()); - EXPECT_EQUAL(392u, usage.usedBytes()); + EXPECT_EQUAL(74108u, usage.allocatedBytes()); + EXPECT_EQUAL(384u, usage.usedBytes()); } TEST_F("require that lid space can be shrunk only after read guards are deleted", Fixture) @@ -1074,7 +1074,6 @@ TEST("require that config equality operator detects inequality") { EXPECT_FALSE(C() == C().setMaxBucketSpread(0.3)); EXPECT_FALSE(C() == C().setMinFileSizeFactor(0.3)); EXPECT_FALSE(C() == C().setFileConfig(WriteableFileChunk::Config({}, 70))); - EXPECT_FALSE(C() == C().disableCrcOnRead(true)); EXPECT_FALSE(C() == C().compactCompression({CompressionConfig::ZSTD})); } diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp index a8982c28805..35166cf8d78 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp @@ -79,10 +79,10 @@ Chunk::Chunk(uint32_t id, const Config & config) : _lids.reserve(4_Ki/sizeof(Entry)); } -Chunk::Chunk(uint32_t id, const void * buffer, size_t len, bool skipcrc) : +Chunk::Chunk(uint32_t id, const void * buffer, size_t len) : _id(id), _lastSerial(static_cast<uint64_t>(-1l)), - _format(ChunkFormat::deserialize(buffer, len, skipcrc)) + _format(ChunkFormat::deserialize(buffer, len)) { vespalib::nbostream &os = getData(); while (os.size() > sizeof(_lastSerial)) { diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.h b/searchlib/src/vespa/searchlib/docstore/chunk.h index e274572c96b..211165934e1 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.h +++ b/searchlib/src/vespa/searchlib/docstore/chunk.h @@ -82,7 +82,7 @@ public: }; using LidList = std::vector<Entry>; Chunk(uint32_t id, const Config & config); - Chunk(uint32_t id, const void * buffer, size_t len, bool skipcrc=false); + Chunk(uint32_t id, const void * buffer, size_t len); ~Chunk(); LidMeta append(uint32_t lid, const void * buffer, size_t len); ssize_t read(uint32_t lid, vespalib::DataBuffer & buffer) const; diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index 5b352b91c38..2f622cfa78c 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -67,7 +67,7 @@ ChunkFormat::verifyCompression(uint8_t type) } ChunkFormat::UP -ChunkFormat::deserialize(const void * buffer, size_t len, bool skipcrc) +ChunkFormat::deserialize(const void * buffer, size_t len) { uint8_t version(0); vespalib::nbostream raw(buffer, len); @@ -82,17 +82,9 @@ ChunkFormat::deserialize(const void * buffer, size_t len, bool skipcrc) raw >> crc32; raw.rp(currPos); if (version == ChunkFormatV1::VERSION) { - if (skipcrc) { - return std::make_unique<ChunkFormatV1>(raw); - } else { - return std::make_unique<ChunkFormatV1>(raw, crc32); - } + return std::make_unique<ChunkFormatV1>(raw, crc32); } else if (version == ChunkFormatV2::VERSION) { - if (skipcrc) { - return std::make_unique<ChunkFormatV2>(raw); - } else { return std::make_unique<ChunkFormatV2>(raw, crc32); - } } else { throw ChunkException(make_string("Unknown version %d", version), VESPA_STRLOC); } diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.h b/searchlib/src/vespa/searchlib/docstore/chunkformat.h index 343f1dbb9c7..29793f8b311 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.h +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.h @@ -36,9 +36,8 @@ public: * Will deserialize and create a representation of the uncompressed data. * param buffer Pointer to the serialized data * @param len Length of serialized data - * @param indicate if crc verification shall be skipped. */ - static ChunkFormat::UP deserialize(const void * buffer, size_t len, bool skipcrc); + static ChunkFormat::UP deserialize(const void * buffer, size_t len); /** * return the maximum size a packet can have. It allows correct size estimation * need for direct io alignment. diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp index 19a612971ad..c895fceb6bf 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp @@ -9,12 +9,6 @@ namespace search { using vespalib::make_string; -ChunkFormatV1::ChunkFormatV1(vespalib::nbostream & is) : - ChunkFormat() -{ - deserializeBody(is); -} - ChunkFormatV1::ChunkFormatV1(vespalib::nbostream & is, uint32_t expectedCrc) : ChunkFormat() { @@ -33,13 +27,6 @@ ChunkFormatV1::computeCrc(const void * buf, size_t sz) const return vespalib::crc_32_type::crc(buf, sz); } -ChunkFormatV2::ChunkFormatV2(vespalib::nbostream & is) : - ChunkFormat() -{ - verifyMagic(is); - deserializeBody(is); -} - ChunkFormatV2::ChunkFormatV2(vespalib::nbostream & is, uint32_t expectedCrc) : ChunkFormat() { diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformats.h b/searchlib/src/vespa/searchlib/docstore/chunkformats.h index 5473cca712a..47ee9ea8a6c 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformats.h +++ b/searchlib/src/vespa/searchlib/docstore/chunkformats.h @@ -10,7 +10,6 @@ class ChunkFormatV1 : public ChunkFormat { public: enum {VERSION=0}; - ChunkFormatV1(vespalib::nbostream & is); ChunkFormatV1(vespalib::nbostream & is, uint32_t expectedCrc); ChunkFormatV1(size_t maxSize); private: @@ -27,7 +26,6 @@ class ChunkFormatV2 : public ChunkFormat { public: enum {VERSION=1, MAGIC=0x5ba32de7}; - ChunkFormatV2(vespalib::nbostream & is); ChunkFormatV2(vespalib::nbostream & is, uint32_t expectedCrc); ChunkFormatV2(size_t maxSize); private: diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp index 368dd31678d..49beed34f08 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp @@ -117,7 +117,6 @@ using docstore::Value; bool DocumentStore::Config::operator == (const Config &rhs) const { return (_maxCacheBytes == rhs._maxCacheBytes) && - (_initialCacheEntries == rhs._initialCacheEntries) && (_updateStrategy == rhs._updateStrategy) && (_compression == rhs._compression); } @@ -131,9 +130,7 @@ DocumentStore::DocumentStore(const Config & config, IDataStore & store) _visitCache(std::make_unique<docstore::VisitCache>(store, config.getMaxCacheBytes(), config.getCompression())), _updateStrategy(config.updateStrategy()), _uncached_lookups(0) -{ - _cache->reserveElements(config.getInitialCacheEntries()); -} +{ } DocumentStore::~DocumentStore() = default; diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.h b/searchlib/src/vespa/searchlib/docstore/documentstore.h index 024b26b79c6..bb62b09123f 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.h @@ -25,28 +25,24 @@ public: public: enum UpdateStrategy {INVALIDATE, UPDATE }; using CompressionConfig = vespalib::compression::CompressionConfig; - Config() : + Config() noexcept : _compression(CompressionConfig::LZ4, 9, 70), _maxCacheBytes(1000000000), - _initialCacheEntries(0), _updateStrategy(INVALIDATE) { } - Config(CompressionConfig compression, size_t maxCacheBytes, size_t initialCacheEntries) : + Config(CompressionConfig compression, size_t maxCacheBytes) noexcept : _compression((maxCacheBytes != 0) ? compression : CompressionConfig::NONE), _maxCacheBytes(maxCacheBytes), - _initialCacheEntries(initialCacheEntries), _updateStrategy(INVALIDATE) { } CompressionConfig getCompression() const { return _compression; } size_t getMaxCacheBytes() const { return _maxCacheBytes; } - size_t getInitialCacheEntries() const { return _initialCacheEntries; } Config & updateStrategy(UpdateStrategy strategy) { _updateStrategy = strategy; return *this; } UpdateStrategy updateStrategy() const { return _updateStrategy; } bool operator == (const Config &) const; private: CompressionConfig _compression; size_t _maxCacheBytes; - size_t _initialCacheEntries; UpdateStrategy _updateStrategy; }; diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index d7922fcca2a..159af42635f 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -62,11 +62,10 @@ FileChunk::createDatFileName(const vespalib::string & name) { } FileChunk::FileChunk(FileId fileId, NameId nameId, const vespalib::string & baseName, - const TuneFileSummary & tune, const IBucketizer * bucketizer, bool skipCrcOnRead) + const TuneFileSummary & tune, const IBucketizer * bucketizer) : _fileId(fileId), _nameId(nameId), _name(nameId.createName(baseName)), - _skipCrcOnRead(skipCrcOnRead), _erasedCount(0), _erasedBytes(0), _diskFootprint(0), @@ -394,7 +393,7 @@ FileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, ChunkInfo c { vespalib::DataBuffer whole(0ul, ALIGNMENT); FileRandRead::FSP keepAlive = _file->read(ci.getOffset(), whole, ci.getSize()); - Chunk chunk(begin->getChunkId(), whole.getData(), whole.getDataLen(), _skipCrcOnRead); + Chunk chunk(begin->getChunkId(), whole.getData(), whole.getDataLen()); for (size_t i(0); i < count; i++) { const LidInfoWithLid & li = *(begin + i); vespalib::ConstBufferRef buf = chunk.getLid(li.getLid()); @@ -419,7 +418,7 @@ FileChunk::read(uint32_t lid, SubChunkId chunkId, const ChunkInfo & chunkInfo, { vespalib::DataBuffer whole(0ul, ALIGNMENT); FileRandRead::FSP keepAlive(_file->read(chunkInfo.getOffset(), whole, chunkInfo.getSize())); - Chunk chunk(chunkId, whole.getData(), whole.getDataLen(), _skipCrcOnRead); + Chunk chunk(chunkId, whole.getData(), whole.getDataLen()); return chunk.read(lid, buffer); } diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.h b/searchlib/src/vespa/searchlib/docstore/filechunk.h index 9e8623060fb..3664da3dfd9 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.h @@ -106,7 +106,7 @@ public: using UP = std::unique_ptr<FileChunk>; using SubChunkId = uint32_t; FileChunk(FileId fileId, NameId nameId, const vespalib::string &baseName, const TuneFileSummary &tune, - const IBucketizer *bucketizer, bool skipCrcOnRead); + const IBucketizer *bucketizer); virtual ~FileChunk(); virtual size_t updateLidMap(const unique_lock &guard, ISetLid &lidMap, uint64_t serialNum, uint32_t docIdLimit); @@ -203,7 +203,6 @@ private: const FileId _fileId; const NameId _nameId; const vespalib::string _name; - const bool _skipCrcOnRead; size_t _erasedCount; size_t _erasedBytes; std::atomic<size_t> _diskFootprint; diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index f35f9fa4cec..bde7492f485 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -44,7 +44,6 @@ LogDataStore::Config::Config() _maxBucketSpread(2.5), _minFileSizeFactor(0.2), _maxNumLids(DEFAULT_MAX_LIDS_PER_FILE), - _skipCrcOnRead(false), _compactCompression(CompressionConfig::LZ4), _fileConfig() { } @@ -54,7 +53,6 @@ LogDataStore::Config::operator == (const Config & rhs) const { return (_maxBucketSpread == rhs._maxBucketSpread) && (_maxFileSize == rhs._maxFileSize) && (_minFileSizeFactor == rhs._minFileSizeFactor) && - (_skipCrcOnRead == rhs._skipCrcOnRead) && (_compactCompression == rhs._compactCompression) && (_fileConfig == rhs._fileConfig); } @@ -632,7 +630,7 @@ LogDataStore::createIdxFileName(NameId id) const { FileChunk::UP LogDataStore::createReadOnlyFile(FileId fileId, NameId nameId) { auto file = std::make_unique<FileChunk>(fileId, nameId, getBaseDir(), _tune, - _bucketizer.get(), _config.crcOnReadDisabled()); + _bucketizer.get()); file->enableRead(); return file; } @@ -650,7 +648,7 @@ LogDataStore::createWritableFile(FileId fileId, SerialNum serialNum, NameId name uint32_t docIdLimit = (getDocIdLimit() != 0) ? getDocIdLimit() : std::numeric_limits<uint32_t>::max(); auto file = std::make_unique< WriteableFileChunk>(_executor, fileId, nameId, getBaseDir(), serialNum,docIdLimit, _config.getFileConfig(), _tune, _fileHeaderContext, - _bucketizer.get(), _config.crcOnReadDisabled()); + _bucketizer.get()); file->enableRead(); return file; } diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index ef2cbf93215..cd302ca0c55 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -57,11 +57,9 @@ public: double getMinFileSizeFactor() const { return _minFileSizeFactor; } uint32_t getMaxNumLids() const { return _maxNumLids; } - bool crcOnReadDisabled() const { return _skipCrcOnRead; } CompressionConfig compactCompression() const { return _compactCompression; } const WriteableFileChunk::Config & getFileConfig() const { return _fileConfig; } - Config & disableCrcOnRead(bool v) { _skipCrcOnRead = v; return *this;} bool operator == (const Config &) const; private: @@ -69,7 +67,6 @@ public: AtomicValueWrapper<double> _maxBucketSpread; double _minFileSizeFactor; uint32_t _maxNumLids; - bool _skipCrcOnRead; CompressionConfig _compactCompression; WriteableFileChunk::Config _fileConfig; }; diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index c642d8b30f0..84bad6b65ac 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -82,9 +82,8 @@ WriteableFileChunk(vespalib::Executor &executor, const Config &config, const TuneFileSummary &tune, const FileHeaderContext &fileHeaderContext, - const IBucketizer * bucketizer, - bool skipCrcOnRead) - : FileChunk(fileId, nameId, baseName, tune, bucketizer, skipCrcOnRead), + const IBucketizer * bucketizer) + : FileChunk(fileId, nameId, baseName, tune, bucketizer), _config(config), _serialNum(initialSerialNum), _frozen(false), diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h index defa9a382c8..b5a52dc83f7 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h @@ -46,7 +46,7 @@ public: const vespalib::string & baseName, uint64_t initialSerialNum, uint32_t docIdLimit, const Config & config, const TuneFileSummary &tune, const common::FileHeaderContext &fileHeaderContext, - const IBucketizer * bucketizer, bool crcOnReadDisabled); + const IBucketizer * bucketizer); ~WriteableFileChunk() override; ssize_t read(uint32_t lid, SubChunkId chunk, vespalib::DataBuffer & buffer) const override; diff --git a/vespalib/src/vespa/vespalib/stllike/cache.h b/vespalib/src/vespa/vespalib/stllike/cache.h index 907fbdd54c9..f7456cda197 100644 --- a/vespalib/src/vespa/vespalib/stllike/cache.h +++ b/vespalib/src/vespa/vespalib/stllike/cache.h @@ -69,10 +69,6 @@ public: * Can be used for controlling max number of elements. */ cache & maxElements(size_t elems); - /** - * Can be used for reserving space for elements. - */ - cache & reserveElements(size_t elems); cache & setCapacityBytes(size_t sz); diff --git a/vespalib/src/vespa/vespalib/stllike/cache.hpp b/vespalib/src/vespa/vespalib/stllike/cache.hpp index 8e449fcfca4..4e7736c9e5f 100644 --- a/vespalib/src/vespa/vespalib/stllike/cache.hpp +++ b/vespalib/src/vespa/vespalib/stllike/cache.hpp @@ -16,13 +16,6 @@ cache<P>::maxElements(size_t elems) { template< typename P > cache<P> & -cache<P>::reserveElements(size_t elems) { - Lru::reserve(elems); - return *this; -} - -template< typename P > -cache<P> & cache<P>::setCapacityBytes(size_t sz) { _maxBytes.store(sz, std::memory_order_relaxed); return *this; |