Wire in for live reconfiguration of DocumentStore.

8 files changed, 64 insertions, 47 deletions

diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
index bf59614a297..63b924d5cc2 100644
--- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
@@ -110,17 +110,19 @@ private:
 class BackingStore {
 public:
     BackingStore(IDataStore &store, const CompressionConfig &compression) :
-            _backingStore(store),
-            _compression(compression) { }
+        _backingStore(store),
+        _compression(compression)
+    { }
 
     bool read(DocumentIdT key, Value &value) const;
     void visit(const IDocumentStore::LidVector &lids, const DocumentTypeRepo &repo, IDocumentVisitor &visitor) const;
     void write(DocumentIdT, const Value &) {}
     void erase(DocumentIdT) {}
-    const CompressionConfig &getCompression(void) const { return _compression; }
+    const CompressionConfig &getCompression() const { return _compression; }
+    void reconfigure(const CompressionConfig &compression);
 private:
     IDataStore &_backingStore;
-    const CompressionConfig _compression;
+    CompressionConfig _compression;
 };
 
 void
@@ -172,6 +174,11 @@ BackingStore::read(DocumentIdT key, Value &value) const {
     return found;
 }
 
+void
+BackingStore::reconfigure(const CompressionConfig &compression) {
+    _compression = compression;
+}
+
 }
 
 using CacheParams = vespalib::CacheParam<
@@ -201,8 +208,15 @@ DocumentStore::DocumentStore(const Config & config, IDataStore & store)
     _cache->reserveElements(config.getInitialCacheEntries());
 }
 
-DocumentStore::~DocumentStore()
-{
+DocumentStore::~DocumentStore() {}
+
+void
+DocumentStore::reconfigure(const Config & config) {
+    _cache->setCapacityBytes(config.getMaxCacheBytes());
+    _store->reconfigure(config.getCompression());
+    _visitCache->reconfigure(_config.getMaxCacheBytes(), config.getCompression());
+
+    _config = config;
 }
 
 bool
diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.h b/searchlib/src/vespa/searchlib/docstore/documentstore.h
index 4ba5c27cd07..af173862295 100644
--- a/searchlib/src/vespa/searchlib/docstore/documentstore.h
+++ b/searchlib/src/vespa/searchlib/docstore/documentstore.h
@@ -100,6 +100,7 @@ public:
     bool canShrinkLidSpace() const override;
     size_t getEstimatedShrinkLidSpaceGain() const override;
     void shrinkLidSpace() override;
+    void reconfigure(const Config & config);
 
 private:
     bool useCache() const;
diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
index 4fa4142813c..53455cb1223 100644
--- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
@@ -67,6 +67,10 @@ LogDataStore::LogDataStore(vespalib::ThreadExecutor &executor,
     updateSerialNum();
 }
 
+void LogDataStore::reconfigure(const Config & config) {
+    _config = config;
+}
+
 void
 LogDataStore::updateSerialNum()
 {
@@ -274,12 +278,9 @@ LogDataStore::compact(uint64_t syncToken)
     uint64_t usage = getDiskFootprint();
     uint64_t bloat = getDiskBloat();
     LOG(debug, "%s", bloatMsg(bloat, usage).c_str());
-    if ((_fileChunks.size() > 1) &&
-           ( isBucketSpreadTooLarge(getMaxBucketSpread()) ||
-             isBloatOverLimit(bloat, usage)))
-    {
+    if (_fileChunks.size() > 1) {
         LOG(info, "%s. Will compact", bloatMsg(bloat, usage).c_str());
-        compactWorst();
+        compactWorst(_config.getMaxDiskBloatFactor(), _config.getMaxBucketSpread());
         usage = getDiskFootprint();
         bloat = getDiskBloat();
         LOG(info, "Done compacting. %s", bloatMsg(bloat, usage).c_str());
@@ -299,7 +300,7 @@ LogDataStore::getMaxCompactGain() const
         bloat = 0;
     }
     size_t spreadAsBloat = diskFootPrint * (1.0 - 1.0/maxSpread);
-    if ( ! isBucketSpreadTooLarge(maxSpread)) {
+    if ( maxSpread < _config.getMaxBucketSpread()) {
         spreadAsBloat = 0;
     }
     return (bloat + spreadAsBloat);
@@ -348,7 +349,7 @@ LogDataStore::getMaxBucketSpread() const
 }
 
 std::pair<bool, LogDataStore::FileId>
-LogDataStore::findNextToCompact()
+LogDataStore::findNextToCompact(double bloatLimit, double spreadLimit)
 {
     typedef std::multimap<double, FileId, std::greater<double>> CostMap;
     CostMap worstBloat;
@@ -376,10 +377,10 @@ LogDataStore::findNextToCompact()
         }
     }
     std::pair<bool, FileId> retval(false, FileId(-1));
-    if ( ! worstBloat.empty() && (worstBloat.begin()->first > _config.getMaxDiskBloatFactor())) {
+    if ( ! worstBloat.empty() && (worstBloat.begin()->first > bloatLimit)) {
         retval.first = true;
         retval.second = worstBloat.begin()->second;
-    } else if ( ! worstSpread.empty() && (worstSpread.begin()->first > _config.getMaxBucketSpread())) {
+    } else if ( ! worstSpread.empty() && (worstSpread.begin()->first > spreadLimit)) {
         retval.first = true;
         retval.second = worstSpread.begin()->second;
     }
@@ -390,8 +391,8 @@ LogDataStore::findNextToCompact()
 }
 
 void
-LogDataStore::compactWorst() {
-    auto worst = findNextToCompact();
+LogDataStore::compactWorst(double bloatLimit, double spreadLimit) {
+    auto worst = findNextToCompact(bloatLimit, spreadLimit);
     if (worst.first) {
         compactFile(worst.second);
     }
diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h
index eb46e5438a9..ab1c0026560 100644
--- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h
+++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h
@@ -74,11 +74,10 @@ public:
 
         size_t getNumThreads() const { return _numThreads; }
         bool crcOnReadDisabled() const { return _skipCrcOnRead; }
-        void disableCrcOnRead(bool v) { _skipCrcOnRead = v; }
         bool compact2ActiveFile() const { return _compactToActiveFile; }
         const CompressionConfig & compactCompression() const { return _compactCompression; }
-
         const WriteableFileChunk::Config & getFileConfig() const { return _fileConfig; }
+        Config & disableCrcOnRead(bool v) { _skipCrcOnRead = v; return *this;}
     private:
         size_t                      _maxFileSize;
         double                      _maxDiskBloatFactor;
@@ -201,18 +200,17 @@ public:
     static NameIdSet findIncompleteCompactedFiles(const NameIdSet & partList);
 
     NameIdSet getAllActiveFiles() const;
+    void reconfigure(const Config & config);
 
 private:
     class WrapVisitor;
     class WrapVisitorProgress;
     class FileChunkHolder;
 
-    void waitForUnblock();
-
     // Implements ISetLid API
     void setLid(const LockGuard & guard, uint32_t lid, const LidInfo & lm) override;
 
-    void compactWorst();
+    void compactWorst(double bloatLimit, double spreadLimit);
     void compactFile(FileId chunkId);
 
     typedef attribute::RcuVector<uint64_t> LidInfoVector;
@@ -257,9 +255,6 @@ private:
         _active = fileId;
     }
 
-    bool isBucketSpreadTooLarge(double spread) const {
-        return (spread >= _config.getMaxBucketSpread());
-    }
     double getMaxBucketSpread() const;
 
     FileChunk::UP createReadOnlyFile(FileId fileId, NameId nameId);
@@ -273,13 +268,6 @@ private:
     bool isReadOnly() const { return _readOnly; }
     void updateSerialNum();
 
-    bool isBloatOverLimit() const {
-        return isBloatOverLimit(getDiskBloat(), getDiskFootprint());
-    }
-    bool isBloatOverLimit(uint64_t bloat, uint64_t usage) const {
-        return (usage*_config.getMaxDiskBloatFactor() < bloat);
-    }
-
     size_t computeNumberOfSignificantBucketIdBits(const IBucketizer & bucketizer, FileId fileId) const;
 
     /*
@@ -301,7 +289,7 @@ private:
         return (_fileChunks.empty() ? 0 : _fileChunks.back()->getLastPersistedSerialNum());
     }
     bool shouldCompactToActiveFile(size_t compactedSize) const;
-    std::pair<bool, FileId> findNextToCompact();
+    std::pair<bool, FileId> findNextToCompact(double bloatLimit, double spreadLimit);
     void incGeneration();
     bool canShrinkLidSpace(const vespalib::LockGuard &guard) const;
 
@@ -326,4 +314,3 @@ private:
 };
 
 } // namespace search
-
diff --git a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp
index 313bc2f52d6..ad9c4f0aa4f 100644
--- a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp
@@ -25,5 +25,11 @@ LogDocumentStore::~LogDocumentStore()
 {
 }
 
+void
+LogDocumentStore::reconfigure(const Config & config) {
+    DocumentStore::reconfigure(config);
+    _backingStore.reconfigure(config.getLogConfig());
+}
+
 } // namespace search
 
diff --git a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.h b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.h
index 3f0e2642529..16fb6036519 100644
--- a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.h
+++ b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.h
@@ -6,15 +6,9 @@
 #include "logdatastore.h"
 #include <vespa/searchlib/common/tunefileinfo.h>
 
-namespace search
-{
-
-namespace common
-{
-
-class FileHeaderContext;
+namespace search {
 
-}
+namespace common { class FileHeaderContext; }
 
 /**
  * Simple document store that contains serialized Document instances.
@@ -56,8 +50,7 @@ public:
                      transactionlog::SyncProxy &tlSyncer,
                      const IBucketizer::SP & bucketizer);
     ~LogDocumentStore();
-    LogDataStore::Config       & getLogConfig()       { return _backingStore.getConfig(); }
-    const LogDataStore::Config & getLogConfig() const { return _backingStore.getConfig(); }
+    void reconfigure(const Config & config);
 private:
     void compact(uint64_t syncToken) override       { _backingStore.compact(syncToken); }
     LogDataStore _backingStore;
diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp
index b3fd236d73d..8f73c9862ae 100644
--- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp
@@ -145,12 +145,25 @@ VisitCache::BackingStore::read(const KeySet &key, CompressedBlobSet &blobs) cons
     return ! blobs.empty();
 }
 
+void
+VisitCache::BackingStore::reconfigure(const CompressionConfig &compression) {
+    _compression = compression;
+}
+
+
 VisitCache::VisitCache(IDataStore &store, size_t cacheSize, const CompressionConfig &compression) :
     _store(store, compression),
     _cache(std::make_unique<Cache>(_store, cacheSize))
 {
 }
 
+void
+VisitCache::reconfigure(size_t cacheSize, const CompressionConfig &compression) {
+    _store.reconfigure(compression);
+    _cache->setCapacityBytes(cacheSize);
+}
+
+
 VisitCache::Cache::IdSet
 VisitCache::Cache::findSetsContaining(const LockGuard &, const KeySet & keys) const {
     IdSet found;
diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h
index a89620b7bde..1bf867c5580 100644
--- a/searchlib/src/vespa/searchlib/docstore/visitcache.h
+++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h
@@ -104,6 +104,7 @@ public:
     void invalidate(uint32_t key) { remove(key); }
 
     CacheStats getCacheStats() const;
+    void reconfigure(size_t cacheSize, const CompressionConfig &compression);
 private:
     /**
      * This implments the interface the cache uses when it has a cache miss.
@@ -120,10 +121,11 @@ private:
         bool read(const KeySet &key, CompressedBlobSet &blobs) const;
         void write(const KeySet &, const CompressedBlobSet &) { }
         void erase(const KeySet &) { }
-        const CompressionConfig &getCompression() const { return _compression; }
+        void reconfigure(const CompressionConfig &compression);
+
     private:
-        IDataStore                        &_backingStore;
-        const CompressionConfig  _compression;
+        IDataStore        &_backingStore;
+        CompressionConfig  _compression;
     };
 
     using CacheParams = vespalib::CacheParam<