Let cache be computed based on available memory.

8 files changed, 57 insertions, 28 deletions

diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp
index 37e4dfc486d..d305cbf532b 100644
--- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp
+++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp
@@ -14,12 +14,12 @@
 #include <vespa/searchcore/proton/server/proton_config_fetcher.h>
 #include <vespa/searchcore/proton/server/proton_config_snapshot.h>
 #include <vespa/searchcore/proton/server/i_proton_configurer.h>
+#include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchsummary/config/config-juniperrc.h>
 #include <vespa/searchcore/config/config-ranking-constants.h>
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/varholder.h>
 #include <vespa/config-bucketspaces.h>
-#include <mutex>
 
 using namespace config;
 using namespace proton;
@@ -60,6 +60,7 @@ struct ConfigTestFixture {
     ConfigSet set;
     IConfigContext::SP context;
     int idcounter;
+    HwInfo hwInfo;
 
     ConfigTestFixture(const std::string & id)
         : configId(id),
@@ -232,14 +233,20 @@ TEST_FFF("require that bootstrap config manager updates config", ConfigTestFixtu
 }
 
 DocumentDBConfig::SP
-getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr)
+getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr, const HwInfo & hwInfo)
 {
     ConfigRetriever retriever(mgr.createConfigKeySet(), f.context);
     mgr.forwardConfig(f.getBootstrapConfig(1));
-    mgr.update(retriever.getBootstrapConfigs()); // Cheating, but we only need the configs
+    mgr.update(retriever.getBootstrapConfigs(), hwInfo); // Cheating, but we only need the configs
     return mgr.getConfig();
 }
 
+DocumentDBConfig::SP
+getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr)
+{
+    return getDocumentDBConfig(f, mgr, HwInfo());
+}
+
 TEST_FF("require that documentdb config manager subscribes for config",
         ConfigTestFixture("search"),
         DocumentDBConfigManager(f1.configId + "/typea", "typea")) {
@@ -274,7 +281,7 @@ TEST_FF("require that documentdb config manager builds schema with imported attr
 TEST_FFF("require that proton config fetcher follows changes to bootstrap",
          ConfigTestFixture("search"),
          ProtonConfigOwner(),
-         ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) {
+         ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) {
     f3.start();
     ASSERT_TRUE(f2._configured);
     ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig()));
@@ -289,7 +296,7 @@ TEST_FFF("require that proton config fetcher follows changes to bootstrap",
 TEST_FFF("require that proton config fetcher follows changes to doctypes",
          ConfigTestFixture("search"),
          ProtonConfigOwner(),
-         ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) {
+         ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) {
     f3.start();
 
     f2._configured = false;
@@ -309,7 +316,7 @@ TEST_FFF("require that proton config fetcher follows changes to doctypes",
 TEST_FFF("require that proton config fetcher reconfigures dbowners",
          ConfigTestFixture("search"),
          ProtonConfigOwner(),
-         ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) {
+         ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) {
     f3.start();
     ASSERT_FALSE(f2.getDocumentDBConfig("typea"));
 
@@ -351,4 +358,18 @@ TEST_FF("require that prune removed documents interval can be set based on age",
     EXPECT_EQUAL(20, config->getMaintenanceConfigSP()->getPruneRemovedDocumentsConfig().getInterval());
 }
 
+TEST_FF("require that docstore config computes cachesize automatically if unset",
+        ConfigTestFixture("test"),
+        DocumentDBConfigManager(f1.configId + "/test", "test"))
+{
+    HwInfo hwInfo(HwInfo::Disk(1, false, false), HwInfo::Memory(1000000), HwInfo::Cpu(1));
+    f1.addDocType("test");
+    f1.protonBuilder.summary.cache.maxbytes = 2000;
+    auto config = getDocumentDBConfig(f1, f2, hwInfo);
+    EXPECT_EQUAL(2000ul, config->getStoreConfig().getMaxCacheBytes());
+    f1.protonBuilder.summary.cache.maxbytes = -1;
+    config = getDocumentDBConfig(f1, f2, hwInfo);
+    EXPECT_EQUAL(50000ul, config->getStoreConfig().getMaxCacheBytes());
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h
index 1f888ac1739..1e2c269178f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h
@@ -46,8 +46,7 @@ public:
                     const ProtonConfigSP &protonConfig,
                     const FiledistributorrpcConfigSP &filedistRpcConfSP,
                     const BucketspacesConfigSP &bucketspaces,
-                    const search::TuneFileDocumentDB::SP &
-                    _tuneFileDocumentDB);
+                    const search::TuneFileDocumentDB::SP &_tuneFileDocumentDB);
     ~BootstrapConfig();
 
     const document::DocumenttypesConfig &
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
index def18b76360..27bbd019ff5 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
@@ -2,6 +2,8 @@
 
 #include "documentdbconfigmanager.h"
 #include "bootstrapconfig.h"
+#include <vespa/searchcore/proton/common/hw_info.h>
+#include <vespa/searchcore/config/config-ranking-constants.h>
 #include <vespa/config-imported-fields.h>
 #include <vespa/config-rank-profiles.h>
 #include <vespa/config-summarymap.h>
@@ -10,7 +12,6 @@
 #include <vespa/searchcommon/common/schemaconfigurer.h>
 #include <vespa/searchlib/index/schemautil.h>
 #include <vespa/searchsummary/config/config-juniperrc.h>
-#include <vespa/searchcore/config/config-ranking-constants.h>
 #include <vespa/vespalib/time/time_box.h>
 
 #include <vespa/log/log.h>
@@ -157,14 +158,15 @@ deriveCompression(const T & config) {
 }
 
 DocumentStore::Config
-getStoreConfig(const ProtonConfig::Summary::Cache & cache)
+getStoreConfig(const ProtonConfig::Summary::Cache & cache, const HwInfo & hwInfo)
 {
-    return DocumentStore::Config(deriveCompression(cache.compression), cache.maxbytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching);
+    size_t maxBytes = (cache.maxbytes < 0) ? hwInfo.memory().sizeBytes()*0.05 : cache.maxbytes;
+    return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching);
 }
 
 LogDocumentStore::Config
-deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush) {
-    DocumentStore::Config config(getStoreConfig(summary.cache));
+deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush, const HwInfo & hwInfo) {
+    DocumentStore::Config config(getStoreConfig(summary.cache, hwInfo));
     const ProtonConfig::Summary::Log & log(summary.log);
     const ProtonConfig::Summary::Log::Chunk & chunk(log.chunk);
     WriteableFileChunk::Config fileConfig(deriveCompression(chunk.compression), chunk.maxbytes);
@@ -177,8 +179,8 @@ deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::M
     return LogDocumentStore::Config(config, logConfig);
 }
 
-search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton) {
-    return deriveConfig(proton.summary, proton.flush.memory);
+search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton, const HwInfo & hwInfo) {
+    return deriveConfig(proton.summary, proton.flush.memory, hwInfo);
 }
 
 using AttributesConfigSP = DocumentDBConfig::AttributesConfigSP;
@@ -201,7 +203,7 @@ filterImportedAttributes(const AttributesConfigSP &attrCfg)
 }
 
 void
-DocumentDBConfigManager::update(const ConfigSnapshot &snapshot)
+DocumentDBConfigManager::update(const ConfigSnapshot &snapshot, const HwInfo & hwInfo)
 {
     using RankProfilesConfigSP = DocumentDBConfig::RankProfilesConfigSP;
     using RankingConstantsConfigSP = std::shared_ptr<vespa::config::search::core::RankingConstantsConfig>;
@@ -313,7 +315,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot)
 
     Schema::SP schema(buildSchema(*newAttributesConfig, *newSummaryConfig, *newIndexschemaConfig));
     newMaintenanceConfig = buildMaintenanceConfig(_bootstrapConfig, _docTypeName);
-    search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig());
+    search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig(), hwInfo);
     if (newMaintenanceConfig && oldMaintenanceConfig && *newMaintenanceConfig == *oldMaintenanceConfig) {
         newMaintenanceConfig = oldMaintenanceConfig;
     }
@@ -375,7 +377,8 @@ forwardConfig(const BootstrapConfig::SP & config)
 }
 
 DocumentDBConfigHelper::DocumentDBConfigHelper(const DirSpec &spec, const vespalib::string &docTypeName)
-    : _mgr("", docTypeName),
+    : _hwInfo(std::make_unique<HwInfo>()),
+      _mgr("", docTypeName),
       _retriever(make_unique<ConfigRetriever>(_mgr.createConfigKeySet(), make_shared<ConfigContext>(spec)))
 { }
 
@@ -387,7 +390,7 @@ DocumentDBConfigHelper::nextGeneration(int timeoutInMillis)
     ConfigSnapshot snapshot(_retriever->getBootstrapConfigs(timeoutInMillis));
     if (snapshot.empty())
         return false;
-    _mgr.update(snapshot);
+    _mgr.update(snapshot, *_hwInfo);
     return true;
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h
index 80654c7588b..44e0c111f47 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h
@@ -10,6 +10,8 @@
 namespace proton {
 
 class BootstrapConfig;
+class HwInfo;
+
 /**
  * This class manages the subscription for documentdb configs.
  */
@@ -35,7 +37,7 @@ private:
 public:
     DocumentDBConfigManager(const vespalib::string &configId, const vespalib::string &docTypeName);
     ~DocumentDBConfigManager();
-    void update(const config::ConfigSnapshot & snapshot);
+    void update(const config::ConfigSnapshot & snapshot, const HwInfo & hwInfo);
 
     DocumentDBConfig::SP getConfig() const;
 
@@ -57,6 +59,7 @@ public:
     DocumentDBConfig::SP getConfig() const;
     void forwardConfig(const std::shared_ptr<BootstrapConfig> & config);
 private:
+    std::unique_ptr<HwInfo> _hwInfo;
     DocumentDBConfigManager _mgr;
     std::unique_ptr<config::ConfigRetriever> _retriever;
 };
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index 8ce7e0c79fe..28811fef84a 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -188,7 +188,8 @@ Proton::Proton(const config::ConfigUri & configUri,
       // serializing startup.
       _executor(1, 128 * 1024),
       _protonConfigurer(_executor, *this),
-      _protonConfigFetcher(configUri, _protonConfigurer, subscribeTimeout),
+      _hwInfo(),
+      _protonConfigFetcher(configUri, _hwInfo, _protonConfigurer, subscribeTimeout),
       _warmupExecutor(),
       _summaryExecutor(),
       _queryLimiter(),
@@ -202,7 +203,6 @@ Proton::Proton(const config::ConfigUri & configUri,
       _initStarted(false),
       _initComplete(false),
       _initDocumentDbsInSequence(false),
-      _hwInfo(),
       _hwInfoSampler(),
       _documentDBReferenceRegistry()
 {
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h
index 8d1026340ca..e6d7093fd44 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.h
@@ -110,6 +110,7 @@ private:
     TransportServer::UP             _fs4Server;
     vespalib::ThreadStackExecutor   _executor;
     ProtonConfigurer                _protonConfigurer;
+    HwInfo                          _hwInfo;
     ProtonConfigFetcher             _protonConfigFetcher;
     std::unique_ptr<vespalib::ThreadStackExecutorBase> _warmupExecutor;
     std::unique_ptr<vespalib::ThreadStackExecutorBase> _summaryExecutor;
@@ -124,7 +125,6 @@ private:
     bool                            _initStarted;
     bool                            _initComplete;
     bool                            _initDocumentDbsInSequence;
-    HwInfo                          _hwInfo;
     std::unique_ptr<HwInfoSampler>  _hwInfoSampler;
     std::shared_ptr<IDocumentDBReferenceRegistry> _documentDBReferenceRegistry;
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp
index deeec695f26..f83b3f784f3 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp
@@ -17,10 +17,11 @@ using namespace std::chrono_literals;
 
 namespace proton {
 
-ProtonConfigFetcher::ProtonConfigFetcher(const config::ConfigUri & configUri, IProtonConfigurer &owner, uint64_t subscribeTimeout)
+ProtonConfigFetcher::ProtonConfigFetcher(const config::ConfigUri & configUri, const HwInfo &hwInfo, IProtonConfigurer &owner, uint64_t subscribeTimeout)
     : _bootstrapConfigManager(configUri.getConfigId()),
       _retriever(_bootstrapConfigManager.createConfigKeySet(), configUri.getContext(), subscribeTimeout),
       _owner(owner),
+      _hwInfo(hwInfo),
       _mutex(),
       _dbManagerMap(),
       _threadPool(128 * 1024, 1),
@@ -81,7 +82,7 @@ ProtonConfigFetcher::updateDocumentDBConfigs(const BootstrapConfig::SP & bootstr
     lock_guard guard(_mutex);
     for (auto & entry : _dbManagerMap) {
         entry.second->forwardConfig(bootstrapConfig);
-        entry.second->update(snapshot);
+        entry.second->update(snapshot, _hwInfo);
     }
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h
index c8d1e55e4e4..7165e364dc5 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h
@@ -17,6 +17,7 @@ namespace proton {
 
 class BootstrapConfig;
 class IProtonConfigurer;
+class HwInfo;
 
 /**
  * A ProtonConfigFetcher monitors all config in proton and document dbs for change
@@ -27,7 +28,7 @@ class ProtonConfigFetcher : public FastOS_Runnable
 public:
     using BootstrapConfigSP = std::shared_ptr<BootstrapConfig>;
 
-    ProtonConfigFetcher(const config::ConfigUri & configUri, IProtonConfigurer &owner, uint64_t subscribeTimeout);
+    ProtonConfigFetcher(const config::ConfigUri & configUri, const HwInfo & hwInfo,IProtonConfigurer &owner, uint64_t subscribeTimeout);
     ~ProtonConfigFetcher();
     /**
      * Get the current config generation.
@@ -52,9 +53,10 @@ private:
     using TimePoint = std::chrono::time_point<Clock>;
     using OldDocumentTypeRepo = std::pair<TimePoint, std::shared_ptr<document::DocumentTypeRepo>>;
 
-    BootstrapConfigManager _bootstrapConfigManager;
+    BootstrapConfigManager  _bootstrapConfigManager;
     config::ConfigRetriever _retriever;
-    IProtonConfigurer & _owner;
+    IProtonConfigurer     & _owner;
+    const HwInfo          & _hwInfo;
 
     mutable std::mutex _mutex; // Protects maps
     using lock_guard = std::lock_guard<std::mutex>;