Add test for proper deduction of target num docs.

1 files changed, 75 insertions, 34 deletions

diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp
index 9dd5ecacd91..ae81a235b0e 100644
--- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp
+++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp
@@ -21,11 +21,9 @@
 #include <vespa/searchcore/proton/common/subdbtype.h>
 #include <vespa/searchcore/proton/test/transport_helper.h>
 #include <vespa/searchsummary/config/config-juniperrc.h>
-#include <vespa/document/config/documenttypes_config_fwd.h>
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/fileacquirer/config-filedistributorrpc.h>
 #include <vespa/vespalib/util/varholder.h>
-#include <vespa/vespalib/util/size_literals.h>
 #include <vespa/vespalib/testkit/testapp.h>
 #include <map>
 #include <thread>
@@ -73,7 +71,7 @@ struct ConfigTestFixture {
     std::shared_ptr<IConfigContext> context;
     int idcounter;
 
-    ConfigTestFixture(const std::string & id)
+    explicit ConfigTestFixture(const std::string & id)
         : configId(id),
           protonBuilder(),
           documenttypesBuilder(),
@@ -93,7 +91,15 @@ struct ConfigTestFixture {
 
     ~ConfigTestFixture() = default;
 
-    DoctypeFixture *addDocType(const std::string &name, bool isGlobal = false) {
+    DoctypeFixture *addDocType(const std::string &name) {
+        return addDocType(name, ProtonConfig::Documentdb::Mode::INDEX);
+    }
+
+    DoctypeFixture *addDocType(const std::string &name, ProtonConfig::Documentdb::Mode mode) {
+        return addDocType(name, mode, false);
+    }
+
+    DoctypeFixture *addDocType(const std::string &name, ProtonConfig::Documentdb::Mode mode, bool isGlobal) {
         DocumenttypesConfigBuilder::Documenttype dt;
         dt.bodystruct = -1270491200;
         dt.headerstruct = 306916075;
@@ -106,6 +112,7 @@ struct ConfigTestFixture {
         db.inputdoctypename = name;
         db.configid = configId + "/" + name;
         db.global = isGlobal;
+        db.mode = mode;
         protonBuilder.documentdb.push_back(db);
 
         DoctypeFixture::UP fixture = std::make_unique<DoctypeFixture>();
@@ -118,7 +125,7 @@ struct ConfigTestFixture {
         set.addBuilder(db.configid, &fixture->summaryBuilder);
         set.addBuilder(db.configid, &fixture->juniperrcBuilder);
         set.addBuilder(db.configid, &fixture->importedFieldsBuilder);
-        return dbConfig.emplace(std::make_pair(name, std::move(fixture))).first->second.get();
+        return dbConfig.emplace(name, std::move(fixture)).first->second.get();
     }
 
     void removeDocType(const std::string & name)
@@ -142,20 +149,20 @@ struct ConfigTestFixture {
         }
     }
 
-    bool configEqual(const std::string & name, DocumentDBConfig::SP dbc) {
+    bool configEqual(const std::string & name, const DocumentDBConfig & dbc) {
         auto itr = dbConfig.find(name);
         ASSERT_TRUE(itr != dbConfig.end());
         const auto *fixture = itr->second.get();
-        return (fixture->attributesBuilder == dbc->getAttributesConfig() &&
-                fixture->rankProfilesBuilder == dbc->getRankProfilesConfig() &&
-                fixture->indexschemaBuilder == dbc->getIndexschemaConfig() &&
-                fixture->summaryBuilder == dbc->getSummaryConfig() &&
-                fixture->juniperrcBuilder == dbc->getJuniperrcConfig());
+        return (fixture->attributesBuilder == dbc.getAttributesConfig() &&
+                fixture->rankProfilesBuilder == dbc.getRankProfilesConfig() &&
+                fixture->indexschemaBuilder == dbc.getIndexschemaConfig() &&
+                fixture->summaryBuilder == dbc.getSummaryConfig() &&
+                fixture->juniperrcBuilder == dbc.getJuniperrcConfig());
     }
 
-    bool configEqual(BootstrapConfig::SP bootstrapConfig) {
-        return (protonBuilder == bootstrapConfig->getProtonConfig() &&
-                documenttypesBuilder == bootstrapConfig->getDocumenttypesConfig());
+    bool configEqual(const BootstrapConfig & bootstrapConfig) const {
+        return (protonBuilder == bootstrapConfig.getProtonConfig() &&
+                documenttypesBuilder == bootstrapConfig.getDocumenttypesConfig());
     }
 
     BootstrapConfig::SP getBootstrapConfig(int64_t generation, const HwInfo & hwInfo) const {
@@ -180,7 +187,7 @@ struct ProtonConfigOwner : public proton::IProtonConfigurer
 
     ProtonConfigOwner() : _configured(false), _config() { }
     ~ProtonConfigOwner() override;
-    bool waitUntilConfigured(vespalib::duration timeout) {
+    bool waitUntilConfigured(vespalib::duration timeout) const {
         vespalib::Timer timer;
         while (timer.elapsed() < timeout) {
             if (getConfigured())
@@ -189,7 +196,7 @@ struct ProtonConfigOwner : public proton::IProtonConfigurer
         }
         return getConfigured();
     }
-    virtual void reconfigure(std::shared_ptr<ProtonConfigSnapshot> cfg) override {
+    void reconfigure(std::shared_ptr<ProtonConfigSnapshot> cfg) override {
         std::lock_guard<std::mutex> guard(_mutex);
         _config.set(cfg);
         _configured = true;
@@ -198,11 +205,11 @@ struct ProtonConfigOwner : public proton::IProtonConfigurer
         std::lock_guard<std::mutex> guard(_mutex);
         return _configured;
     }
-    BootstrapConfig::SP getBootstrapConfig() {
+    BootstrapConfig::SP getBootstrapConfig() const {
         auto snapshot = _config.get();
         return snapshot->getBootstrapConfig();
     }
-    DocumentDBConfig::SP getDocumentDBConfig(const vespalib::string &name)
+    DocumentDBConfig::SP getDocumentDBConfig(const vespalib::string &name) const
     {
         auto snapshot = _config.get();
         auto &dbcs = snapshot->getDocumentDBConfigs();
@@ -210,7 +217,7 @@ struct ProtonConfigOwner : public proton::IProtonConfigurer
         if (dbitr != dbcs.end()) {
             return dbitr->second;
         } else {
-            return DocumentDBConfig::SP();
+            return {};
         }
     }
 };
@@ -232,18 +239,18 @@ TEST_FFF("require that bootstrap config manager updates config", ConfigTestFixtu
                                                                  BootstrapConfigManager(f1.configId),
                                                                  ConfigRetriever(f2.createConfigKeySet(), f1.context)) {
     f2.update(f3.getBootstrapConfigs());
-    ASSERT_TRUE(f1.configEqual(f2.getConfig()));
+    ASSERT_TRUE(f1.configEqual(*f2.getConfig()));
     f1.protonBuilder.rpcport = 9010;
-    ASSERT_FALSE(f1.configEqual(f2.getConfig()));
+    ASSERT_FALSE(f1.configEqual(*f2.getConfig()));
     f1.reload();
     f2.update(f3.getBootstrapConfigs());
-    ASSERT_TRUE(f1.configEqual(f2.getConfig()));
+    ASSERT_TRUE(f1.configEqual(*f2.getConfig()));
 
     f1.addDocType("foobar");
-    ASSERT_FALSE(f1.configEqual(f2.getConfig()));
+    ASSERT_FALSE(f1.configEqual(*f2.getConfig()));
     f1.reload();
     f2.update(f3.getBootstrapConfigs());
-    ASSERT_TRUE(f1.configEqual(f2.getConfig()));
+    ASSERT_TRUE(f1.configEqual(*f2.getConfig()));
 }
 
 DocumentDBConfig::SP
@@ -267,7 +274,7 @@ TEST_FF("require that documentdb config manager subscribes for config",
     f1.addDocType("typea");
     const ConfigKeySet keySet(f2.createConfigKeySet());
     ASSERT_EQUAL(9u, keySet.size());
-    ASSERT_TRUE(f1.configEqual("typea", getDocumentDBConfig(f1, f2)));
+    ASSERT_TRUE(f1.configEqual("typea", *getDocumentDBConfig(f1, f2)));
 }
 
 TEST_FF("require that documentdb config manager builds schema with imported attribute fields"
@@ -305,12 +312,12 @@ TEST_FFF("require that proton config fetcher follows changes to bootstrap",
          ProtonConfigFetcher(f1.transport.transport(), ConfigUri(f1.configId, f1.context), f2, 60s)) {
     f3.start();
     ASSERT_TRUE(f2._configured);
-    ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig()));
+    ASSERT_TRUE(f1.configEqual(*f2.getBootstrapConfig()));
     f2._configured = false;
     f1.protonBuilder.rpcport = 9010;
     f1.reload();
     ASSERT_TRUE(f2.waitUntilConfigured(120s));
-    ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig()));
+    ASSERT_TRUE(f1.configEqual(*f2.getBootstrapConfig()));
     f3.close();
 }
 
@@ -324,13 +331,13 @@ TEST_FFF("require that proton config fetcher follows changes to doctypes",
     f1.addDocType("typea");
     f1.reload();
     ASSERT_TRUE(f2.waitUntilConfigured(60s));
-    ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig()));
+    ASSERT_TRUE(f1.configEqual(*f2.getBootstrapConfig()));
 
     f2._configured = false;
     f1.removeDocType("typea");
     f1.reload();
     ASSERT_TRUE(f2.waitUntilConfigured(60s));
-    ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig()));
+    ASSERT_TRUE(f1.configEqual(*f2.getBootstrapConfig()));
     f3.close();
 }
 
@@ -346,9 +353,9 @@ TEST_FFF("require that proton config fetcher reconfigures dbowners",
     f1.addDocType("typea");
     f1.reload();
     ASSERT_TRUE(f2.waitUntilConfigured(60s));
-    ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig()));
+    ASSERT_TRUE(f1.configEqual(*f2.getBootstrapConfig()));
     ASSERT_TRUE(static_cast<bool>(f2.getDocumentDBConfig("typea")));
-    ASSERT_TRUE(f1.configEqual("typea", f2.getDocumentDBConfig("typea")));
+    ASSERT_TRUE(f1.configEqual("typea", *f2.getDocumentDBConfig("typea")));
 
     // Remove and verify that config for db is no longer provided
     f2._configured = false;
@@ -363,11 +370,45 @@ TEST_FF("require that lid space compaction is disabled for globally distributed
         ConfigTestFixture("search"),
         DocumentDBConfigManager(f1.configId + "/global", "global"))
 {
-    f1.addDocType("global", true);
+    f1.addDocType("global", ProtonConfig::Documentdb::Mode::INDEX, true);
     auto config = getDocumentDBConfig(f1, f2);
     EXPECT_TRUE(config->getMaintenanceConfigSP()->getLidSpaceCompactionConfig().isDisabled());
 }
 
+HwInfo
+createHwInfoWithMemory(uint64_t mem) {
+    return {HwInfo::Disk(1, false, false), HwInfo::Memory(mem), HwInfo::Cpu(1)};
+}
+
+TEST_FF("require that target numdocs is fixed 1k for indexed mode",
+        ConfigTestFixture("search"),
+        DocumentDBConfigManager(f1.configId + "/test", "test"))
+{
+    f1.addDocType("test", ProtonConfig::Documentdb::Mode::INDEX, true);
+    for (uint64_t memory : {1_Gi, 10_Gi}) {
+        auto config = getDocumentDBConfig(f1, f2, createHwInfoWithMemory(memory));
+        AllocStrategy strategy = config->get_alloc_config().make_alloc_strategy(SubDbType::READY);
+        EXPECT_EQUAL(1024u, strategy.get_grow_strategy().getMinimumCapacity());
+    }
+}
+
+TEST_FF("require that target numdocs follows memory for streaming mode",
+        ConfigTestFixture("search"),
+        DocumentDBConfigManager(f1.configId + "/test", "test"))
+{
+    f1.addDocType("test", ProtonConfig::Documentdb::Mode::STREAMING, true);
+    {
+        auto config = getDocumentDBConfig(f1, f2, createHwInfoWithMemory(1_Gi));
+        AllocStrategy strategy = config->get_alloc_config().make_alloc_strategy(SubDbType::READY);
+        EXPECT_EQUAL(23342213u, strategy.get_grow_strategy().getMinimumCapacity());
+    }
+    {
+        auto config = getDocumentDBConfig(f1, f2, createHwInfoWithMemory(10_Gi));
+        AllocStrategy strategy = config->get_alloc_config().make_alloc_strategy(SubDbType::READY);
+        EXPECT_EQUAL(233422135u, strategy.get_grow_strategy().getMinimumCapacity());
+    }
+}
+
 TEST_FF("require that prune removed documents interval can be set based on age",
         ConfigTestFixture("test"),
         DocumentDBConfigManager(f1.configId + "/test", "test"))
@@ -383,7 +424,7 @@ TEST_FF("require that docstore config computes cachesize automatically if unset"
         ConfigTestFixture("test"),
         DocumentDBConfigManager(f1.configId + "/test", "test"))
 {
-    HwInfo hwInfo(HwInfo::Disk(1, false, false), HwInfo::Memory(1000000), HwInfo::Cpu(1));
+    HwInfo hwInfo = createHwInfoWithMemory(1000000);
     f1.addDocType("test");
     f1.protonBuilder.summary.cache.maxbytes = 2000;
     auto config = getDocumentDBConfig(f1, f2, hwInfo);
@@ -400,7 +441,7 @@ TEST_FF("require that docstore config computes cachesize automatically if unset"
 
 GrowStrategy
 growStrategy(uint32_t initial) {
-    return GrowStrategy(initial, 0.1, 1, initial, 0.15);
+    return {initial, 0.1, 1, initial, 0.15};
 }
 TEST_FF("require that allocation config is propagated",
         ConfigTestFixture("test"),