diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-10-27 12:58:36 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2023-10-27 12:59:36 +0000 |
commit | 3be986861d51ae28e6f0df92ca71842e51949c96 (patch) | |
tree | ce2b4eaa36af62693a93db22917a4be4fa9629c7 | |
parent | 72cd1b31537080ee318d545656c0d4570e1e826a (diff) |
Add test for proper deduction of target num docs.
-rw-r--r-- | searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp | 109 |
1 files changed, 75 insertions, 34 deletions
diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 9dd5ecacd91..ae81a235b0e 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -21,11 +21,9 @@ #include <vespa/searchcore/proton/common/subdbtype.h> #include <vespa/searchcore/proton/test/transport_helper.h> #include <vespa/searchsummary/config/config-juniperrc.h> -#include <vespa/document/config/documenttypes_config_fwd.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/fileacquirer/config-filedistributorrpc.h> #include <vespa/vespalib/util/varholder.h> -#include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/testkit/testapp.h> #include <map> #include <thread> @@ -73,7 +71,7 @@ struct ConfigTestFixture { std::shared_ptr<IConfigContext> context; int idcounter; - ConfigTestFixture(const std::string & id) + explicit ConfigTestFixture(const std::string & id) : configId(id), protonBuilder(), documenttypesBuilder(), @@ -93,7 +91,15 @@ struct ConfigTestFixture { ~ConfigTestFixture() = default; - DoctypeFixture *addDocType(const std::string &name, bool isGlobal = false) { + DoctypeFixture *addDocType(const std::string &name) { + return addDocType(name, ProtonConfig::Documentdb::Mode::INDEX); + } + + DoctypeFixture *addDocType(const std::string &name, ProtonConfig::Documentdb::Mode mode) { + return addDocType(name, mode, false); + } + + DoctypeFixture *addDocType(const std::string &name, ProtonConfig::Documentdb::Mode mode, bool isGlobal) { DocumenttypesConfigBuilder::Documenttype dt; dt.bodystruct = -1270491200; dt.headerstruct = 306916075; @@ -106,6 +112,7 @@ struct ConfigTestFixture { db.inputdoctypename = name; db.configid = configId + "/" + name; db.global = isGlobal; + db.mode = mode; protonBuilder.documentdb.push_back(db); DoctypeFixture::UP fixture = std::make_unique<DoctypeFixture>(); @@ -118,7 +125,7 @@ struct ConfigTestFixture { set.addBuilder(db.configid, &fixture->summaryBuilder); set.addBuilder(db.configid, &fixture->juniperrcBuilder); set.addBuilder(db.configid, &fixture->importedFieldsBuilder); - return dbConfig.emplace(std::make_pair(name, std::move(fixture))).first->second.get(); + return dbConfig.emplace(name, std::move(fixture)).first->second.get(); } void removeDocType(const std::string & name) @@ -142,20 +149,20 @@ struct ConfigTestFixture { } } - bool configEqual(const std::string & name, DocumentDBConfig::SP dbc) { + bool configEqual(const std::string & name, const DocumentDBConfig & dbc) { auto itr = dbConfig.find(name); ASSERT_TRUE(itr != dbConfig.end()); const auto *fixture = itr->second.get(); - return (fixture->attributesBuilder == dbc->getAttributesConfig() && - fixture->rankProfilesBuilder == dbc->getRankProfilesConfig() && - fixture->indexschemaBuilder == dbc->getIndexschemaConfig() && - fixture->summaryBuilder == dbc->getSummaryConfig() && - fixture->juniperrcBuilder == dbc->getJuniperrcConfig()); + return (fixture->attributesBuilder == dbc.getAttributesConfig() && + fixture->rankProfilesBuilder == dbc.getRankProfilesConfig() && + fixture->indexschemaBuilder == dbc.getIndexschemaConfig() && + fixture->summaryBuilder == dbc.getSummaryConfig() && + fixture->juniperrcBuilder == dbc.getJuniperrcConfig()); } - bool configEqual(BootstrapConfig::SP bootstrapConfig) { - return (protonBuilder == bootstrapConfig->getProtonConfig() && - documenttypesBuilder == bootstrapConfig->getDocumenttypesConfig()); + bool configEqual(const BootstrapConfig & bootstrapConfig) const { + return (protonBuilder == bootstrapConfig.getProtonConfig() && + documenttypesBuilder == bootstrapConfig.getDocumenttypesConfig()); } BootstrapConfig::SP getBootstrapConfig(int64_t generation, const HwInfo & hwInfo) const { @@ -180,7 +187,7 @@ struct ProtonConfigOwner : public proton::IProtonConfigurer ProtonConfigOwner() : _configured(false), _config() { } ~ProtonConfigOwner() override; - bool waitUntilConfigured(vespalib::duration timeout) { + bool waitUntilConfigured(vespalib::duration timeout) const { vespalib::Timer timer; while (timer.elapsed() < timeout) { if (getConfigured()) @@ -189,7 +196,7 @@ struct ProtonConfigOwner : public proton::IProtonConfigurer } return getConfigured(); } - virtual void reconfigure(std::shared_ptr<ProtonConfigSnapshot> cfg) override { + void reconfigure(std::shared_ptr<ProtonConfigSnapshot> cfg) override { std::lock_guard<std::mutex> guard(_mutex); _config.set(cfg); _configured = true; @@ -198,11 +205,11 @@ struct ProtonConfigOwner : public proton::IProtonConfigurer std::lock_guard<std::mutex> guard(_mutex); return _configured; } - BootstrapConfig::SP getBootstrapConfig() { + BootstrapConfig::SP getBootstrapConfig() const { auto snapshot = _config.get(); return snapshot->getBootstrapConfig(); } - DocumentDBConfig::SP getDocumentDBConfig(const vespalib::string &name) + DocumentDBConfig::SP getDocumentDBConfig(const vespalib::string &name) const { auto snapshot = _config.get(); auto &dbcs = snapshot->getDocumentDBConfigs(); @@ -210,7 +217,7 @@ struct ProtonConfigOwner : public proton::IProtonConfigurer if (dbitr != dbcs.end()) { return dbitr->second; } else { - return DocumentDBConfig::SP(); + return {}; } } }; @@ -232,18 +239,18 @@ TEST_FFF("require that bootstrap config manager updates config", ConfigTestFixtu BootstrapConfigManager(f1.configId), ConfigRetriever(f2.createConfigKeySet(), f1.context)) { f2.update(f3.getBootstrapConfigs()); - ASSERT_TRUE(f1.configEqual(f2.getConfig())); + ASSERT_TRUE(f1.configEqual(*f2.getConfig())); f1.protonBuilder.rpcport = 9010; - ASSERT_FALSE(f1.configEqual(f2.getConfig())); + ASSERT_FALSE(f1.configEqual(*f2.getConfig())); f1.reload(); f2.update(f3.getBootstrapConfigs()); - ASSERT_TRUE(f1.configEqual(f2.getConfig())); + ASSERT_TRUE(f1.configEqual(*f2.getConfig())); f1.addDocType("foobar"); - ASSERT_FALSE(f1.configEqual(f2.getConfig())); + ASSERT_FALSE(f1.configEqual(*f2.getConfig())); f1.reload(); f2.update(f3.getBootstrapConfigs()); - ASSERT_TRUE(f1.configEqual(f2.getConfig())); + ASSERT_TRUE(f1.configEqual(*f2.getConfig())); } DocumentDBConfig::SP @@ -267,7 +274,7 @@ TEST_FF("require that documentdb config manager subscribes for config", f1.addDocType("typea"); const ConfigKeySet keySet(f2.createConfigKeySet()); ASSERT_EQUAL(9u, keySet.size()); - ASSERT_TRUE(f1.configEqual("typea", getDocumentDBConfig(f1, f2))); + ASSERT_TRUE(f1.configEqual("typea", *getDocumentDBConfig(f1, f2))); } TEST_FF("require that documentdb config manager builds schema with imported attribute fields" @@ -305,12 +312,12 @@ TEST_FFF("require that proton config fetcher follows changes to bootstrap", ProtonConfigFetcher(f1.transport.transport(), ConfigUri(f1.configId, f1.context), f2, 60s)) { f3.start(); ASSERT_TRUE(f2._configured); - ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig())); + ASSERT_TRUE(f1.configEqual(*f2.getBootstrapConfig())); f2._configured = false; f1.protonBuilder.rpcport = 9010; f1.reload(); ASSERT_TRUE(f2.waitUntilConfigured(120s)); - ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig())); + ASSERT_TRUE(f1.configEqual(*f2.getBootstrapConfig())); f3.close(); } @@ -324,13 +331,13 @@ TEST_FFF("require that proton config fetcher follows changes to doctypes", f1.addDocType("typea"); f1.reload(); ASSERT_TRUE(f2.waitUntilConfigured(60s)); - ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig())); + ASSERT_TRUE(f1.configEqual(*f2.getBootstrapConfig())); f2._configured = false; f1.removeDocType("typea"); f1.reload(); ASSERT_TRUE(f2.waitUntilConfigured(60s)); - ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig())); + ASSERT_TRUE(f1.configEqual(*f2.getBootstrapConfig())); f3.close(); } @@ -346,9 +353,9 @@ TEST_FFF("require that proton config fetcher reconfigures dbowners", f1.addDocType("typea"); f1.reload(); ASSERT_TRUE(f2.waitUntilConfigured(60s)); - ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig())); + ASSERT_TRUE(f1.configEqual(*f2.getBootstrapConfig())); ASSERT_TRUE(static_cast<bool>(f2.getDocumentDBConfig("typea"))); - ASSERT_TRUE(f1.configEqual("typea", f2.getDocumentDBConfig("typea"))); + ASSERT_TRUE(f1.configEqual("typea", *f2.getDocumentDBConfig("typea"))); // Remove and verify that config for db is no longer provided f2._configured = false; @@ -363,11 +370,45 @@ TEST_FF("require that lid space compaction is disabled for globally distributed ConfigTestFixture("search"), DocumentDBConfigManager(f1.configId + "/global", "global")) { - f1.addDocType("global", true); + f1.addDocType("global", ProtonConfig::Documentdb::Mode::INDEX, true); auto config = getDocumentDBConfig(f1, f2); EXPECT_TRUE(config->getMaintenanceConfigSP()->getLidSpaceCompactionConfig().isDisabled()); } +HwInfo +createHwInfoWithMemory(uint64_t mem) { + return {HwInfo::Disk(1, false, false), HwInfo::Memory(mem), HwInfo::Cpu(1)}; +} + +TEST_FF("require that target numdocs is fixed 1k for indexed mode", + ConfigTestFixture("search"), + DocumentDBConfigManager(f1.configId + "/test", "test")) +{ + f1.addDocType("test", ProtonConfig::Documentdb::Mode::INDEX, true); + for (uint64_t memory : {1_Gi, 10_Gi}) { + auto config = getDocumentDBConfig(f1, f2, createHwInfoWithMemory(memory)); + AllocStrategy strategy = config->get_alloc_config().make_alloc_strategy(SubDbType::READY); + EXPECT_EQUAL(1024u, strategy.get_grow_strategy().getMinimumCapacity()); + } +} + +TEST_FF("require that target numdocs follows memory for streaming mode", + ConfigTestFixture("search"), + DocumentDBConfigManager(f1.configId + "/test", "test")) +{ + f1.addDocType("test", ProtonConfig::Documentdb::Mode::STREAMING, true); + { + auto config = getDocumentDBConfig(f1, f2, createHwInfoWithMemory(1_Gi)); + AllocStrategy strategy = config->get_alloc_config().make_alloc_strategy(SubDbType::READY); + EXPECT_EQUAL(23342213u, strategy.get_grow_strategy().getMinimumCapacity()); + } + { + auto config = getDocumentDBConfig(f1, f2, createHwInfoWithMemory(10_Gi)); + AllocStrategy strategy = config->get_alloc_config().make_alloc_strategy(SubDbType::READY); + EXPECT_EQUAL(233422135u, strategy.get_grow_strategy().getMinimumCapacity()); + } +} + TEST_FF("require that prune removed documents interval can be set based on age", ConfigTestFixture("test"), DocumentDBConfigManager(f1.configId + "/test", "test")) @@ -383,7 +424,7 @@ TEST_FF("require that docstore config computes cachesize automatically if unset" ConfigTestFixture("test"), DocumentDBConfigManager(f1.configId + "/test", "test")) { - HwInfo hwInfo(HwInfo::Disk(1, false, false), HwInfo::Memory(1000000), HwInfo::Cpu(1)); + HwInfo hwInfo = createHwInfoWithMemory(1000000); f1.addDocType("test"); f1.protonBuilder.summary.cache.maxbytes = 2000; auto config = getDocumentDBConfig(f1, f2, hwInfo); @@ -400,7 +441,7 @@ TEST_FF("require that docstore config computes cachesize automatically if unset" GrowStrategy growStrategy(uint32_t initial) { - return GrowStrategy(initial, 0.1, 1, initial, 0.15); + return {initial, 0.1, 1, initial, 0.15}; } TEST_FF("require that allocation config is propagated", ConfigTestFixture("test"), |