diff options
author | Tor Egge <Tor.Egge@yahoo-inc.com> | 2017-04-21 15:13:05 +0000 |
---|---|---|
committer | Tor Egge <Tor.Egge@yahoo-inc.com> | 2017-04-24 09:28:11 +0000 |
commit | c9429079a48d8cc03039f811abd456e9865911b1 (patch) | |
tree | 90d12a7f72900ed429be12a19d7ed3d220ac1d89 /searchcore | |
parent | c2a6df2cd34f8a9f989d47f422939a7ab6874c91 (diff) |
Remove history schema and wiping of old removed fields.
Diffstat (limited to 'searchcore')
25 files changed, 51 insertions, 699 deletions
diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 75c37394e83..a0bb3cf8591 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -164,7 +164,7 @@ public: vespalib::string inputCfg = _baseDir + "/" + docType.toString() + "/baseconfig"; { FileConfigManager fileCfg(inputCfg, "", docType.getName()); - fileCfg.saveConfig(*snapshot, Schema(), 1); + fileCfg.saveConfig(*snapshot, 1); } config::DirSpec spec(inputCfg + "/config-1"); TuneFileDocumentDB::SP tuneFileDocDB(new TuneFileDocumentDB()); diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 5e3b576cf9c..63ec80e1f8a 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -139,7 +139,6 @@ struct MyResourceWriteFilter : public IResourceWriteFilter struct MyReplayConfig : public IReplayConfig { virtual void replayConfig(SerialNum) override {} - virtual void replayWipeHistory(SerialNum, fastos::TimeStamp) override {} }; void ackToken(FeedToken *token) { @@ -480,17 +479,13 @@ struct FeedHandlerFixture struct MyConfigStore : ConfigStore { virtual SerialNum getBestSerialNum() const override { return 1; } virtual SerialNum getOldestSerialNum() const override { return 1; } - virtual void saveConfig(const DocumentDBConfig &, - const search::index::Schema &, SerialNum) override {} + virtual void saveConfig(const DocumentDBConfig &, SerialNum) override {} virtual void loadConfig(const DocumentDBConfig &, SerialNum, - DocumentDBConfig::SP &, - search::index::Schema::SP &) override {} + DocumentDBConfig::SP &) override {} virtual void removeInvalid() override {} void prune(SerialNum) override {} virtual bool hasValidSerial(SerialNum) const override { return true; } virtual SerialNum getPrevValidSerial(SerialNum) const override { return 1; } - virtual void saveWipeHistoryConfig(SerialNum, - fastos::TimeStamp) override {} virtual void serializeConfig(SerialNum, vespalib::nbostream &) override {} virtual void deserializeConfig(SerialNum, vespalib::nbostream &) override {} virtual void setProtonConfig(const ProtonConfigSP &) override { } diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index 579e33250b3..4fd2f7f0332 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -32,31 +32,6 @@ using vespalib::nbostream; vespalib::string myId("myconfigid"); -namespace -{ - -DocumentDBConfig::SP -getConfig(int64_t generation, const Schema::SP &schema) -{ - return test::DocumentDBConfigBuilder(generation, schema, "client", "test").build(); -} - -Schema::SP -getSchema(int step) -{ - Schema::SP schema(new Schema); - schema->addIndexField(Schema::IndexField("foo1", schema::DataType::STRING)); - if (step < 2) { - schema->addIndexField(Schema::IndexField("foo2", schema::DataType::STRING)); - } - if (step < 1) { - schema->addIndexField(Schema::IndexField("foo3", schema::DataType::STRING)); - } - return schema; -} - - } - DocumentDBConfig::SP makeBaseConfigSnapshot() { @@ -79,19 +54,11 @@ makeBaseConfigSnapshot() return snap; } -Schema -makeHistorySchema() -{ - Schema hs; - hs.addIndexField(Schema::IndexField("history", schema::DataType::STRING)); - return hs; -} - void -saveBaseConfigSnapshot(const DocumentDBConfig &snap, const Schema &history, SerialNum num) +saveBaseConfigSnapshot(const DocumentDBConfig &snap, SerialNum num) { FileConfigManager cm("out", myId, snap.getDocTypeName()); - cm.saveConfig(snap, history, num); + cm.saveConfig(snap, num); } @@ -157,26 +124,22 @@ addConfigsThatAreNotSavedToDisk(const DocumentDBConfig &cfg) return builder.build(); } -TEST_FF("requireThatConfigCanBeSavedAndLoaded", DocumentDBConfig::SP(makeBaseConfigSnapshot()), - Schema(makeHistorySchema())) +TEST_F("requireThatConfigCanBeSavedAndLoaded", DocumentDBConfig::SP(makeBaseConfigSnapshot())) { - DocumentDBConfig::SP fullCfg = addConfigsThatAreNotSavedToDisk(*f1); - saveBaseConfigSnapshot(*fullCfg, f2, 20); + DocumentDBConfig::SP fullCfg = addConfigsThatAreNotSavedToDisk(*f); + saveBaseConfigSnapshot(*fullCfg, 20); DocumentDBConfig::SP esnap(makeEmptyConfigSnapshot()); - Schema::SP ehs; { FileConfigManager cm("out", myId, "dummy"); - cm.loadConfig(*esnap, 20, esnap, ehs); + cm.loadConfig(*esnap, 20, esnap); } - assertEqualSnapshot(*f1, *esnap); - EXPECT_TRUE(f2 == *ehs); + assertEqualSnapshot(*f, *esnap); } -TEST_FF("requireThatConfigCanBeSerializedAndDeserialized", DocumentDBConfig::SP(makeBaseConfigSnapshot()), - Schema(makeHistorySchema())) +TEST_F("requireThatConfigCanBeSerializedAndDeserialized", DocumentDBConfig::SP(makeBaseConfigSnapshot())) { - saveBaseConfigSnapshot(*f1, f2, 30); + saveBaseConfigSnapshot(*f, 30); nbostream stream; { FileConfigManager cm("out", myId, "dummy"); @@ -187,100 +150,32 @@ TEST_FF("requireThatConfigCanBeSerializedAndDeserialized", DocumentDBConfig::SP( cm.deserializeConfig(40, stream); } DocumentDBConfig::SP fsnap(makeEmptyConfigSnapshot()); - Schema::SP fhs; { FileConfigManager cm("out", myId, "dummy"); - cm.loadConfig(*fsnap, 40, fsnap, fhs); + cm.loadConfig(*fsnap, 40, fsnap); } - assertEqualSnapshot(*f1, *fsnap); - EXPECT_TRUE(f2 == *fhs); + assertEqualSnapshot(*f, *fsnap); EXPECT_EQUAL("dummy", fsnap->getDocTypeName()); } -TEST_FF("requireThatWipeHistoryCanBeSaved", DocumentDBConfig::SP(makeBaseConfigSnapshot()), - Schema(makeHistorySchema())) -{ - saveBaseConfigSnapshot(*f1, f2, 50); - { - FileConfigManager cm("out", myId, "dummy"); - cm.saveWipeHistoryConfig(60, 0); - } - DocumentDBConfig::SP gsnap(makeEmptyConfigSnapshot()); - Schema::SP ghs; - { - FileConfigManager cm("out", myId, "dummy"); - cm.loadConfig(*gsnap, 60, gsnap, ghs); - } - assertEqualSnapshot(*f1, *gsnap); - EXPECT_TRUE(f2 != *ghs); - EXPECT_TRUE(!f2.empty()); - EXPECT_TRUE(ghs->empty()); -} - - -TEST("require that wipe history clears only portions of history") -{ - FileConfigManager cm("out2", myId, "dummy"); - Schema::SP schema(getSchema(0)); - Schema::SP history(new Schema); - DocumentDBConfig::SP config(getConfig(5, schema)); - cm.saveConfig(*config, *history, 5); - Schema::SP oldSchema(schema); - schema = getSchema(1); - config = getConfig(6, schema); - history = SchemaUtil::makeHistorySchema(*schema, *oldSchema, *history, - 100); - cm.saveConfig(*config, *history, 10); - oldSchema = schema; - schema = getSchema(2); - config = getConfig(7, schema); - history = SchemaUtil::makeHistorySchema(*schema, *oldSchema, *history, - 200); - cm.saveConfig(*config, *history, 15); - cm.saveWipeHistoryConfig(20, 50); - cm.saveWipeHistoryConfig(25, 100); - cm.saveWipeHistoryConfig(30, 150); - cm.saveWipeHistoryConfig(35, 200); - cm.saveWipeHistoryConfig(40, 250); - DocumentDBConfig::SP oldconfig(config); - cm.loadConfig(*oldconfig, 20, config, history); - EXPECT_EQUAL(2u, history->getNumIndexFields()); - oldconfig = config; - cm.loadConfig(*oldconfig, 25, config, history); - EXPECT_EQUAL(2u, history->getNumIndexFields()); - oldconfig = config; - cm.loadConfig(*oldconfig, 30, config, history); - EXPECT_EQUAL(1u, history->getNumIndexFields()); - oldconfig = config; - cm.loadConfig(*oldconfig, 35, config, history); - EXPECT_EQUAL(1u, history->getNumIndexFields()); - oldconfig = config; - cm.loadConfig(*oldconfig, 40, config, history); - EXPECT_EQUAL(0u, history->getNumIndexFields()); -} - -TEST_FF("requireThatConfigCanBeLoadedWithoutExtraConfigsDataFile", DocumentDBConfig::SP(makeBaseConfigSnapshot()), - Schema(makeHistorySchema())) +TEST_F("requireThatConfigCanBeLoadedWithoutExtraConfigsDataFile", DocumentDBConfig::SP(makeBaseConfigSnapshot())) { - saveBaseConfigSnapshot(*f1, f2, 70); + saveBaseConfigSnapshot(*f, 70); EXPECT_TRUE(vespalib::unlink("out/config-70/extraconfigs.dat")); DocumentDBConfig::SP esnap(makeEmptyConfigSnapshot()); - Schema::SP ehs; { FileConfigManager cm("out", myId, "dummy"); - cm.loadConfig(*esnap, 70, esnap, ehs); + cm.loadConfig(*esnap, 70, esnap); } EXPECT_EQUAL(0u, esnap->getExtraConfigs().size()); } -TEST_FF("requireThatVisibilityDelayIsPropagated", - DocumentDBConfig::SP(makeBaseConfigSnapshot()), - Schema(makeHistorySchema())) +TEST_F("requireThatVisibilityDelayIsPropagated", + DocumentDBConfig::SP(makeBaseConfigSnapshot())) { - saveBaseConfigSnapshot(*f1, f2, 80); + saveBaseConfigSnapshot(*f, 80); DocumentDBConfig::SP esnap(makeEmptyConfigSnapshot()); - Schema::SP ehs; { ProtonConfigBuilder protonConfigBuilder; ProtonConfigBuilder::Documentdb ddb; @@ -292,7 +187,7 @@ TEST_FF("requireThatVisibilityDelayIsPropagated", using ProtonConfigSP = BootstrapConfig::ProtonConfigSP; cm.setProtonConfig( ProtonConfigSP(new ProtonConfig(protonConfigBuilder))); - cm.loadConfig(*esnap, 70, esnap, ehs); + cm.loadConfig(*esnap, 70, esnap); } EXPECT_EQUAL(0u, esnap->getExtraConfigs().size()); EXPECT_EQUAL(61.0, esnap->getMaintenanceConfigSP()->getVisibilityDelay().sec()); diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 4ed6953ca14..2605beadf3b 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -195,14 +195,12 @@ struct MySessionCachePruner : public ISessionCachePruner class MyFeedHandler : public IDocumentMoveHandler, public IPruneRemovedDocumentsHandler, public IHeartBeatHandler, - public IWipeOldRemovedFieldsHandler, public IOperationStorer { FastOS_ThreadId _executorThreadId; std::vector<MyDocumentSubDB *> _subDBs; SerialNum _serialNum; uint32_t _heartBeats; - fastos::TimeStamp _wipeTimeLimit; public: MyFeedHandler(FastOS_ThreadId &executorThreadId); @@ -221,9 +219,6 @@ public: virtual void heartBeat() override; - virtual void - wipeOldRemovedFields(TimeStamp wipeTimeLimit) override; - void setSubDBs(const std::vector<MyDocumentSubDB *> &subDBs); @@ -242,12 +237,6 @@ public: { return _heartBeats; } - - fastos::TimeStamp - getWipeTimeLimit() - { - return _wipeTimeLimit; - } }; @@ -499,7 +488,6 @@ public: newCfg(new DocumentDBMaintenanceConfig( pruneConfig, _mcCfg->getHeartBeatConfig(), - _mcCfg->getWipeOldRemovedFieldsConfig(), _mcCfg->getSessionCachePruneInterval(), _mcCfg->getVisibilityDelay(), _mcCfg->getLidSpaceCompactionConfig(), @@ -517,7 +505,6 @@ public: newCfg(new DocumentDBMaintenanceConfig( _mcCfg->getPruneRemovedDocumentsConfig(), heartBeatConfig, - _mcCfg->getWipeOldRemovedFieldsConfig(), _mcCfg->getSessionCachePruneInterval(), _mcCfg->getVisibilityDelay(), _mcCfg->getLidSpaceCompactionConfig(), @@ -529,32 +516,12 @@ public: } void - setWipeOldRemovedFieldsConfig(const DocumentDBWipeOldRemovedFieldsConfig &wipeConfig) - { - DocumentDBMaintenanceConfig::SP - newCfg(new DocumentDBMaintenanceConfig( - _mcCfg->getPruneRemovedDocumentsConfig(), - _mcCfg->getHeartBeatConfig(), - wipeConfig, - _mcCfg->getSessionCachePruneInterval(), - _mcCfg->getVisibilityDelay(), - _mcCfg->getLidSpaceCompactionConfig(), - _mcCfg->getAttributeUsageFilterConfig(), - _mcCfg->getAttributeUsageSampleInterval(), - _mcCfg->getResourceLimitFactor())); - _mcCfg = newCfg; - forwardMaintenanceConfig(); - } - - - void setGroupingSessionPruneInterval(double groupingSessionPruneInterval) { DocumentDBMaintenanceConfig::SP newCfg(new DocumentDBMaintenanceConfig( _mcCfg->getPruneRemovedDocumentsConfig(), _mcCfg->getHeartBeatConfig(), - _mcCfg->getWipeOldRemovedFieldsConfig(), groupingSessionPruneInterval, _mcCfg->getVisibilityDelay(), _mcCfg->getLidSpaceCompactionConfig(), @@ -570,7 +537,6 @@ public: newCfg(new DocumentDBMaintenanceConfig( _mcCfg->getPruneRemovedDocumentsConfig(), _mcCfg->getHeartBeatConfig(), - _mcCfg->getWipeOldRemovedFieldsConfig(), _mcCfg->getSessionCachePruneInterval(), _mcCfg->getVisibilityDelay(), cfg, @@ -790,8 +756,7 @@ MyFeedHandler::MyFeedHandler(FastOS_ThreadId &executorThreadId) _executorThreadId(executorThreadId), _subDBs(), _serialNum(0u), - _heartBeats(0u), - _wipeTimeLimit() + _heartBeats(0u) { } @@ -849,14 +814,6 @@ MyFeedHandler::heartBeat() void -MyFeedHandler::wipeOldRemovedFields(fastos::TimeStamp wipeTimeLimit) -{ - assert(isExecutorThread()); - _wipeTimeLimit = wipeTimeLimit; -} - - -void MyFeedHandler::setSubDBs(const std::vector<MyDocumentSubDB *> &subDBs) { _subDBs = subDBs; @@ -1000,7 +957,7 @@ void MaintenanceControllerFixture::injectMaintenanceJobs() { if (_injectDefaultJobs) { - MaintenanceJobsInjector::injectJobs(_mc, *_mcCfg, _fh, _gsp, _fh, + MaintenanceJobsInjector::injectJobs(_mc, *_mcCfg, _fh, _gsp, _lscHandlers, _fh, _mc, _docTypeName.getName(), _fh, _fh, _bmc, _clusterStateHandler, _bucketHandler, _calc, @@ -1224,38 +1181,6 @@ TEST_F("require that periodic session prunings are scheduled", ASSERT_TRUE(f._gsp.isInvoked); } -TEST_F("require that wipe old removed fields are scheduled", - MaintenanceControllerFixture) -{ - f.notifyClusterStateChanged(); - f.startMaintenance(); - TimeStamp now0 = TimeStamp(ClockSystem::now()); - f.setWipeOldRemovedFieldsConfig(DocumentDBWipeOldRemovedFieldsConfig(0.2, 100)); - TimeStamp now = TimeStamp(ClockSystem::now()); - TimeStamp expWipeTimeLimit = now - TimeStamp(100 * TimeStamp::SEC); - TimeStamp wtLim; - for (uint32_t i = 0; i < 600; ++i) { - FastOS_Thread::Sleep(100); - wtLim = f._fh.getWipeTimeLimit(); - if (wtLim.sec() != 0u) { - break; - } - } - TimeStamp now1 = TimeStamp(ClockSystem::now()); - double fuzz = now1.sec() - now0.sec(); - LOG(info, - "WipeOldRemovedFields: " - "now(%" PRIu64 "), " - "expWipeTimeLimit(%" PRIu64 "), " - "actWipeTimeLimit(%" PRIu64 "), " - "fuzz(%05.3f)", - (uint64_t)now.sec(), - (uint64_t)expWipeTimeLimit.sec(), - (uint64_t)wtLim.sec(), - fuzz); - EXPECT_APPROX(expWipeTimeLimit.sec(), wtLim.sec(), 4u + fuzz); -} - TEST_F("require that active bucket is not moved until de-activated", MaintenanceControllerFixture) { f._builder.createDocs(1, 1, 4); // 3 docs @@ -1477,13 +1402,13 @@ TEST_F("require that lid space compaction jobs can be disabled", MaintenanceCont f.forwardMaintenanceConfig(); { auto jobs = f._mc.getJobList(); - EXPECT_EQUAL(7u, jobs.size()); + EXPECT_EQUAL(6u, jobs.size()); EXPECT_TRUE(containsJob(jobs, "lid_space_compaction.my_handler")); } f.setLidSpaceCompactionConfig(DocumentDBLidSpaceCompactionConfig::createDisabled()); { auto jobs = f._mc.getJobList(); - EXPECT_EQUAL(6u, jobs.size()); + EXPECT_EQUAL(5u, jobs.size()); EXPECT_FALSE(containsJob(jobs, "lid_space_compaction.my_handler")); } } @@ -1492,10 +1417,9 @@ TEST_F("Require that maintenance jobs are run by correct executor", MaintenanceC { f.injectMaintenanceJobs(); auto jobs = f._mc.getJobList(); - EXPECT_EQUAL(6u, jobs.size()); + EXPECT_EQUAL(5u, jobs.size()); EXPECT_TRUE(containsJobAndExecutedBy(jobs, "heart_beat", f._threadService)); EXPECT_TRUE(containsJobAndExecutedBy(jobs, "prune_session_cache", f._genericExecutor)); - EXPECT_TRUE(containsJobAndExecutedBy(jobs, "wipe_old_removed_fields", f._threadService)); EXPECT_TRUE(containsJobAndExecutedBy(jobs, "prune_removed_documents.searchdocument", f._threadService)); EXPECT_TRUE(containsJobAndExecutedBy(jobs, "move_buckets.searchdocument", f._threadService)); EXPECT_TRUE(containsJobAndExecutedBy(jobs, "sample_attribute_usage.searchdocument", f._threadService)); diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index 9f8f7fd9e4a..baf5819fdaf 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -52,7 +52,6 @@ MyFeedView::~MyFeedView() {} struct MyReplayConfig : IReplayConfig { virtual void replayConfig(SerialNum) override {} - virtual void replayWipeHistory(SerialNum, fastos::TimeStamp) override {} }; struct InstantExecutor : vespalib::Executor { diff --git a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp index 8b21ed40fc8..8fc1bd2d9ed 100644 --- a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp +++ b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp @@ -33,65 +33,49 @@ getConfig(int64_t generation) } -Schema::SP -getSchema(int step) -{ - Schema::SP schema(new Schema); - schema->addIndexField(Schema::IndexField("foo1", DataType::STRING)); - if (step < 2) { - schema->addIndexField(Schema::IndexField("foo2", DataType::STRING)); - } - if (step < 1) { - schema->addIndexField(Schema::IndexField("foo3", DataType::STRING)); - } - return schema; -} - TEST("require that configs can be stored and loaded") { MemoryConfigStore config_store; SerialNum serial(12); - config_store.saveConfig(*getConfig(10), Schema(), serial); + config_store.saveConfig(*getConfig(10), serial); DocumentDBConfig::SP config; - Schema::SP history; - config_store.loadConfig(*getConfig(14), serial, config, history); + config_store.loadConfig(*getConfig(14), serial, config); ASSERT_TRUE(config.get()); - ASSERT_TRUE(history.get()); EXPECT_EQUAL(10, config->getGeneration()); } TEST("require that best serial number is the most recent one") { MemoryConfigStore config_store; EXPECT_EQUAL(0u, config_store.getBestSerialNum()); - config_store.saveConfig(*getConfig(10), Schema(), 5); + config_store.saveConfig(*getConfig(10), 5); EXPECT_EQUAL(5u, config_store.getBestSerialNum()); - config_store.saveConfig(*getConfig(10), Schema(), 2); + config_store.saveConfig(*getConfig(10), 2); EXPECT_EQUAL(5u, config_store.getBestSerialNum()); } TEST("require that oldest serial number is the first one or 0") { MemoryConfigStore config_store; EXPECT_EQUAL(0u, config_store.getOldestSerialNum()); - config_store.saveConfig(*getConfig(10), Schema(), 5); + config_store.saveConfig(*getConfig(10), 5); EXPECT_EQUAL(5u, config_store.getOldestSerialNum()); - config_store.saveConfig(*getConfig(10), Schema(), 2); + config_store.saveConfig(*getConfig(10), 2); EXPECT_EQUAL(2u, config_store.getOldestSerialNum()); } TEST("require that existing serial numbers are valid") { MemoryConfigStore config_store; EXPECT_FALSE(config_store.hasValidSerial(5)); - config_store.saveConfig(*getConfig(10), Schema(), 5); + config_store.saveConfig(*getConfig(10), 5); EXPECT_TRUE(config_store.hasValidSerial(5)); } TEST("require that prev valid serial number is the last one before the arg") { MemoryConfigStore config_store; EXPECT_EQUAL(0u, config_store.getPrevValidSerial(10)); - config_store.saveConfig(*getConfig(10), Schema(), 5); + config_store.saveConfig(*getConfig(10), 5); EXPECT_EQUAL(5u, config_store.getPrevValidSerial(10)); EXPECT_EQUAL(0u, config_store.getPrevValidSerial(5)); EXPECT_EQUAL(0u, config_store.getPrevValidSerial(4)); - config_store.saveConfig(*getConfig(10), Schema(), 2); + config_store.saveConfig(*getConfig(10), 2); EXPECT_EQUAL(0u, config_store.getPrevValidSerial(1)); EXPECT_EQUAL(0u, config_store.getPrevValidSerial(2)); EXPECT_EQUAL(2u, config_store.getPrevValidSerial(4)); @@ -101,8 +85,8 @@ TEST("require that prev valid serial number is the last one before the arg") { TEST("require that prune removes old configs") { MemoryConfigStore config_store; - config_store.saveConfig(*getConfig(10), Schema(), 5); - config_store.saveConfig(*getConfig(10), Schema(), 6); + config_store.saveConfig(*getConfig(10), 5); + config_store.saveConfig(*getConfig(10), 6); EXPECT_TRUE(config_store.hasValidSerial(5)); config_store.prune(5); EXPECT_FALSE(config_store.hasValidSerial(5)); @@ -111,82 +95,12 @@ TEST("require that prune removes old configs") { EXPECT_FALSE(config_store.hasValidSerial(6)); } -TEST("require that wipe history clears previous history schema " - "and adds new, identical entry for current serial num") { - MemoryConfigStore config_store; - Schema::SP history(new Schema); - history->addIndexField(Schema::IndexField("foo", DataType::STRING)); - config_store.saveConfig(*getConfig(10), *history, 5); - DocumentDBConfig::SP config; - config_store.loadConfig(*getConfig(14), 5, config, history); - EXPECT_EQUAL(1u, history->getNumIndexFields()); - config_store.saveWipeHistoryConfig(6, 0); - EXPECT_TRUE(config_store.hasValidSerial(6)); - config_store.loadConfig(*getConfig(14), 5, config, history); - EXPECT_EQUAL(1u, history->getNumIndexFields()); - config_store.loadConfig(*getConfig(14), 6, config, history); - ASSERT_TRUE(config.get()); - ASSERT_TRUE(history.get()); - EXPECT_EQUAL(0u, history->getNumIndexFields()); -} - - -TEST("require that wipe history clears only portions of history") -{ - MemoryConfigStore config_store; - Schema::SP schema(getSchema(0)); - Schema::SP history(new Schema); - DocumentDBConfig::SP config(getConfig(5, schema)); - config_store.saveConfig(*config, *history, 5); - Schema::SP oldSchema(schema); - schema = getSchema(1); - history = SchemaUtil::makeHistorySchema(*schema, *oldSchema, *history, - 100); - config_store.saveConfig(*config, *history, 10); - oldSchema = schema; - schema = getSchema(2); - history = SchemaUtil::makeHistorySchema(*schema, *oldSchema, *history, - 200); - config_store.saveConfig(*config, *history, 15); - config_store.saveWipeHistoryConfig(20, 50); - config_store.saveWipeHistoryConfig(25, 100); - config_store.saveWipeHistoryConfig(30, 150); - config_store.saveWipeHistoryConfig(35, 200); - config_store.saveWipeHistoryConfig(40, 250); - DocumentDBConfig::SP oldconfig(config); - config_store.loadConfig(*oldconfig, 20, config, history); - EXPECT_EQUAL(2u, history->getNumIndexFields()); - oldconfig = config; - config_store.loadConfig(*oldconfig, 25, config, history); - EXPECT_EQUAL(2u, history->getNumIndexFields()); - oldconfig = config; - config_store.loadConfig(*oldconfig, 30, config, history); - EXPECT_EQUAL(1u, history->getNumIndexFields()); - oldconfig = config; - config_store.loadConfig(*oldconfig, 35, config, history); - EXPECT_EQUAL(1u, history->getNumIndexFields()); - oldconfig = config; - config_store.loadConfig(*oldconfig, 40, config, history); - EXPECT_EQUAL(0u, history->getNumIndexFields()); -} - -TEST("require that wipe history does nothing if serial num exists") { - MemoryConfigStore config_store; - Schema::SP history(new Schema); - history->addIndexField(Schema::IndexField("foo", DataType::STRING)); - config_store.saveConfig(*getConfig(10), *history, 5); - DocumentDBConfig::SP config; - config_store.saveWipeHistoryConfig(5, 0); - config_store.loadConfig(*getConfig(14), 5, config, history); - EXPECT_EQUAL(1u, history->getNumIndexFields()); -} - TEST("require that MemoryConfigStores preserves state of " "MemoryConfigStore between instantiations") { MemoryConfigStores config_stores; const std::string name("foo"); ConfigStore::UP config_store = config_stores.getConfigStore(name); - config_store->saveConfig(*getConfig(10), Schema(), 5); + config_store->saveConfig(*getConfig(10), 5); EXPECT_TRUE(config_store->hasValidSerial(5)); config_store.reset(); config_store = config_stores.getConfigStore(name); diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 2bccad4f824..18c000e5f4e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -97,7 +97,6 @@ vespa_add_library(searchcore_server STATIC transactionlogmanagerbase.cpp updatedonecontext.cpp visibilityhandler.cpp - wipe_old_removed_fields_job.cpp DEPENDS searchcore_attribute searchcore_bucketdb diff --git a/searchcore/src/vespa/searchcore/proton/server/configstore.h b/searchcore/src/vespa/searchcore/proton/server/configstore.h index 38d0dcfe7bd..1c76cf2d4da 100644 --- a/searchcore/src/vespa/searchcore/proton/server/configstore.h +++ b/searchcore/src/vespa/searchcore/proton/server/configstore.h @@ -21,15 +21,11 @@ struct ConfigStore : FeedConfigStore { * @param serialNum the serial number of the config snapshot to load. * @param loadedSnapshot the shared pointer in which to store the * resulting config snapshot. - * @param historySchema the shared pointer in which to store the - * resulting history schema. */ virtual void loadConfig(const DocumentDBConfig ¤tSnapshot, SerialNum serialNum, - DocumentDBConfig::SP &loadedSnapshot, - search::index::Schema::SP &historySchema) = 0; + DocumentDBConfig::SP &loadedSnapshot) = 0; virtual void saveConfig(const DocumentDBConfig &snapshot, - const search::index::Schema &historySchema, SerialNum serialNum) = 0; virtual void removeInvalid() = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp index f3911c5e122..04e2557e2ca 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp @@ -99,8 +99,6 @@ DocumentDBMaintenanceConfig:: DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig & pruneRemovedDocuments, const DocumentDBHeartBeatConfig &heartBeat, - const DocumentDBWipeOldRemovedFieldsConfig & - wipeOldRemovedFields, double groupingSessionPruneInterval, fastos::TimeStamp visibilityDelay, const DocumentDBLidSpaceCompactionConfig &lidSpaceCompaction, @@ -109,7 +107,6 @@ DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig & double resourceLimitFactor) : _pruneRemovedDocuments(pruneRemovedDocuments), _heartBeat(heartBeat), - _wipeOldRemovedFields(wipeOldRemovedFields), _sessionCachePruneInterval(groupingSessionPruneInterval), _visibilityDelay(visibilityDelay), _lidSpaceCompaction(lidSpaceCompaction), @@ -125,7 +122,6 @@ operator==(const DocumentDBMaintenanceConfig &rhs) const { return _pruneRemovedDocuments == rhs._pruneRemovedDocuments && _heartBeat == rhs._heartBeat && - _wipeOldRemovedFields == rhs._wipeOldRemovedFields && _sessionCachePruneInterval == rhs._sessionCachePruneInterval && _visibilityDelay == rhs._visibilityDelay && _lidSpaceCompaction == rhs._lidSpaceCompaction && diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h index 199a7716dd1..583a676a01d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h @@ -23,7 +23,6 @@ public: }; typedef DocumentDBPruneConfig DocumentDBPruneRemovedDocumentsConfig; -typedef DocumentDBPruneConfig DocumentDBWipeOldRemovedFieldsConfig; class DocumentDBHeartBeatConfig { @@ -72,7 +71,6 @@ public: private: DocumentDBPruneRemovedDocumentsConfig _pruneRemovedDocuments; DocumentDBHeartBeatConfig _heartBeat; - DocumentDBWipeOldRemovedFieldsConfig _wipeOldRemovedFields; double _sessionCachePruneInterval; fastos::TimeStamp _visibilityDelay; DocumentDBLidSpaceCompactionConfig _lidSpaceCompaction; @@ -85,7 +83,6 @@ public: DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig &pruneRemovedDocuments, const DocumentDBHeartBeatConfig &heartBeat, - const DocumentDBWipeOldRemovedFieldsConfig &wipeOldRemovedFields, double sessionCachePruneInterval, fastos::TimeStamp visibilityDelay, const DocumentDBLidSpaceCompactionConfig &lidSpaceCompaction, @@ -108,12 +105,6 @@ public: return _heartBeat; } - const DocumentDBWipeOldRemovedFieldsConfig & - getWipeOldRemovedFieldsConfig() const - { - return _wipeOldRemovedFields; - } - double getSessionCachePruneInterval() const { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index c639b36b592..631e9d0ff18 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -102,7 +102,6 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, IFeedHandlerOwner(), IDocumentSubDBOwner(), IClusterStateChangedHandler(), - IWipeOldRemovedFieldsHandler(), search::transactionlog::SyncProxy(), _docTypeName(docTypeName), _baseDir(baseDir + "/" + _docTypeName.toString()), @@ -147,7 +146,6 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _writeFilter, *this, tlsDirectWriter), - _historySchema(), _subDBs(*this, *this, _feedHandler, @@ -185,14 +183,13 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, assert(configSerial > 0); DocumentDBConfig::SP loaded_config; _config_store->loadConfig(*configSnapshot, configSerial, - loaded_config, _historySchema); + loaded_config); // Grab relevant parts from pending config loaded_config = DocumentDBConfigScout::scout(loaded_config, *_pendingConfigSnapshot.get()); // Ignore configs that are not relevant during replay of transaction log loaded_config = DocumentDBConfig::makeReplayConfig(loaded_config); - reconfigureSchema(*loaded_config, *loaded_config); _initConfigSnapshot = loaded_config; _initConfigSerialNum = configSerial; // Forward changes of cluster state to feed view via us @@ -438,9 +435,6 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, cmpres = _activeConfigSnapshot->compare(*configSnapshot); } const ReconfigParams params(cmpres); - if (params.shouldSchemaChange()) { - reconfigureSchema(*configSnapshot, *_activeConfigSnapshot); - } // Save config via config manager if replay is done. bool equalReplayConfig = *DocumentDBConfig::makeReplayConfig(configSnapshot) == @@ -449,7 +443,7 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, if (!equalReplayConfig && tlsReplayDone) { sync(_feedHandler.getSerialNum()); serialNum = _feedHandler.incSerialNum(); - _config_store->saveConfig(*configSnapshot, *_historySchema, serialNum); + _config_store->saveConfig(*configSnapshot, serialNum); // save entry in transaction log NewConfigOperation op(serialNum, *_config_store); _feedHandler.storeOperation(op); @@ -502,19 +496,6 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, } -void -DocumentDB::reconfigureSchema(const DocumentDBConfig &configSnapshot, - const DocumentDBConfig &oldConfigSnapshot) -{ - // Called by CTOR and executor thread - const Schema &newSchema = *configSnapshot.getSchemaSP(); - const Schema &oldSchema = *oldConfigSnapshot.getSchemaSP(); - Schema::SP oldHistory = _historySchema; - _historySchema = - SchemaUtil::makeHistorySchema(newSchema, oldSchema, *oldHistory); -} - - namespace { void doNothing(IFeedView::SP) @@ -663,7 +644,7 @@ DocumentDB::saveInitialConfig(const DocumentDBConfig &configSnapshot) LOG(warning, "DocumentDB(%s): saveInitialConfig() failed pruning due to '%s'", _docTypeName.toString().c_str(), e.what()); } - _config_store->saveConfig(configSnapshot, Schema(), confSerial); + _config_store->saveConfig(configSnapshot, confSerial); } @@ -932,40 +913,6 @@ void DocumentDB::performWipeHistory() { // Called by executor thread - if (_historySchema->empty()) - return; - if (_feedHandler.getTransactionLogReplayDone()) { - sync(_feedHandler.getSerialNum()); // Sync before wiping history - DocumentDBConfig::SP configSnapshot = getActiveConfig(); - SerialNum wipeSerial = _feedHandler.incSerialNum(); - Schema::UP newHistory(new Schema); - writeWipeHistoryTransactionLogEntry(wipeSerial, 0, - *configSnapshot, *newHistory); - internalWipeHistory(wipeSerial, std::move(newHistory)); - } -} - - -void DocumentDB::writeWipeHistoryTransactionLogEntry( - SerialNum wipeSerial, fastos::TimeStamp wipeTimeLimit, - const DocumentDBConfig &configSnapshot, - const Schema &newHistorySchema) { - // Caller must have synced transaction log - _config_store->saveConfig(configSnapshot, newHistorySchema, wipeSerial); - // save entry in transaction log - WipeHistoryOperation op(wipeSerial, wipeTimeLimit); - _feedHandler.storeOperation(op); - sync(op.getSerialNum()); -} - - -void -DocumentDB::internalWipeHistory(SerialNum wipeSerial, - Schema::UP newHistorySchema) -{ - // Called by executor thread - _subDBs.wipeHistory(wipeSerial); - _historySchema.reset(newHistorySchema.release()); } @@ -980,10 +927,9 @@ DocumentDB::replayConfig(search::SerialNum serialNum) _docTypeName.toString().c_str(), serialNum); return; } - // Load historyschema before applyConfig to preserve the history - // field timestamps. + // Load config to replay _config_store->loadConfig(*configSnapshot, serialNum, - configSnapshot, _historySchema); + configSnapshot); // Grab relevant parts from pending config configSnapshot = DocumentDBConfigScout::scout(configSnapshot, *_pendingConfigSnapshot.get()); @@ -996,36 +942,6 @@ DocumentDB::replayConfig(search::SerialNum serialNum) } void -DocumentDB::replayWipeHistory(search::SerialNum serialNum, - fastos::TimeStamp wipeTimeLimit) -{ - // Called by executor thread - DocumentDBConfig::SP configSnapshot = getActiveConfig(); - if (configSnapshot.get() == NULL) { - LOG(warning, - "DocumentDB(%s): Missing old config when replaying wipe history, serialNum=%" PRIu64, - _docTypeName.toString().c_str(), - serialNum); - return; - } - Schema::UP wipeSchemaOwner; - Schema *wipeSchema; - Schema::UP newHistory; - if (wipeTimeLimit) { - wipeSchemaOwner = _historySchema->getOldFields(wipeTimeLimit); - wipeSchema = wipeSchemaOwner.get(); - newHistory = Schema::set_difference(*_historySchema, *wipeSchema); - } else { // wipeTimeLimit == 0 means old style wipeHistory. - wipeSchema = _historySchema.get(); - newHistory.reset(new Schema); - } - LOG(info, "DocumentDB(%s): Replayed history wipe with serialNum=%" PRIu64, - _docTypeName.toString().c_str(), serialNum); - internalWipeHistory(serialNum, std::move(newHistory)); -} - - -void DocumentDB::listSchema(std::vector<vespalib::string> &fieldNames, std::vector<vespalib::string> &fieldDataTypes, std::vector<vespalib::string> &fieldCollectionTypes, @@ -1087,7 +1003,6 @@ DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config) config, _feedHandler, // IHeartBeatHandler *_sessionManager, // ISessionCachePruner - *this, // IWipeOldRemovedFieldsHandler _lidSpaceCompactionHandlers, _feedHandler, // IOperationStorer _maintenanceController, // IFrozenBucketHandler @@ -1198,51 +1113,6 @@ DocumentDB::notifyAllBucketsChanged() } -namespace { - -vespalib::string -getSchemaFieldsList(const Schema &schema) -{ - vespalib::asciistream oss; - for (uint32_t i = 0; i < schema.getNumIndexFields(); ++i) { - if (i > 0) oss << ","; - oss << schema.getIndexField(i).getName(); - } - for (uint32_t i = 0; i < schema.getNumAttributeFields(); ++i) { - if (!oss.str().empty()) oss << ","; - oss << schema.getAttributeField(i).getName(); - } - return oss.str(); -} - -} - -void -DocumentDB::wipeOldRemovedFields(fastos::TimeStamp wipeTimeLimit) -{ - // Called by executor thread - - if (_historySchema->empty()) - return; - DocumentDBConfig::SP configSnapshot = getActiveConfig(); - - Schema::UP wipeSchema = _historySchema->getOldFields(wipeTimeLimit); - Schema::UP newHistorySchema = - Schema::set_difference(*_historySchema, *wipeSchema); - - sync(_feedHandler.getSerialNum()); // Sync before wiping history - SerialNum wipeSerial = _feedHandler.incSerialNum(); - writeWipeHistoryTransactionLogEntry(wipeSerial, wipeTimeLimit, - *configSnapshot, *newHistorySchema); - internalWipeHistory(wipeSerial, std::move(newHistorySchema)); - - LOG(debug, "DocumentDB(%s): Done wipeOldRemovedFields: wipe(%s), history(%s) timeLimit(%" PRIu64 ")", - _docTypeName.toString().c_str(), - getSchemaFieldsList(*wipeSchema).c_str(), - getSchemaFieldsList(*_historySchema).c_str(), - static_cast<uint64_t>(wipeTimeLimit.sec())); -} - searchcorespi::IIndexManagerFactory::SP DocumentDB::getIndexManagerFactory(const vespalib::stringref &name) const { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 3705c8d786d..fea0439aede 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -13,7 +13,6 @@ #include "i_lid_space_compaction_handler.h" #include "ifeedview.h" #include "ireplayconfig.h" -#include "iwipeoldremovedfieldshandler.h" #include "maintenancecontroller.h" #include "i_document_db_config_owner.h" #include "executorthreadingservice.h" @@ -65,7 +64,6 @@ class DocumentDB : public IDocumentDBConfigOwner, public IFeedHandlerOwner, public IDocumentSubDBOwner, public IClusterStateChangedHandler, - public IWipeOldRemovedFieldsHandler, public search::transactionlog::SyncProxy { private: @@ -133,11 +131,6 @@ private: AttributeUsageFilter _writeFilter; FeedHandler _feedHandler; - // Members only accessed by executor thread - // (+ ctor and after executor stops) - Schema::SP _historySchema; // Removed fields - // End members only accessed by executor thread. - DocumentSubDBCollection _subDBs; MaintenanceController _maintenanceController; VisibilityHandler _visibility; @@ -172,10 +165,6 @@ private: */ void resumeSaveConfig(void); - void - reconfigureSchema(const DocumentDBConfig &configSnapshot, - const DocumentDBConfig &oldConfigSnapshot); - void setIndexSchema(const DocumentDBConfig &configSnapshot, SerialNum serialNum); /** @@ -227,10 +216,6 @@ private: virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; void notifyAllBucketsChanged(); - /** - * Implements IWipeOldRemovedFieldsHandler - */ - virtual void wipeOldRemovedFields(TimeStamp wipeTimeLimit) override; void updateLegacyMetrics(LegacyDocumentDBMetrics &metrics); void updateMetrics(DocumentDBTaggedMetrics &metrics); void updateMetrics(DocumentDBTaggedMetrics::AttributeMetrics &metrics); @@ -430,8 +415,6 @@ public: */ virtual void replayConfig(SerialNum serialNum) override; - virtual void replayWipeHistory(SerialNum serialNum, TimeStamp wipeTimeLimit) override; - const DocTypeName & getDocTypeName(void) const { return _docTypeName; } void diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index a258a282f81..0006007caed 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -120,9 +120,6 @@ buildMaintenanceConfig(const BootstrapConfig::SP &bootstrapConfig, pruneRemovedDocumentsInterval, pruneRemovedDocumentsAge), DocumentDBHeartBeatConfig(), - DocumentDBWipeOldRemovedFieldsConfig( - proton.wipeoldremovedfieldsinterval, - proton.wipeoldremovedfieldsage), proton.grouping.sessionmanager.pruning.interval, visibilityDelay, DocumentDBLidSpaceCompactionConfig( diff --git a/searchcore/src/vespa/searchcore/proton/server/feedconfigstore.h b/searchcore/src/vespa/searchcore/proton/server/feedconfigstore.h index bfc653ac966..5320793d9ab 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedconfigstore.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedconfigstore.h @@ -9,8 +9,6 @@ namespace proton { struct FeedConfigStore : NewConfigOperation::IStreamHandler { virtual ~FeedConfigStore() {} - virtual void saveWipeHistoryConfig(search::SerialNum, - fastos::TimeStamp wipeTimeLimit) = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp index 2e79db1e599..516dd8f74cc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp @@ -91,11 +91,7 @@ public: virtual void replay(const NewConfigOperation &op) override { _replay_config.replayConfig(op.getSerialNum()); } - virtual void replay(const WipeHistoryOperation &op) override { - _config_store.saveWipeHistoryConfig(op.getSerialNum(), - op.getWipeTimeLimit()); - _replay_config.replayWipeHistory(op.getSerialNum(), - op.getWipeTimeLimit()); + virtual void replay(const WipeHistoryOperation &) override { } virtual void replay(const DeleteBucketOperation &op) override { _feed_view_ptr->handleDeleteBucket(op); diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index 07d8ffabb62..fe8f3210a51 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -298,7 +298,6 @@ FileConfigManager::getOldestSerialNum(void) const void FileConfigManager::saveConfig(const DocumentDBConfig &snapshot, - const search::index::Schema &historySchema, SerialNum serialNum) { if (getBestSerialNum() >= serialNum) { @@ -326,10 +325,6 @@ FileConfigManager::saveConfig(const DocumentDBConfig &snapshot, assert(saveSchemaRes); (void) saveSchemaRes; - bool saveHistorySchemaRes = historySchema.saveToFile(snapDir + "/historyschema.txt"); - assert(saveHistorySchemaRes); - (void) saveHistorySchemaRes; - writeExtraConfigs(snapDir, snapshot); _info.validateSnapshot(serialNum); @@ -361,8 +356,7 @@ void addEmptyFile(vespalib::string snapDir, vespalib::string fileName) void FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, search::SerialNum serialNum, - DocumentDBConfig::SP &loadedSnapshot, - search::index::Schema::SP &historySchema) + DocumentDBConfig::SP &loadedSnapshot) { vespalib::string snapDirBaseName(makeSnapDirBaseName(serialNum)); vespalib::string snapDir(_baseDir + "/" + snapDirBaseName); @@ -404,15 +398,9 @@ FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, dbc.forwardConfig(bootstrap); dbc.nextGeneration(0); - Schema::UP newHistorySchema(new Schema); - bool loadHistorySchemaRes = newHistorySchema->loadFromFile(snapDir + "/historyschema.txt"); - assert(loadHistorySchemaRes); - (void) loadHistorySchemaRes; - loadedSnapshot = dbc.getConfig(); loadedSnapshot->setConfigId(_configId); loadedSnapshot->setExtraConfigs(readExtraConfigs(snapDir)); - historySchema.reset(newHistorySchema.release()); } @@ -498,65 +486,6 @@ FileConfigManager::getPrevValidSerial(SerialNum serialNum) const void -FileConfigManager::saveWipeHistoryConfig(SerialNum serialNum, - fastos::TimeStamp wipeTimeLimit) -{ - vespalib::string snapDirBaseName(makeSnapDirBaseName(serialNum)); - vespalib::string snapDir(_baseDir + "/" + snapDirBaseName); - - if (hasValidSerial(serialNum)) - return; // config already saved. - - SerialNum prevSerialNum = getPrevValidSerial(serialNum); - - assert(prevSerialNum > 0); - - Snapshot snap(false, serialNum, snapDirBaseName); - _info.addSnapshot(snap); - - bool saveInvalidSnap = _info.save(); - assert(saveInvalidSnap); - (void) saveInvalidSnap; - - vespalib::mkdir(snapDir, false); - - vespalib::string prevSnapDirBaseName(makeSnapDirBaseName(prevSerialNum)); - vespalib::string prevSnapDir(_baseDir + "/" + prevSnapDirBaseName); - - std::vector<vespalib::string> configs = getFileList(prevSnapDir); - for (const auto &config : configs) { - if (config == "historyschema.txt") { - Schema::UP historySchema(new Schema); - if (wipeTimeLimit != 0) { - Schema oldHistorySchema; - bool loadOldHistorySchemaRes = - oldHistorySchema.loadFromFile(prevSnapDir + - "/historyschema.txt"); - assert(loadOldHistorySchemaRes); - (void) loadOldHistorySchemaRes; - Schema::UP wipeSchema; - wipeSchema = oldHistorySchema.getOldFields(wipeTimeLimit); - historySchema = Schema::set_difference(oldHistorySchema, - *wipeSchema); - } - bool saveHistorySchemaRes = - historySchema->saveToFile(snapDir + "/historyschema.txt"); - assert(saveHistorySchemaRes); - (void) saveHistorySchemaRes; - continue; - } - ConfigFile file(config, - prevSnapDir + "/" + config); - file.save(snapDir); - } - _info.validateSnapshot(serialNum); - bool saveValidSnap = _info.save(); - assert(saveValidSnap); - (void) saveValidSnap; -} - - -void FileConfigManager::serializeConfig(SerialNum serialNum, nbostream &stream) { vespalib::string snapDirBaseName(makeSnapDirBaseName(serialNum)); diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h index 3a108e866b3..eb8b1a8c690 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h @@ -40,7 +40,6 @@ public: virtual SerialNum getOldestSerialNum() const override; virtual void saveConfig(const DocumentDBConfig &snapshot, - const search::index::Schema &historySchema, SerialNum serialNum) override; /** @@ -53,13 +52,10 @@ public: * @param serialNum the serial number of the config snapshot to load. * @param loadedSnapshot the shared pointer in which to store the * resulting config snapshot. - * @param historySchema the shared pointer in which to store the - * resulting history schema. */ virtual void loadConfig(const DocumentDBConfig ¤tSnapshot, SerialNum serialNum, - DocumentDBConfig::SP &loadedSnapshot, - search::index::Schema::SP &historySchema) override; + DocumentDBConfig::SP &loadedSnapshot) override; virtual void removeInvalid() override; virtual void prune(SerialNum serialNum) override; @@ -68,14 +64,6 @@ public: virtual SerialNum getPrevValidSerial(SerialNum serialNum) const override; /** - * Clone config except for history schema. - * Used when wiping history. - */ - virtual void saveWipeHistoryConfig(SerialNum serialNum, - fastos::TimeStamp wipeTimeLimit) override; - - - /** * Serialize config files. * * Used for serializing config into transaction log. diff --git a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h index 866eeba4bea..6da62888208 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h @@ -15,10 +15,6 @@ public: virtual void replayConfig(search::SerialNum serialNum) = 0; - - virtual void - replayWipeHistory(search::SerialNum serialNum, - fastos::TimeStamp wipeTimeLimit) = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/iwipeoldremovedfieldshandler.h b/searchcore/src/vespa/searchcore/proton/server/iwipeoldremovedfieldshandler.h deleted file mode 100644 index 95f70ca308c..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/iwipeoldremovedfieldshandler.h +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -namespace proton { - -class IWipeOldRemovedFieldsHandler -{ -public: - virtual void wipeOldRemovedFields(fastos::TimeStamp wipeTimeLimit) = 0; - - virtual ~IWipeOldRemovedFieldsHandler() {} -}; - -} // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index 9217913e6a6..a1b23ea65c5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -9,7 +9,6 @@ #include "prune_session_cache_job.h" #include "pruneremoveddocumentsjob.h" #include "sample_attribute_usage_job.h" -#include "wipe_old_removed_fields_job.h" using fastos::ClockSystem; using fastos::TimeStamp; @@ -81,7 +80,6 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, const DocumentDBMaintenanceConfig &config, IHeartBeatHandler &hbHandler, matching::ISessionCachePruner &scPruner, - IWipeOldRemovedFieldsHandler &worfHandler, const ILidSpaceCompactionHandler::Vector &lscHandlers, IOperationStorer &opStorer, IFrozenBucketHandler &fbHandler, @@ -104,8 +102,6 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, if (config.getVisibilityDelay() > 0) { controller.registerJobInMasterThread(MUP(new DocumentDBCommitJob(commit, config.getVisibilityDelay()))); } - controller.registerJobInMasterThread( - MUP(new WipeOldRemovedFieldsJob(worfHandler, config.getWipeOldRemovedFieldsConfig()))); const MaintenanceDocumentSubDB &mRemSubDB(controller.getRemSubDB()); MUP pruneRDjob(new PruneRemovedDocumentsJob(config.getPruneRemovedDocumentsConfig(), *mRemSubDB._metaStore, mRemSubDB._subDbId, docTypeName, prdHandler, fbHandler)); diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h index dfad0947c8c..eff3442c49e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h @@ -6,7 +6,6 @@ #include "i_lid_space_compaction_handler.h" #include "i_operation_storer.h" #include "iheartbeathandler.h" -#include "iwipeoldremovedfieldshandler.h" #include "icommitable.h" #include <vespa/searchcore/proton/matching/isessioncachepruner.h> #include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h> @@ -34,7 +33,6 @@ struct MaintenanceJobsInjector const DocumentDBMaintenanceConfig &config, IHeartBeatHandler &hbHandler, matching::ISessionCachePruner &scPruner, - IWipeOldRemovedFieldsHandler &worfHandler, const ILidSpaceCompactionHandler::Vector &lscHandlers, IOperationStorer &opStorer, IFrozenBucketHandler &fbHandler, diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp index b070e2d8fc2..c687675e0d9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp @@ -33,21 +33,17 @@ MemoryConfigStore::getPrevValidSerial(SerialNum serial) const { } void MemoryConfigStore::saveConfig(const DocumentDBConfig &config, - const Schema &history, SerialNum serial) { _maps->configs[serial].reset(new DocumentDBConfig(config)); - _maps->histories[serial].reset(new Schema(history)); _maps->_valid.insert(serial); } void MemoryConfigStore::loadConfig(const DocumentDBConfig &, SerialNum serial, - DocumentDBConfig::SP &loaded_config, - Schema::SP &history_schema) + DocumentDBConfig::SP &loaded_config) { assert(hasValidSerial(serial)); loaded_config = _maps->configs[serial]; - history_schema = _maps->histories[serial]; } void MemoryConfigStore::removeInvalid() @@ -60,44 +56,16 @@ MemoryConfigStore::removeInvalid() } ++it; } - for (auto it = _maps->histories.begin(); - it != _maps->histories.end();) { - if (!hasValidSerial(it->first)) { - it = _maps->histories.erase(it); - continue; - } - ++it; - } } + void MemoryConfigStore::prune(SerialNum serial) { _maps->configs.erase(_maps->configs.begin(), _maps->configs.upper_bound(serial)); - _maps->histories.erase(_maps->histories.begin(), - _maps->histories.upper_bound(serial)); _maps->_valid.erase(_maps->_valid.begin(), _maps->_valid.upper_bound(serial)); } -void -MemoryConfigStore::saveWipeHistoryConfig(SerialNum serial, - fastos::TimeStamp wipeTimeLimit) -{ - if (hasValidSerial(serial)) { - return; - } - SerialNum prev = getPrevValidSerial(serial); - Schema::SP schema(new Schema); - if (wipeTimeLimit != 0) { - Schema::SP oldHistorySchema(_maps->histories[prev]); - Schema::UP wipeSchema; - wipeSchema = oldHistorySchema->getOldFields(wipeTimeLimit); - schema.reset(Schema::set_difference(*oldHistorySchema, *wipeSchema).release()); - - } - _maps->histories[serial] = schema; - _maps->configs[serial] = _maps->configs[prev]; - _maps->_valid.insert(serial); -} + void MemoryConfigStore::serializeConfig(SerialNum, vespalib::nbostream &) { LOG(info, "Serialization of config not implemented."); diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h index 57411c81e61..24d6e96106d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h +++ b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h @@ -12,7 +12,6 @@ namespace proton { struct ConfigMaps { typedef std::shared_ptr<ConfigMaps> SP; std::map<search::SerialNum, DocumentDBConfig::SP> configs; - std::map<search::SerialNum, search::index::Schema::SP> histories; std::set<search::SerialNum> _valid; }; @@ -32,14 +31,11 @@ public: virtual bool hasValidSerial(SerialNum serial) const override; virtual SerialNum getPrevValidSerial(SerialNum serial) const override; virtual void saveConfig(const DocumentDBConfig &config, - const Schema &history, SerialNum serial) override; virtual void loadConfig(const DocumentDBConfig &, SerialNum serial, - DocumentDBConfig::SP &loaded_config, - Schema::SP &history_schema) override; + DocumentDBConfig::SP &loaded_config) override; virtual void removeInvalid() override; void prune(SerialNum serial) override; - virtual void saveWipeHistoryConfig(SerialNum serial, fastos::TimeStamp wipeTimeLimit) override; virtual void serializeConfig(SerialNum, vespalib::nbostream &) override; virtual void deserializeConfig(SerialNum, vespalib::nbostream &) override; virtual void setProtonConfig(const ProtonConfigSP &) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.cpp b/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.cpp deleted file mode 100644 index dff3ff57375..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.cpp +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include "wipe_old_removed_fields_job.h" -#include <vespa/fastos/timestamp.h> - -using fastos::ClockSystem; -using fastos::TimeStamp; - -namespace proton { - -WipeOldRemovedFieldsJob::WipeOldRemovedFieldsJob(IWipeOldRemovedFieldsHandler &handler, - const DocumentDBWipeOldRemovedFieldsConfig &config) - : IMaintenanceJob("wipe_old_removed_fields", config.getInterval(), config.getInterval()), - _handler(handler), - _ageLimitSeconds(config.getAge()) -{ -} - -bool -WipeOldRemovedFieldsJob::run() -{ - TimeStamp wipeTimeLimit = TimeStamp(ClockSystem::now()) - - TimeStamp(int64_t(_ageLimitSeconds) * TimeStamp::SEC); - _handler.wipeOldRemovedFields(wipeTimeLimit); - return true; -} - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.h b/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.h deleted file mode 100644 index eb8ec4a7649..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "document_db_maintenance_config.h" -#include "i_maintenance_job.h" -#include "iwipeoldremovedfieldshandler.h" - -namespace proton { - -/** - * Job that regularly wipes old removed fields from a document database. - */ -class WipeOldRemovedFieldsJob : public IMaintenanceJob -{ -private: - IWipeOldRemovedFieldsHandler &_handler; - const double _ageLimitSeconds; - -public: - WipeOldRemovedFieldsJob(IWipeOldRemovedFieldsHandler &handler, - const DocumentDBWipeOldRemovedFieldsConfig &config); - - // Implements IMaintenanceJob - virtual bool run() override; -}; - -} // namespace proton - |